From 484caeea988ebc447d61caab6a65dea4e7fdeffd Mon Sep 17 00:00:00 2001 From: Stefan Kebekus Date: Fri, 14 Aug 2026 13:14:41 +0200 Subject: [PATCH] Corpus --- .claude/settings.json | 8 +- Plan-AZ-Tutor.md | 734 +++++++++++++++++++++++++ deploy.sh | 4 + tools/corpus/README.md | 125 +++++ tools/corpus/az_convert.py | 507 +++++++++++++++++ tools/corpus/az_indexes.py | 255 +++++++++ tools/corpus/az_prelude.tex | 115 ++++ tools/corpus/az_registry.py | 689 +++++++++++++++++++++++ tools/corpus/build-corpus.sh | 70 +++ tools/corpus/systemprompt-erklaeren.md | 80 +++ 10 files changed, 2586 insertions(+), 1 deletion(-) create mode 100644 Plan-AZ-Tutor.md create mode 100644 tools/corpus/README.md create mode 100644 tools/corpus/az_convert.py create mode 100644 tools/corpus/az_indexes.py create mode 100644 tools/corpus/az_prelude.tex create mode 100644 tools/corpus/az_registry.py create mode 100755 tools/corpus/build-corpus.sh create mode 100644 tools/corpus/systemprompt-erklaeren.md diff --git a/.claude/settings.json b/.claude/settings.json index acd5ab8..301358d 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -77,7 +77,13 @@ "Bash(/usr/bin/grep 'Index' AlgebraZahlentheorie.toc)", "Bash(qpdf --show-npages AlgebraZahlentheorie.pdf)", "Bash(/usr/bin/grep Pages)", - "Bash(sort -k2)" + "Bash(sort -k2)", + "Bash(curl -s -o /tmp/claude-1000/-home-kebekus-Mathe-Skript-AlgebraZahlentheorie/5bc6452b-d0cb-4458-9444-531c06b9a479/scratchpad/models.json -w \"HTTP %{http_code} size=%{size_download}\\\\n\" --max-time 25 https://openwebui.uni-freiburg.de/api/v1/models)", + "Bash(awk '{s+=$1}END{print s}')", + "Bash(python3 tools/corpus/az_registry.py)", + "Bash(pdftotext -layout AlgebraZahlentheorie.pdf -)", + "Bash(grep -n -B1 -E '^\\\\s*\\\\W{0,3}\\\\s*\\(13\\\\.2\\\\.3|15\\\\.3\\\\.8|21\\\\.0\\\\.5\\)\\\\s*[.\\(]')", + "Bash(python3 tools/corpus/az_convert.py)" ] } } diff --git a/Plan-AZ-Tutor.md b/Plan-AZ-Tutor.md new file mode 100644 index 0000000..c5be989 --- /dev/null +++ b/Plan-AZ-Tutor.md @@ -0,0 +1,734 @@ +# AZ-Tutor — KI-Tutor zum Skript „Algebra und Zahlentheorie" + +**Living document.** Work packages are independent and can be done in any order; +each names its dependencies. Update the `Status:` line in place as you go. The +Reference Appendix records everything already established, so a later session +does not have to re-research it. + +> **First action of this plan:** save this document into the repository as +> `Plan-AZ-Tutor.md` (next to the existing `Plan-for-script.md`), so it survives +> across sessions and is versioned with the material it describes. A memory note +> should point at it. + +--- + +## Context + +Homework no longer counts towards passing the course, because AI can solve it. +Rather than fight that, the aim is to make it a learning tool: a chatbot that + +1. explains the course material **as written in this Skript**, and +2. helps with exercises — above all with the thing students demonstrably + struggle with, **writing down a correct proof**. + +Freiburg's Rechenzentrum runs Open WebUI at +(since Oct 2025, UniAccount login). Users can create *Wissensspeicher* (RAG +knowledge bases) and *Modelle* (base model + system prompt + knowledge + +capabilities), and share them. No server, no container, no Ollama needed. + +What makes this worth doing is the Skript: 193 pages, 26 chapters, 79 sections, +**467 numbered statements**, a strict labelling convention, a curated index, and +59 `\video{n-m}` links to lecture recordings. A tutor that cites *these* theorem +numbers and page numbers and links *these* videos does something no generic +ChatGPT can do. + +**Decisions already taken** + +- **Three student-selectable bots:** *Erklären & Hinweise*, *Musterlösungen*, + *Beweis prüfen*. +- **Exercise sheets:** previous semesters' available now; new ones weekly during + term. Pipeline must ingest incrementally. +- **Delivery:** Open WebUI shared model, plus the OpenAI-compatible API so + students can use a harness like OpenCode. +- **Photos of handwritten work:** deliberately undecided — settled by + experiment E3 in WP4, not by guessing. + +--- + +## Reality check — is this realistic? + +Largely yes, but the three goals have very different odds. Being explicit here +is what stops the project from disappointing. + +### Will work well — build with confidence + +- **Explaining the material with exact references.** The corpus is small, very + well structured, and every statement has a canonical printed number and page. + *„Was besagt Satz 16.1.4?"*, *„Wo wird separabel eingeführt?"*, *„Warum + braucht der Hauptsatz die Normalität?"* are squarely what RAG does well. +- **Pointing into your own material.** Page numbers, video links, „lies dazu + Abschnitt 15.2". Nearly free once the corpus carries the metadata, and it is + the feature students will actually notice. +- **Notation bridging.** Students get lost between your notation and the + textbooks'. A bot that knows *your* conventions is genuinely useful. + +### Will work, but needs deliberate design — do not oversell + +- **Feedback on proofs.** The published evidence is sobering. + [QEDBench (2026)](https://arxiv.org/html/2602.20629) had LLMs judge + university-level proofs: human pass rate 67.7%, while GPT-5.2 Pro showed a + **38% leniency rate** (accepting flawed proofs) and Llama-4-Maverick approved + **74.8%** of logically broken ones. The dominant failure is *„hallucinated + rigor"* — authoritative-looking LaTeX with broken logical dependencies gets + waved through. The paper also finds **rubric instructions have negligible + effect** („model priors override written constraints"), so prompt engineering + alone will not fix it. + + Baked into the design: the bot **never certifies a proof as correct**; it + reports *specific, located doubts* and asks the student to justify named + steps; and the Merkblatt says so plainly. +- **Model choice matters most here — and the news is better than expected.** + The RZ's published guides describe GPT-OSS 120b as the flagship, which would + have been the weak link. The live model list shows much stronger local + options: **GLM 5.2 (735B, „reicht an Claude Opus 4.6 heran")** and **Qwen 3.5 + 397B**, both at 256k context and both accessible to every user. So the + data-protection-friendly path is no longer the compromise path. E2 still + decides on evidence, with an external model (`gpt-5.6-terra`) as the yardstick + — if the gap is small, stay local and save ~$3 000 a term. + +### Not worth it, or not yet + +- **Grading.** Forbidden by the Nutzungsbedingungen („Automatisierte Bewertung + von Prüfungsleistungen und Analyse von Verhalten Studierender sind nicht + zulässig") and unreliable anyway. Since homework no longer counts, nothing + here is a Prüfungsleistung — say so explicitly in the Merkblatt. +- **Hands-off critique of photographed work.** Documented failure: VLMs + **silently „fix" student errors** instead of transcribing them + ([arXiv 2604.22774](https://arxiv.org/abs/2604.22774)), destroying exactly the + mistakes you want found. In automated grading of handwritten maths, 87% of + errors trace to *transcription*, not to misapplying the rubric + ([arXiv 2605.19043](https://arxiv.org/abs/2605.19043)). Hence E3. +- **Speech for mathematics.** Dictating *„sei alpha ein Element von K adjungiert + beta"* is painful; TTS reading formulas aloud is unusable. Both already exist + in the Freiburg UI at **zero build cost** (microphone icon; voice-call button; + per-model TTS voice). Mention in the Merkblatt as accessibility, invest nothing. + +### One tension to decide early + +Old sheets *plus their Lösungsvorschläge* in the corpus, with new problems +„pretty similar", means the *Hinweise* bot has this year's answers in its pocket +— making the Socratic ladder theatre. **Recommendation:** old **problems** go in +the shared corpus; old **Lösungsvorschläge** go in a *separate* collection +attached only to *Musterlösungen*. + +--- + +## Work packages + +Dependencies are real; the order is not. WP1 and WP2 can run in parallel and +neither blocks the other. + +### WP0 — Persist this plan +**Status:** done (2026-08-13), except the commit · **Depends on:** nothing + +- [x] Write this document to `Plan-AZ-Tutor.md` in the repo root +- [x] Add a memory note pointing at it (so future sessions find it immediately) +- [ ] Commit — *your call, whenever it suits; nothing else depends on it* + +--- + +### WP1 — Clarify with the Rechenzentrum +**Status:** not started · **Depends on:** nothing · **Effort:** one email + wait + +Several load-bearing facts are admin-controlled and not publicly documented. +To : + +1. **Sharing.** May a non-admin set a custom model's visibility to *Public* + (instance-wide)? If not: can a group for course participants be created, or + is the ILIAS *KI-Assistent* module the intended route for teaching? +2. **Student access.** Do all students have accounts? The **KI-Basis-Schulung on + ILIAS is mandatory before first login** — real friction for a few hundred + students; must be in the Merkblatt. +3. ~~**Base models for students.**~~ **Answered** from `/api/v1/models`: nearly + every model grants `user:* read`, external OpenAI and Mistral included. Base + model access is not a constraint. Worth one confirming sentence only. +4. **API.** Is API-key generation enabled for non-admins (Settings → Account)? + Is `https://openwebui.uni-freiburg.de/api/chat/completions` reachable + off-campus? Rate limits? (This is the OpenCode path. Note `/api/v1/models` + answers 401 without auth, so the endpoint is at least reachable.) +5. ~~**Vision/OCR.**~~ **Largely answered:** two OCR-specialised local models + exist (`numarkdown-8b-thinking-llmlb`, `nuextract3-llmlb`), and most local + models are vision-capable. One thing worth reporting to them as a bug: the + *UFR: Standard Reasoning* preset has `capabilities.vision: false` while its + tags and description advertise vision. +6. **Wissensspeicher settings.** Max files and file size; and the configured + chunk size / overlap / embedding model / whether hybrid search + reranking + are on. **Ask to have chunk size raised to ~4000 with overlap ~300** — see + the chunking note in WP2. +7. **Quota and billing.** The model list publishes per-token prices (e.g. + `ufr/reasoning-complex` $0.10/$0.10, `gpt-5.6-terra` $2/$12 per 1M). **Who is + billed** — the institute, the RZ centrally, nobody? At ~63 000 queries a term + the difference between local and external is roughly $130 vs $3 300, so this + determines whether the external option is even on the table. Any rate cap? + +**Done when:** answers recorded in the Reference Appendix below. + +--- + +### WP2 — Build the corpus +**Status:** **done (2026-08-13)** — `./tools/corpus/build-corpus.sh`, wired into +`deploy.sh`; see `tools/corpus/README.md`. Details of what was actually built +and verified are in *WP2 — Ergebnis* directly below this section. +· **Depends on:** nothing + +This is the durable asset — it outlives any particular chatbot, model or +platform. **Do not upload the PDF**; PDF extraction mangles mathematics and the +RZ's own guide warns about it. Convert from LaTeX source. + +#### Architecture: Python drives, pandoc is the text engine + +This was settled empirically, not by preference. Pandoc passes the Unicode math +(`ℤ ℚ → α ∈ ·`) through **byte-identically** inside `$…$`, and handles +`align*`/`multline*`/matrices, description lists, tables, `\href`, `tikzcd` +(kept as display math) and `tikzpicture` (dropped cleanly) correctly — 511 KB +through it with zero errors. Reimplementing that in Python would be weeks of +work and worse. + +But naively it **silently deletes** theorem titles, `\video{…}`, `\schritt{…}` +and `\cite{…}`. Prepending a `\newcommand`/`\newenvironment` **macro prelude** +brings all of them back — pandoc honours those and expands macros inside math, +so `\factor{K[x]}{(f)}` becomes `K[x]/(f)`, which matters because KaTeX in Open +WebUI cannot render private macros. + +Everything number-, page-, label-, index-, video- and reference-related must be +Python, injected into the LaTeX **before** pandoc, because pandoc has no +counters, `.aux`/`.toc`/synctex are external, `\vref` vs `\ref` is already lost +in pandoc's AST, and footnotes must be inlined before splitting. + +Output format: `pandoc -f latex -t markdown_strict+tex_math_dollars+pipe_tables --wrap=none`. + +#### Files to create + +``` +tools/corpus/ + az_registry.py # scan sources, simulate counters, join with .aux/.toc/synctex → registry.json + az_prelude.tex # \newcommand/\newenvironment prelude prepended to each chapter + az_convert.py # per chapter: preprocess → pandoc → postprocess → corpus/ + az_indexes.py # the six 00-*.md auxiliary files + build-corpus.sh # orchestrator, with build-freshness assertions +``` + +`az_registry.py` must **hard-fail** if any of the 174 `.aux` cross-checks +mismatch or any `\ref` target is unresolvable. That self-check is what makes the +rest trustworthy. + +#### Preprocessing order (all line-anchored, all registry-driven) + +1. Drop `\selectlanguage`, `\sideremark`, `\questionSign`, `\todo`, + `\approvals`, `\Preprint`/`\Publication` (unwrap arg), `\qed`, `\qedhere`, + `\eqno\qed` — pandoc renders `\qed` as the literal string `0◻`. +2. Strip `\mathlib{…}` (193) and `\leanlink{…}` (12) everywhere — Lean and + Mathlib are **not** part of the course — and clean the resulting `", ]"`. + Note these live *inside theorem titles*, so this must precede title parsing. +3. Rewrite references to literal numbers: `\ref{X}` → `14.1.6`, + `\vref{X}` → `14.1.6 (S. 95)`, `\eqref`, `\ref*`. All 494 resolve, so there + is no fallback case. +4. **Inline `\footnote{…}`** as ` (Fußnote: …)`. Must precede pandoc, else + footnote bodies land at chapter end and are orphaned when splitting. +5. Substitute `\cite{K}` from the `.bbl` table (pandoc deletes these silently). +6. Harvest and delete `\index{…}` (brace-aware), buffered per statement. +7. Number `\item`s inside statements as `\item[(c.s.n.k)]` — the + `\setlist[enumerate]{label=(\thethm.\arabic*)}` numbering is not reproduced + by pandoc, and references point at those numbers. +8. Replace each theorem environment by a heading + metadata + keyword line; + each `\begin{proof}[…]` by `## Beweis von `. +9. Prepend the prelude, run pandoc, then postprocess: strip + `` label noise, normalise heading levels, split per section + (and at statement boundaries if >8000 chars), emit YAML + breadcrumb. + +Two brace/bracket-counting scanners are needed (a naive `[^}]*` regex breaks in +exactly two places): theorem optional arguments containing nested braces +(`\begin{satz}[Satz von Gauß\footnote{\href{…}{…}}, \mathlib{…}]`) and `\index` +keys with nested braces (`\index{*ap@$\left(\frac{a}{p}\right)$…|textbf}`). + +#### Chunking — the critical constraint + +**Open WebUI re-chunks whatever you upload** (default 1500 chars / 100 overlap, +splitting on `\n\n` first). So **file-level front matter does not reach the +chunks.** Two consequences: + +- Granularity: one file per **section** (~4–5 KB median), split at a statement + boundary if >8000 chars → ~115–125 files. Keeps each theorem with its proof + and motivation. +- **Identification must be inline and repeated**, not only in front matter: + - every statement: `## Satz 14.1.4 — Rechenregeln für die formale Ableitung` + - then a metadata line: `*Kapitel 14 · Abschnitt 14.1 · gedruckte Seite 95*` + - **every proof: `## Beweis von Satz 14.1.4`** — so a chunk containing only a + proof still names its theorem. This is the highest-value trick in the design. + - `\index` keys become a trailing `*Stichwörter: …*` line — recall boost and a + free keyword layer. + +#### Numbering and pages — solved for all 467 statements + +- Simulating the `thm` counter (which resets per section, shared by all + numbered environments) over the master's `\input` order yields **all 467** + numbers, and **all 174 that carry a label match `.aux` exactly**. Gotcha: + `00.tex` uses `\section*{Vorbemerkung}` and must not advance the chapter + counter. +- Pages: from `.aux` where a label exists, else from **synctex** (`synctex view + -i line:1:file -o pdf`). Verified 172/174 exact, 2 off-by-one at page breaks. + → 100% coverage, ±1 page in <1.5% of cases. + +#### Auxiliary files (generated, not hand-written) + +| File | Source | Purpose | +|---|---|---| +| `00-inhaltsverzeichnis.md` | `.toc` | parts/chapters/sections + pages; navigation questions | +| `00-verzeichnis-aussagen.md` | registry | all 467: Nr, Typ, Titel, Seite, Kurzinhalt, Video | +| `00-verzeichnis-aussagen-kompakt.md` | registry | same without Kurzinhalt, ~9k tokens — **paste into the system prompt** | +| `00-stichwortverzeichnis.md` | the 474 `\index` in source | *Begriff → Satznummer* (`.ind` only has pages) | +| `00-personen.md` | ~40 biography footnotes | „wer war Emmy Noether" becomes a hit | +| `00-literatur.md` | `.bbl` | 18 references + the `\cite` substitution table | + +`00-verzeichnis-aussagen-kompakt.md` is the **anti-hallucination device**: with +the complete inventory permanently in the system prompt, the bot always knows +which numbers exist, can answer navigation questions without retrieval, and +targets its retrieval far better. This is what makes E1's „zero invented theorem +numbers" bar reachable. + +Environment→German-name mapping matters: 61 `defn`, 106 `bsp`, 9 `lem`, +6 `rem`, 4 `prop`, 2 `obs` are English-named in the source but must be listed in +German (`defn`→Definition, `kor`/`cor`→Korollar, `bsp`→Beispiel, `rem`→Bemerkung, +`obs`→Beobachtung, …). Build the table from the `\newtheorem` declarations in the +master file and `gfx/stdPreamble.tex`, not by hand. + +#### Build integration + +Append to `deploy.sh` **after** `latexmk` — the corpus build depends on +`.aux`/`.toc`/`.synctex.gz`, so it must never run on a stale build (hence the +freshness assertions in `build-corpus.sh`): + +```bash +./tools/corpus/build-corpus.sh +mkdir -p public && zip -qr public/AlgebraZahlentheorie-korpus.zip corpus +``` + +Also: pin synctex in `.latexmkrc` +(`$pdflatex = 'pdflatex -synctex=1 -interaction=nonstopmode %O %S';`) — it +happens to be produced today, but the corpus build now depends on it. Add +`corpus/` and `tools/corpus/registry.json` to `.gitignore` (derived artefacts); +keep the scripts in git. + +*Optional convenience, your call:* the read-only commands (`pandoc`, +`synctex view`, `python3 tools/corpus/*.py`) could be added to +`.claude/settings.json` `permissions.allow` for prompt-free iteration. + +**Done when:** `build-corpus.sh` runs clean from a fresh `latexmk`, the registry +self-check passes, and a spot-check of 5 sections against the PDF looks right. + +--- + +### WP2 — Ergebnis (2026-08-13) + +Built as designed; the pandoc-hybrid architecture held up. What the build +produces and what it guarantees: + +- **103 files, 618 722 characters (~206k tokens)** — 96 section files plus + 7 generated indexes. Matches the pre-build estimate almost exactly. +- **Zero LaTeX residue, zero raw HTML** in the corpus. Verified by grep over + `\index`, `\mathlib`, `\ref`, `\footnote`, `\input`, `\qed`, … and over + ``/`
`/``. +- **All 467 statements carry their printed number and page.** Sources: 173 from + `.aux`, 292 from the PDF text, 2 from synctex. +- **The self-check hard-fails the build** on any inconsistency: 174/174 labelled + statement numbers and 68/68 labelled item numbers match `.aux`; all 494 + `\ref`/`\vref`/`\eqref` resolve (no `??` anywhere). +- **Text fidelity spot-checked against the PDF**: of the statements whose + opening is pure prose, 18/22 matched the printed text verbatim; every + exception traced to the checker (inlined footnotes, `**Z**erlegung`, + operator names living inside `$…$`), not to the corpus. + +Things worth knowing that only emerged while building: + +- **The counter simulation needed two corrections** the design did not + anticipate: a nested `itemize` inside an `enumerate` item must not advance + the item counter, and `00.tex`'s `\section*` must not advance the chapter + counter. Both are now pinned by the `.aux` regression test. +- **`\href` URLs contain percent-encoded characters** (`…/Kantenl%c3%a4nge%20…`). + A naive comment stripper eats them and destroys the braces, which then breaks + pandoc. URL arguments are now hidden before comment stripping. +- **`\input{figures/…}`** made pandoc try to load the file and abort. Now + replaced by a figure placeholder, like tikz. +- **Collapsing multi-line inline math must be parity-based, not regex-based** — + a regex pairs the *closing* `$` of one formula with the *opening* `$` of the + next and silently turns prose into mathematics. +- **10 statements print English type names** in a German Skript (`Remark` ×6, + `Observation` ×2, `Warning`, `Problem`), because the source uses `rem`, `obs`, + `warning`, `problem`. The corpus mirrors the printed name deliberately, so a + citation matches the page. Changing the source to `bemerkung`/`beobachtung`/ + `warnung` would remove the oddity — your call, it is cosmetic. + +Open item deferred from WP2: **Übungsblätter** (WP3) — nothing is ingested yet. + +--- + +### WP3 — Ingest the exercise sheets +**Status:** not started · **Depends on:** WP2 + +- [ ] Previous semesters' sheets → convert once (`pdftotext` + manual cleanup, + ~10–20 min each) or re-key as `.tex` +- [ ] Establish `tools/corpus/uebungen/blattNN.tex` for new sheets, same style, + converted by the same script +- [ ] **Problems** → the shared `AZ-Skript` collection +- [ ] **Lösungsvorschläge** → a *separate* collection, attached only to + *Musterlösungen* (see „One tension" above) + +--- + +### WP4 — Three pilot experiments +**Status:** not started · **Depends on:** WP2 for E1/E2; **E3 needs nothing** + +Run in **temporary chats**, not in a shared model. E3 is independent of all +other work and can be done first, on any idle afternoon. + +#### E1 — Retrieval quality (~1 h) · needs WP2 + +20 fixed questions with answers you already know, in five categories: exact +lookup (*„Was besagt Satz 16.1.4?"*), definition location (*„Wo wird separabel +eingeführt?"*), cross-section conceptual (*„Warum braucht der Hauptsatz die +Normalität?"*), example retrieval (*„Ein faktorieller, nicht euklidischer +Ring?"*), and **one trap** — something deliberately not in the Skript (*„Was +sagt der Satz von Wedderburn?"*), which must produce *„steht nicht im Skript"*. + +Score: correct statement, correct number, correct page, **and zero invented +theorem numbers**. That last is make-or-break: a bot citing a non-existent +„Satz 12.3.9" loses student trust permanently. + +Repeat per candidate base model, and with vs. without the compact statement +inventory in the system prompt, to confirm it does its job. + +#### E2 — Proof-critique quality (~2 h) · needs WP2 + +Six short student-style solutions: 3 sound, 3 each with **one specific planted +flaw** — a circular argument, an unjustified appeal to a later theorem, a +silently dropped separability hypothesis. All 6 to each candidate model with the +*Beweis prüfen* prompt. + +**Candidates (4 runs, 24 judgements):** `glm-5.2-llmlb` (strongest local), +`qwen-3.5-397b-llmlb` (local, vision-capable, cheapest), `gpt-oss-120b-llmlb` +(the RZ's own recommendation — worth including precisely to check whether it is +adequate), and `openai/gpt-5.6-terra-llmlb` as the external yardstick. The +question is not „which is best" but **„is the best local model close enough to +the external one to justify staying local"** — that is a ~$3 000/term decision. + +Score: does it find the planted flaw; does it **invent** flaws in the sound +ones; does it say „korrekt" when it must not. This measures the QEDBench +leniency effect on *your* material with *your* models. Single most informative +pre-launch action. + +#### E3 — Handwriting transcription (~2 h) · needs nothing · decides the photo question + +**Material:** write 8 short solutions **by hand yourself**. Do *not* use real +student work — personal data, and the Nutzungsbedingungen prohibit uploading it. +Vary neat/messy, pencil/pen, phone photo/flatbed scan, with and without a small +diagram. 4 sound, 4 with a planted error (wrong subgroup index, swapped +quantifier, an invalid „also ist G zyklisch", a missing case). + +**Prompt (one per image, nothing else):** +> Transkribiere den folgenden handschriftlichen Text vollständig und wörtlich. +> Korrigiere NICHTS, auch keine offensichtlichen Fehler. Gib Unleserliches als +> [unleserlich] wieder. + +**Metrics:** (1) symbol-level fidelity; (2) **the decisive one — over-correction: +in the 4 flawed samples, does the planted error survive into the transcript?** +(3) does it mark illegible parts instead of guessing? + +**Candidates — and a design insight.** The RZ runs two models built for exactly +this: `numarkdown-8b-thinking-llmlb` (document understanding) and +`nuextract3-llmlb` (image → structured Markdown). **Small, transcription-tuned +models over-correct *less* than large reasoning models**, because they are not +trained to fix what they read — which is precisely the property needed here. +Compare all four: the two OCR models, `qwen-3.5-397b-llmlb` (large local, vision) +and `openai/gpt-5.6-terra-llmlb` (external, vision). + +That suggests a **two-stage architecture** for *Beweis prüfen*: transcribe with a +faithful model, critique with a strong one. Since one Open WebUI model cannot +chain two base models, the practical form is a single vision-capable base +(`qwen-3.5-397b-llmlb` — local, 256k, reasoning, vision) driven through a forced +two-turn protocol: transcribe → student confirms → critique. Note `glm-5.2` has +`vision: false`, so the strongest local reasoner **cannot** take images at all — +if E2 crowns GLM 5.2 and E3 blesses photos, they are mutually exclusive in one +bot, and *Beweis prüfen* must either drop images or accept the second-best +reasoner. + +**Go/no-go:** if ≥3 of 4 planted errors survive in at least one available model, +ship photos *with* a transcript-confirm loop. If not, restrict *Beweis prüfen* +to typed input and allow photos only for the **problem statement**, where +over-correction is harmless. + +**Done when:** the go/no-go decision is written down in one paragraph here. + +--- + +### WP5 — Build the three models in Open WebUI +**Status:** not started · **Depends on:** WP1 (base model + sharing), WP2, WP4 + +All three share the `AZ-Skript` Wissensspeicher; *Musterlösungen* additionally +gets `AZ-Loesungsvorschlaege`. + +| Model | Base (provisional — E2/E3 confirm) | Key prompt behaviour | +|---|---|---| +| `AZ-Tutor – Erklären & Hinweise` | `ufr/chat-standard` (Gemma 4 31B, 256k, vision, local) | Socratic ladder: idea → relevant Satz → skeleton → full proof only on explicit demand. Highest volume, so cheapest adequate model wins. | +| `AZ-Tutor – Musterlösungen` | `ufr/reasoning-complex` (Qwen 3.5 397B, 256k, local, $0.10) | full model proof *plus* „warum dieser Schritt" annotations. Writing a correct proof needs reasoning; this is strong and nearly free. | +| `AZ-Tutor – Beweis prüfen` | `glm-5.2-llmlb` if text-only, else `qwen-3.5-397b-llmlb` | transcript-confirm loop if E3 allows photos; *located* doubts; **never certifies correctness**. See the vision/reasoning conflict noted in E3. | + +Build on the **raw base models**, not on the RZ presets — presets carry their own +system prompts and capability toggles (one of them has vision wrongly disabled). + +Per model in *Arbeitsbereich → Modelle*: name, model id (needed for the API), +base model (from E2), description, tags, **Visibility**, system prompt, +**Knowledge** binding, Vision capability only where needed, 4 German prompt +suggestions, temperature ~0.2–0.3. + +Rules common to all three system prompts: +- Answer in German; follow the Skript's notation. +- Include `00-verzeichnis-aussagen-kompakt.md` verbatim. +- **Cite only theorem numbers from that inventory or the retrieved context.** + If unsure, name the section instead. +- If it is not in the Skript, say *„das steht nicht im Skript"* first, then + answer from general knowledge, clearly marked. +- Offer the `\video` link where one exists for the relevant section. +- Lean/Mathlib are **not** part of the course — never mention them. +- The Skript is the authority; the bot can be wrong. + +--- + +### WP6 — Rollout and student documentation +**Status:** not started · **Depends on:** WP5 + +- [ ] **Merkblatt (1 page, German, PDF):** which bot for what; that the + KI-Basis-Schulung comes first; that the bot never grades and never + certifies a proof; that wrong theorem numbers can happen and how to check; + dictation/read-aloud as accessibility; data protection (no personal data, + no real names on scans) +- [ ] **Subsection in the Skript** under the existing + `\addchap{Offenlegung / KI Einsatz}` — natural fit, no new structure +- [ ] **API / OpenCode instructions:** personal API key (Settings → Account), + harness pointed at `https://openwebui.uni-freiburg.de/api`, model id + `az-tutor-erklaeren`. Use the `/api/...` base — it supplements the request + with the custom model's system prompt, parameters and knowledge, whereas + the `/ollama/...` passthrough **bypasses** all of it +- [ ] Announce in the first lecture + ILIAS + +**Fallback if sharing is blocked (WP1 answer):** publish corpus + system prompts +to `cplx.vm.uni-freiburg.de:/var/www/storage/az-tutor/` — the same server and +rsync path `AlgebraInLean/deploy.sh` already uses — with a one-page setup guide +so each student builds their own Wissensspeicher. + +--- + +### WP7 — Maintenance during term +**Status:** not started · **Depends on:** WP6 + +- [ ] Weekly: new Übungsblatt → `build-corpus.sh` → replace the file in the + Wissensspeicher (should be one command) +- [ ] Re-index after file changes; note that changing the embedding model forces + a full re-index +- [ ] Feedback channel — one ILIAS forum thread. The bot's own chats are + invisible to you, so without this you learn nothing +- [ ] Re-run E1/E2 once mid-term, especially after the RZ swaps model versions + +--- + +## Verification + +1. **Corpus fidelity.** 5 random sections vs. the PDF page: all numbered + statements present, correctly numbered, math intact, no + `\index`/`\mathlib`/`\todo`/`0◻` leakage. +2. **Numbering integrity.** `az_registry.py`'s self-check passes (174/174 + `.aux` matches, 494/494 references resolved); grep the corpus for surviving + `\ref`, `\vref`, `??`. +3. **Retrieval.** E1 re-run against the finished shared model. Bar: ≥18/20 + substantively correct, **0 invented theorem numbers**, trap correctly refused. +4. **Proof critique.** E2 re-run against *Beweis prüfen*. Bar: all 3 planted + flaws located; no „korrekt" verdict on any of the 6. +5. **API path.** + ```bash + curl -H "Authorization: Bearer $KEY" -H 'Content-Type: application/json' \ + https://openwebui.uni-freiburg.de/api/chat/completions \ + -d '{"model":"az-tutor-erklaeren", + "messages":[{"role":"user","content":"Was besagt Satz 16.1.4?"}]}' + ``` + Must cite the correct Satz — proving system prompt *and* Wissensspeicher + apply through the API, not just in the web UI. +6. **Student view.** A second account confirms all three models appear in the + dropdown and answer correctly. + +--- + +## Reference Appendix — established facts + +*So later sessions need not re-research this.* + +### Freiburg Open WebUI + +- URL , UniAccount login. Introduced Oct 2025, + replaced HAWKI. Support ; ILIAS support room + `ilias.uni-freiburg.de/openwebui-support`. +- **KI-Basis-Schulung on ILIAS is mandatory before first use**, plus accepting + the Nutzungsbedingungen at first login. +- Model inventory: see the dedicated section below (fetched from + `/api/v1/models`, far newer than the RZ's published guides). +- Everything prefixed `UFR` / `ufr/` runs at the university (data stays); + external models are OpenAI (USA) and Mistral (France) via the university's LLM + proxy — data leaves the university. +- `+` menu: Datei(en) hochladen · Aufzeichnen · Webseite anhängen · Notizen + anhängen · **Wissensspeicher anhängen** · Chats referenzieren. Also web search, + code interpreter, image generation. Microphone icon = dictation; voice-call + button = spoken conversation. +- *Arbeitsbereich* → *Wissen* (create Wissensspeicher, attach with `#`) and + → *Modelle*. Model editor offers name, id, base model, description, tags, + **Visibility (Private/Public)**, system prompt with `{{USER_NAME}}` etc., + Knowledge binding, Tools, Vision/Web Search/Code Interpreter toggles, per-model + TTS voice, prompt suggestions, temperature. **A custom model always requires + access to its base model.** +- *Ordner* in the sidebar can also carry their own system prompt + knowledge — + a lighter-weight alternative to a custom model. +- Nutzungsbedingungen: personal data not permitted (research exceptions); + **automated assessment of examination performance and analysis of student + behaviour explicitly prohibited**; inputs are not used as training data; + uploads stored in a vector database; cross-unit knowledge-base sharing needs a + legal basis. +- ILIAS has a separate **KI-Assistenten** module letting lecturers embed + configurable chatbots in their course room — the alternative delivery channel + if model sharing turns out to be admin-only. +- API: key from Settings → Account (admin-toggleable per group). + `POST /api/chat/completions` applies custom-model config; `/ollama/...` + bypasses it. Collections can be attached per request via + `"files":[{"type":"collection","id":"…"}]`. + +### Available models — fetched from `/api/v1/models`, 2026-08-13 + +**Access is not a constraint.** Almost every entry carries an access grant +`principal_type: user, principal_id: "*", permission: read` — i.e. **all users, +students included, can read essentially every model**, the external OpenAI and +Mistral ones included. The RZ presets additionally grant read+write to one group +(presumably RZ staff). This answers WP1 Q3. + +**Build on the raw base models, not on the RZ presets.** The presets +(`preset: true`) carry their own system prompts and capability toggles, and at +least one is inconsistent: *UFR: Standard Reasoning* has +`capabilities.vision: false` even though its tags say `Vision` and its +description advertises „sehr gute Bildanalysefähigkeiten". The underlying +`ufr/reasoning-complex` does support vision. Layering on a preset means +inheriting such surprises. + +**RZ presets** — what students see by default: + +| Preset | Base | Model | Context | Vision | +|---|---|---|---|---| +| UFR: Standard Chat | `ufr/chat-standard` | Gemma 4 31B | 256k | yes | +| UFR: Standard Reasoning | `ufr/reasoning-complex` | Qwen 3.5 397B | 256k | flag says no (see above) | +| UFR: Standard Coding | `ufr/coding-complex` | GLM 5.2 | 256k | no | +| UFR: Standard Vision | `ufr/vision-standard` | Gemma 4 | — | yes | + +**Local models** (`connection_type: local`, data stays at the university): + +| ID | Model | Context | Vision | Cost /1M in-out | +|---|---|---|---|---| +| `glm-5.2-llmlb` | GLM 5.2, 735B (40B active) — „reicht an Claude Opus 4.6 heran" | 256k | **no** | $0.40 / $0.40 | +| `qwen-3.5-397b-llmlb` | Qwen 3.5, 397B (17B active), reasoning | 256k | yes | — | +| `ufr/reasoning-complex` | same family, API alias | 256k | yes | $0.10 / $0.10 | +| `gemma-4-31b-llmlb` | Gemma 4 31B | 256k | yes | — | +| `qwen-3.6-27b-llmlb` | Qwen 3.6 27B | 256k | yes | — | +| `gpt-oss-120b-llmlb` | GPT-OSS 120B — the RZ's recommended default | 128k | no | — | +| `mistral-small-4-llmlb` | Mistral Small 4, 119B | — | yes | — | +| `numarkdown-8b-thinking-llmlb` | NuMarkdown 8B Thinking — document understanding / OCR | — | yes | — | +| `nuextract3-llmlb` | nuExtract3 (Qwen 3.5 4B) — image → structured Markdown/JSON | — | yes | — | +| `gemma-4-12b-llmlb`, `gemma-3-27b-llmlb`, `qwen3.5-9b-llmlb` | smaller general models | 128k (Gemma 3) | yes | — | + +**External** (`connection_type: external`, data leaves the university): + +| ID | Model | Cost /1M in-out | Vision | +|---|---|---|---| +| `openai/gpt-5.6-sol-llmlb` | GPT 5.6 Sol — strongest | $5.00 / $30.00 | yes | +| `openai/gpt-5.6-terra-llmlb` | GPT 5.6 Terra — „Spitzenleistungen … Mathematik" | $2.00 / $12.00 | yes | +| `openai/gpt-5.6-llmlb` | GPT 5.6 Luna — smallest of the 5.6 family | $0.20 / $1.20 | yes | +| `openai/gpt-5.4-llmlb`, `openai/gpt-5.3-codex-llmlb` | being retired in favour of 5.6 | — | yes | +| `mistral/mistral-large-latest-llmlb` | Mistral Large | — | no | +| `mistral/codestral-latest-llmlb` | Codestral | — | no | + +**API aliases** (tag `API`) — stable names that survive model swaps underneath, +so these are the right targets for the OpenCode path: `ufr/chat`, +`ufr/chat-fast`, `ufr/chat-standard`, `ufr/text`, `ufr/reasoning-fast`, +`ufr/reasoning-complex`, `ufr/vision-fast`, `ufr/vision-standard`, +`ufr/vision-complex`, `ufr/coding-fast`, `ufr/coding-complex`. + +**Cost at course scale.** Assume ~20k input per query (12k always-on inventory + +~6k retrieved chunks + history) and ~1k output; 300 students × 15 queries/week × +14 weeks ≈ 63 000 queries: + +| Base model | Term cost (order of magnitude) | +|---|---| +| `ufr/reasoning-complex` @ $0.10/$0.10 | **~$130** | +| `glm-5.2-llmlb` @ $0.40/$0.40 | ~$530 | +| `openai/gpt-5.6-terra` @ $2/$12 | **~$3 300** | +| `openai/gpt-5.6-sol` @ $5/$30 | ~$8 200 | + +Two consequences: **default to local**, and note that the always-on statement +inventory is ~60% of the input cost. If cost ever bites, moving the inventory +out of the system prompt and into the knowledge base is the lever — at the price +of weaker hallucination protection. Confirm with the RZ who is actually billed +(WP1 Q7). + +### The Skript + +- 26 files `00.tex`–`25.tex`, 10 538 lines, 511 933 B. PDF 193 pages, 1.4 MB. + 8 `\part`s, 25 numbered chapters, 79 sections, 23 subsections. +- **467 numbered statements** across 51 distinct environment names, all sharing + the `thm` counter (resets per section). +- 296 `\label`, 494 `\ref`/`\vref`/`\eqref` (all resolve), 476 `\index`, + 59 `\video`, 193 `\mathlib`, 12 `\leanlink`, 71 `\footnote`, 18 `\cite`. +- `.aux` `\newlabel` grammar: `{LABEL}{{PRINTED}{PAGE}{TITLE}{ANCHOR.NUM}{}}` + — 296 five-field entries (varioref adds 360 two-field ones; filter by field + count). `.toc` is one regex, 138 lines, 100% parseable. +- Source is remarkably regular: **zero** cases of `\begin` off column 0, + multi-line optional arguments, nested theorem environments, multi-line + `\index`, or unbalanced environments. +- Only two places need brace-counting rather than regex: theorem optional + arguments with nested braces, and `\index` keys with nested braces. +- Exercises are essentially absent from the notes (the `aufgabe` environment is + used twice, both jokes). Sheets must come from outside the repo. +- Repo: `git@git.cplx.vm.uni-freiburg.de:kebekus/AlgebraZahlentheorie.git`; + submodules `bibliography/`, `tex/`. `deploy.sh` = `latexmk --pdf` + `cp` to + `public/` (which does not currently exist). Videos live at + `cplx.vm.uni-freiburg.de/storage/video/az/`; `AlgebraInLean/deploy.sh` already + rsyncs to `/var/www/storage` on that host. + +### Corpus size + +| Measurement | Value | +|---|---| +| `.tex` chapters | 511 933 B | +| Markdown after pandoc | 478 098 B (~449 600 chars) | +| Auxiliary files (est.) | ~60 000 chars | +| **Total corpus** | **~510 000 chars** | +| chars/4 estimate | 128k tokens | +| **Realistic (German + inline LaTeX, ~2.5–3.0 chars/token)** | **170k–205k tokens** | + +**Revised 2026-08-13, after seeing the actual model list.** The earlier +conclusion („does not fit a 128k model, RAG is mandatory") was based on the RZ's +published guides, which are out of date. Several models offer **256k context** +(`ufr/reasoning-complex`, `glm-5.2-llmlb`, `gemma-4-31b-llmlb`, +`qwen-3.6-27b-llmlb`), so the whole corpus at 170–205k tokens **does** fit as +full context, leaving 50–85k for the conversation. + +That does not make RAG obsolete — every query would carry ~200k input tokens, +which is slow and, on a metered model, expensive. But it hands us something +valuable: **a full-context oracle**. Run E1 once in full-context mode to +establish the ceiling, then measure how much retrieval gives away. That is a far +stronger experiment than judging RAG answers in isolation. + +Design therefore unchanged, now for cost and latency rather than necessity: +keep `00-inhaltsverzeichnis.md` (~3k) plus +`00-verzeichnis-aussagen-kompakt.md` (~9k) permanently in the system prompt, and +retrieve the ~120 section files on demand. A single chapter (max ~44 KB ≈ +15–18k tokens) fits easily, so „expand to whole chapter" is a viable strategy +for deep questions — and with 256k available, „expand to whole *Part*" is too. + +### Local tooling + +pandoc 3.7.0.2, python3 3.14.6, podman 5.8.4, rsync, latexmk, upmendex, +pdftotext/pdftoppm, synctex. No docker, no Ollama, no containers, no LLM Python +packages. GPU is an integrated Intel Arc — local inference is not a realistic +path, which is fine because the RZ hosts the models. diff --git a/deploy.sh b/deploy.sh index ec2ffe3..1a2c7ba 100755 --- a/deploy.sh +++ b/deploy.sh @@ -3,3 +3,7 @@ set -e latexmk --pdf AlgebraZahlentheorie.tex cp AlgebraZahlentheorie.pdf public/AlgebraZahlentheorie.pdf + +# Korpus für den AZ-Tutor. Muss NACH latexmk laufen: Nummern, Seiten und +# Querverweise kommen aus .aux/.toc/.synctex.gz und dem frisch gebauten PDF. +./tools/corpus/build-corpus.sh diff --git a/tools/corpus/README.md b/tools/corpus/README.md new file mode 100644 index 0000000..23016f0 --- /dev/null +++ b/tools/corpus/README.md @@ -0,0 +1,125 @@ +# AZ-Tutor — Korpus und Prototyp + +Erzeugt aus den LaTeX-Quellen des Skriptes einen Markdown-Korpus, den man in +Open WebUI () als *Wissensspeicher* hochlädt. +Gesamtplan: `Plan-AZ-Tutor.md` im Wurzelverzeichnis. + +## Bauen + +```bash +latexmk --pdf AlgebraZahlentheorie.tex # muss zuerst laufen +./tools/corpus/build-corpus.sh +``` + +`deploy.sh` ruft den Korpusbau bereits auf. Die Reihenfolge ist nicht +verhandelbar: Nummern, Seiten und Querverweise stammen aus `.aux`, `.toc`, +`.synctex.gz` und dem gebauten PDF. Läuft der Korpusbau auf einem veralteten +Stand, wird der Korpus still und leise falsch — deshalb warnt +`build-corpus.sh`, wenn die Hilfsdateien älter sind als die Quellen. + +## Was entsteht + +``` +corpus/ (abgeleitet, nicht in git) + 00-vorbemerkung.md + 00-inhaltsverzeichnis.md Navigation, Seitenzahlen + 00-verzeichnis-aussagen.md alle 467 Aussagen + Kurzinhalt + 00-verzeichnis-aussagen-kompakt.md dieselbe Liste für den Prompt + 00-stichwortverzeichnis.md Begriff → Satznummer + 00-videos.md 59 Erklärvideos → Aussage + 00-personen.md Kurzbiographien aus den Fußnoten + 00-literatur.md + kapitel-01/ … kapitel-25/ eine Datei je Abschnitt +tools/corpus/systemprompt-erklaeren-komplett.md fertiger System-Prompt +``` + +## Prototyp in Open WebUI einrichten + +1. **Wissensspeicher anlegen.** Arbeitsbereich → *Wissen* → `+`. Alle Dateien + aus `corpus/` hochladen (96 Abschnitts- und 7 Verzeichnisdateien). + + *Name:* `AZ-Skript — Algebra und Zahlentheorie` + + *Beschreibung* (Open WebUI zeigt sie dem Modell mit, wenn der Speicher + angehängt ist — deshalb lohnt sich der letzte Absatz): + + > Volltext des Skriptes „Algebra und Zahlentheorie“ von Stefan Kebekus + > (Universität Freiburg, CC-BY 4.0), automatisch aus den LaTeX-Quellen + > erzeugt. + > + > Inhalt: 25 Kapitel in 79 Abschnitten (Körpererweiterungen, Ringe, + > Körper- und Galoistheorie, Gruppentheorie, Anwendungen), dazu + > Inhaltsverzeichnis, Stichwortverzeichnis, ein Verzeichnis aller 467 + > numerierten Aussagen, eine Liste der 59 Erklärvideos, Kurzbiographien + > und die Literaturliste. + > + > Besonderheit: Jede Aussage und jeder Beweis trägt Typ, gedruckte Nummer + > und Seitenzahl in der eigenen Überschrift (z. B. „Satz 15.2.3“, „Beweis + > von Satz 15.2.3“, „gedruckte Seite 107“). Querverweise sind bereits zu + > Nummern aufgelöst. Damit kann jede Antwort präzise auf die Stelle im PDF + > verweisen. + > + > Zweck: Grundlage für den AZ-Tutor, der Studierenden den Stoff erklärt und + > beim eigenständigen Lösen der Übungsaufgaben hilft. Das Skript ist die + > maßgebliche Quelle — bei Abweichungen zwischen allgemeinem Lehrbuchwissen + > und dem Skript gilt das Skript. Lean und Mathlib gehören nicht zum Stoff + > dieser Vorlesung und sind im Korpus bewusst nicht enthalten. + + Ein späterer zweiter Speicher mit **Musterlösungen** (WP3) braucht eine + deutlich andere Beschreibung, damit er nicht versehentlich am + *Hinweise*-Modell landet — dort wäre die Hinweisleiter sonst sinnlos. +2. **Modell anlegen.** Arbeitsbereich → *Modelle* → `+`. + - Name: `AZ-Tutor — Erklären & Hinweise` + - Basismodell: `ufr/chat-standard` (Gemma 4 31B, 256k, lokal) — für den + ersten Eindruck genügt das. Zum Vergleich lohnt ein zweites Modell auf + `qwen-3.5-397b-llmlb` oder `glm-5.2-llmlb`. + - System Prompt: Inhalt von + `tools/corpus/systemprompt-erklaeren-komplett.md` einfügen (~6k Token). + - Wissen: `AZ-Skript` verknüpfen. + - Temperatur: 0,2–0,3. +3. **Chunk-Größe.** Open WebUI zerlegt hochgeladene Dateien noch einmal selbst + (Standard 1500 Zeichen). Die Abschnittsdateien sind auf ~4000 Zeichen + ausgelegt; falls das RZ die Einstellung freigibt, Chunk-Größe 4000 / + Overlap 300 setzen. Sonst funktioniert es auch mit dem Standard — jede + Aussage und jeder Beweis trägt seine Nummer in der eigenen Überschrift, + damit ein einzelner Chunk sich selbst identifiziert. + +## Erste Testfragen (Kurzform von E1 aus dem Plan) + +| Frage | Erwartung | +|---|---| +| Was besagt Satz 16.1.4? | korrekte Aussage + Seite | +| Wo wird „separabel“ eingeführt? | Kapitel 14, richtiger Abschnitt | +| Warum braucht der Hauptsatz der Galoistheorie die Normalität? | inhaltlich, mit Verweis | +| Nenne einen faktoriellen, nicht euklidischen Ring | Beispiel aus dem Skript | +| Was sagt der Satz von Wedderburn? | **„steht nicht im Skript“** | + +Entscheidend ist die letzte Zeile und die Frage, ob irgendeine Satznummer +auftaucht, die es nicht gibt. Genau dagegen steht das Aussagenverzeichnis im +System-Prompt. + +## Aufbau der Skripte + +| Datei | Aufgabe | +|---|---| +| `az_registry.py` | Zählersimulation, Seiten, Selbsttest. Bricht ab, wenn eine Nummer nicht zur `.aux` passt. Auch als Modul nutzbar. | +| `az_prelude.tex` | Makrodefinitionen, die pandoc vorangestellt werden — ohne sie löscht pandoc Titel, `\video`, `\schritt` und `\factor` samt Argument. | +| `az_convert.py` | Vorverarbeitung → pandoc → Nachbereitung → Abschnittsdateien. | +| `az_indexes.py` | die sieben Verzeichnisdateien. | +| `build-corpus.sh` | Orchestrierung samt Frischeprüfung. | + +Arbeitsteilung: **Python** macht alles, was mit Zählern, Seiten, Labels, +Querverweisen, Index und Videos zu tun hat — davon weiß pandoc nichts. +**pandoc** macht den Textsatz; die Unicode-Mathematik des Skriptes (ℤ, α, →, ∈) +geht dabei unverändert durch. + +## Was der Selbsttest prüft + +* 174 gelabelte Aussagen: simulierte Nummer == `.aux` +* 68 gelabelte Listenpunkte: simulierte Nummer == `.aux` +* 464 von 467 Aussagen: Typ + Nummer tauchen wörtlich im PDF-Text auf + (die restlichen 3 über `synctex`) +* alle 494 `\ref`/`\vref`/`\eqref` auflösbar — kein `??` im Korpus + +Schlägt etwas davon fehl, bricht der Bau ab. Das ist Absicht: ein Korpus mit +falschen Nummern ist schlimmer als gar keiner. diff --git a/tools/corpus/az_convert.py b/tools/corpus/az_convert.py new file mode 100644 index 0000000..e241c68 --- /dev/null +++ b/tools/corpus/az_convert.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +""" +Konvertiert die LaTeX-Quellen des Skriptes in den Markdown-Korpus für den +AZ-Tutor (eine Datei pro Abschnitt, plus Kapitel-Einleitungen). + +Arbeitsteilung: + * Python macht alles, was Zähler, Seiten, Labels, Querverweise, Index und + Videos betrifft -- pandoc kennt davon nichts. + * pandoc macht den eigentlichen Textsatz (Mathematik, Listen, Tabellen, + Fußnoten, Links). Die Unicode-Mathematik des Skriptes geht dabei + unverändert durch. + +Wichtig für RAG: Open WebUI zerlegt hochgeladene Dateien noch einmal in +eigene Chunks. Deshalb steht die Identifikation *im Text* und wiederholt -- +jede Aussage und jeder Beweis trägt seine Nummer in der Überschrift. +""" + +from __future__ import annotations + +import re +import shutil +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import az_registry as R # noqa: E402 + +OUT = R.ROOT / "corpus" +SPLIT = "AZSPLIT7F3A" +PANDOC_TO = "markdown_strict+tex_math_dollars+pipe_tables" +PDF_URL = ("https://cplx.vm.uni-freiburg.de/storage/" + "AlgebraZahlentheorie.pdf#page={}") + +# Umgebungen ohne extrahierbaren Text -> Platzhalter statt Halluzination +DIAGRAM_ENVS = {"tikzpicture", "tikzcd", "xy", "xymatrix"} + +DROP_MACROS_WITH_ARG = ["index", "mathlib", "leanlink", "label", + "selectlanguage", "sideremark", "questionSign", + "disaster", "watchOut", "approvals", + "constructionWarning", "todo"] +KEEP_ARG_MACROS = ["Preprint", "Publication"] + + +# ---------------------------------------------------------------------------- +# LaTeX-Vorverarbeitung +# ---------------------------------------------------------------------------- + +def strip_comments(text: str) -> str: + """ + LaTeX-Kommentare entfernen. + + Achtung: URLs enthalten prozentkodierte Zeichen (…/Kantenl%c3%a4nge%20…). + hyperref liest das URL-Argument mit geändertem Catcode, dort ist % kein + Kommentarzeichen. Solche Argumente werden hier vor dem Strippen + ausgeblendet und danach zurückgeschrieben. + """ + stash: list[str] = [] + + def hide(m): + stash.append(m.group(0)) + return f"\x00URL{len(stash) - 1}\x00" + + text = re.sub(r"\\(?:href|url)\{[^}]*\}", hide, text) + + out = [] + for line in text.split("\n"): + res, i = [], 0 + while i < len(line): + c = line[i] + if c == "\\" and i + 1 < len(line): + res.append(line[i:i + 2]) + i += 2 + continue + if c == "%": + break + res.append(c) + i += 1 + out.append("".join(res).rstrip()) + text = "\n".join(out) + return re.sub(r"\x00URL(\d+)\x00", lambda m: stash[int(m.group(1))], text) + + +def inline_footnotes(text: str) -> str: + """\\footnote{...} -> " (Fußnote: ...)". + + Muss vor pandoc geschehen: pandoc sammelt Fußnoten am Dokumentende, und + beim Aufteilen in Abschnittsdateien gingen die Texte sonst verloren. + """ + while True: + k = text.find("\\footnote{") + if k < 0: + return text + try: + arg, end = R.read_group(text, k + len("\\footnote") ) + except ValueError: + return text + arg = arg.strip() + text = text[:k] + f" (Fußnote: {arg})" + text[end:] + + +def resolve_refs(text: str, aux: dict) -> tuple[str, list[str]]: + """\\ref/\\vref/\\eqref/\\ref* -> gedruckte Nummern (bei \\vref + Seite).""" + unresolved: list[str] = [] + + def repl(m): + kind, label = m.group(1), m.group(3) + entry = aux.get(label) + if entry is None: + unresolved.append(label) + return f"[{label}?]" + printed = entry["printed"] + if kind == "vref": + return f"{printed} (S. {entry['page']})" + return printed + + text = re.sub(r"\\(ref|vref|eqref)\s*(\*?)\s*\{([^}]*)\}", repl, text) + return text, unresolved + + +def substitute_cites(text: str, bib: dict) -> str: + def repl(m): + keys = [k.strip() for k in m.group(1).split(",")] + return "[" + ", ".join(bib.get(k, k) for k in keys) + "]" + return re.sub(r"\\cite\{([^}]*)\}", repl, text) + + +def replace_diagrams(text: str, page: int | None) -> str: + """Diagramme tragen keinen extrahierbaren Text -> Verweis aufs PDF.""" + where = f"siehe Skript S. {page}" if page else "siehe Skript" + for env in DIAGRAM_ENVS: + text = re.sub( + r"\\begin\{" + env + r"\}.*?\\end\{" + env + r"\}", + f"\n\n[Diagramm: {where}]\n\n", text, flags=re.S) + text = re.sub(r"\\includegraphics(\[[^]]*\])?\{[^}]*\}", + f"[Abbildung: {where}]", text) + # \input{figures/...}: pandoc versucht die Datei zu laden und bricht ab. + # Die Figuren sind TikZ/EPS und enthalten keinen Text -- der \caption des + # umgebenden figure-Blocks bleibt erhalten und trägt die Information. + text = re.sub(r"\\input\{[^}]*\}", f"[Abbildung: {where}]", text) + return text + + +def number_items(text: str, stmt) -> str: + """ + enumerate -> itemize mit fett gesetzter gedruckter Nummer. + + Die Nummern (14.1.4.1) stammen aus der Zählersimulation; \\ref zeigt auf + genau diese Nummern, deshalb müssen sie im Chunk stehen. Ein normales + ordered list würde daneben eine zweite, falsche Numerierung erzeugen. + """ + if not stmt or not stmt["items"]: + return text + numbers = [it["number"] for it in stmt["items"]] + lines = text.split("\n") + out, depth, idx = [], 0, 0 + for line in lines: + s = line.lstrip() + if s.startswith("\\begin{enumerate}"): + depth += 1 + out.append(line.replace("\\begin{enumerate}", "\\begin{itemize}")) + continue + if s.startswith("\\end{enumerate}"): + depth -= 1 + out.append(line.replace("\\end{enumerate}", "\\end{itemize}")) + continue + if depth == 1 and s.startswith("\\item") and idx < len(numbers): + rest = s[len("\\item"):] + rest = re.sub(r"^\[[^]]*\]", "", rest) + indent = line[:len(line) - len(s)] + out.append(f"{indent}\\item \\textbf{{{numbers[idx]}}} {rest}") + idx += 1 + continue + out.append(line) + return "\n".join(out) + + +def preprocess(text: str, aux: dict, bib: dict, page: int | None, + stmt=None) -> tuple[str, list[str]]: + text = strip_comments(text) + text = re.sub(r"\\setcounter\{[^}]*\}\{[^}]*\}", "", text) + text = number_items(text, stmt) + for m in KEEP_ARG_MACROS: + text = R.strip_macro(text, m, keep_arg=True) + for m in DROP_MACROS_WITH_ARG: + text = R.strip_macro(text, m) + text = re.sub(r"\\video\{([0-9-]+)\}", + lambda m: f"(Erklärvideo {m.group(1)}: " + f"{R.VIDEO_URL.format(m.group(1))})", text) + text = inline_footnotes(text) + text, unresolved = resolve_refs(text, aux) + text = substitute_cites(text, bib) + text = replace_diagrams(text, page) + text = re.sub(r"\\eqno\s*\\qed\b", "", text) + text = re.sub(r"\\qedhere\b|\\qed\b", "", text) + text = re.sub(r"\\vspace\{[^}]*\}|\\smallskip\b|\\medskip\b|\\bigskip\b", + "", text) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip(), unresolved + + +# ---------------------------------------------------------------------------- +# Blöcke einer Einheit +# ---------------------------------------------------------------------------- + +def unit_blocks(unit: dict, lines: list[str], stmt_at: dict) -> list[dict]: + """Zerlegt eine Einheit in Prosa-, Aussage- und Beweisblöcke.""" + blocks: list[dict] = [] + buf: list[str] = [] + i = unit["line_begin"] - 1 + end = unit["line_end"] + + def flush(): + if any(x.strip() for x in buf): + blocks.append({"kind": "prose", "latex": "\n".join(buf)}) + buf.clear() + + while i < end and i < len(lines): + line = lines[i] + s = line.lstrip() + + if re.match(r"\\(chapter|section)\*?\{", s): + i += 1 + while i < end and lines[i].lstrip().startswith("\\label{"): + i += 1 + continue + + key = (unit["file"], i + 1) + if key in stmt_at: + flush() + st = stmt_at[key] + j = st["line_end"] - 1 if st["line_end"] else i + body = lines[i + 1:j] + blocks.append({"kind": "stmt", "stmt": st, + "latex": "\n".join(body)}) + i = j + 1 + continue + + if s.startswith("\\begin{proof}"): + flush() + depth, j = 1, i + 1 + while j < len(lines) and depth: + t = lines[j].lstrip() + if t.startswith("\\begin{proof}"): + depth += 1 + elif t.startswith("\\end{proof}"): + depth -= 1 + j += 1 + rest = s[len("\\begin{proof}"):] + opt = "" + if rest.startswith("["): + try: + opt, _ = R.read_optional(rest, 0) + except ValueError: + opt = "" + blocks.append({"kind": "proof", "opt": opt, + "videos": R.all_macro_args(rest, "video"), + "latex": "\n".join(lines[i + 1:j - 1])}) + i = j + continue + + buf.append(line) + i += 1 + + flush() + return blocks + + +# ---------------------------------------------------------------------------- +# pandoc +# ---------------------------------------------------------------------------- + +def run_pandoc(chunks: list[str], prelude: str) -> list[str]: + """Alle Blöcke einer Einheit in einem pandoc-Lauf.""" + sep = f"\n\n{SPLIT}\n\n" + doc = prelude + "\n" + sep.join(chunks) + proc = subprocess.run( + ["pandoc", "-f", "latex", "-t", PANDOC_TO, "--wrap=none"], + input=doc, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError(f"pandoc: {proc.stderr[:800]}") + parts = re.split(rf"^\s*{SPLIT}\s*$", proc.stdout, flags=re.M) + if len(parts) != len(chunks): + raise RuntimeError(f"pandoc: {len(parts)} Teile statt {len(chunks)}") + return [clean_markdown(p) for p in parts] + + +def collapse_inline_math(md: str) -> str: + """ + Zeilenumbrüche innerhalb von $…$ zu Leerzeichen. + + Die Quelle bricht Formeln mitten im Satz um; der Umbruch überlebt pandoc + und macht die Formel für Markdown/KaTeX brüchig. Abgesetzte Formeln + ($$…$$) bleiben unangetastet -- dort ist der Umbruch gewollt. + """ + parts = md.split("$$") + for k in range(0, len(parts), 2): # nur außerhalb von $$…$$ + # Nach Parität trennen, nicht per Regex: ein Muster wie \$…\n…\$ paart + # sonst das *schließende* Dollar einer Formel mit dem öffnenden der + # nächsten und würde Fließtext in Mathematik verwandeln. + chunks = parts[k].split("$") + for i in range(1, len(chunks), 2): + if "\n" in chunks[i]: + chunks[i] = re.sub(r"\s+", " ", chunks[i]).strip() + parts[k] = "$".join(chunks) + return "$$".join(parts) + + +def strip_raw_html(md: str) -> str: + """ + pandoc gibt für einige Konstrukte rohes HTML aus (\\foreignlanguage -> + , \\textsf -> , minipage/center + ->
, gelegentlich statt Markdown-Link). Im Korpus ist das + nur Ballast, den das Modell mitlesen müsste. + """ + md = re.sub(r']*>(.*?)', + lambda m: f"[{m.group(2)}]({m.group(1)})", md, flags=re.S) + md = re.sub(r"]*>", "", md) + return md + + +def clean_markdown(md: str) -> str: + md = re.sub(r']*>\s*', "", md) + md = strip_raw_html(md) + md = re.sub(r'\{#[^}]*\}', "", md) + md = md.replace("0◻", "").replace("◻", "") + md = collapse_inline_math(md) + md = re.sub(r"[ \t]+$", "", md, flags=re.M) + md = re.sub(r"\n{3,}", "\n\n", md) + return md.strip() + + +# ---------------------------------------------------------------------------- +# Zusammenbau +# ---------------------------------------------------------------------------- + +def video_lines(ids: list[str]) -> list[str]: + return [f"Erklärvideo {v}: {R.VIDEO_URL.format(v)}" for v in ids] + + +def clean_index_key(key: str) -> str: + """\\index-Schlüssel -> lesbares Stichwort (siehe CLAUDE.md).""" + key = key.split("|")[0] + parts = [] + for part in key.split("!"): + if "@" in part: + part = part.split("@", 1)[1] # Sortierschlüssel weg + parts.append(part.strip()) + return " / ".join(p for p in parts if p) + + +def build_unit_markdown(unit: dict, blocks: list[str], meta: list[dict], + reg: dict) -> str: + part = f"Teil {unit['part_number']} — {unit['part_title']}" \ + if unit["part_title"] else "" + if unit["kind"] == "section": + head = f"Abschnitt {unit['number']} — {unit['title']}" + elif unit["kind"] == "front-matter": + head = unit["title"] + else: + head = f"Kapitel {unit['number']} — {unit['title']} (Einleitung)" + + out = [f"# {head}", ""] + crumb = [c for c in ( + part, + (f"Kapitel {unit['chapter']}: {unit['chapter_title']}" + if unit["kind"] != "front-matter" else ""), + head) if c] + out.append("Quelle: Skript „Algebra und Zahlentheorie“ von Stefan Kebekus. " + + ", ".join(crumb) + ".") + if unit["page"]: + out.append(f"Gedruckte Seite {unit['page']}. " + f"PDF: {PDF_URL.format(unit['page'])}") + out.append("") + + for md, m in zip(blocks, meta): + if m["kind"] == "prose": + if md: + out += [md, ""] + continue + if m["kind"] == "stmt": + st = m["stmt"] + title = f" — {st['title']}" if st["title"] else "" + out.append(f"## {st['display']} {st['number']}{title}") + out.append("") + bits = [f"Kapitel {st['chapter']}", f"Abschnitt " + f"{st['chapter']}.{st['section']}"] + if st["page"]: + bits.append(f"gedruckte Seite {st['page']}") + out.append("*" + " · ".join(bits) + "*") + out.append("") + if md: + out += [md, ""] + for line in video_lines(st["videos"]): + out += [line, ""] + keys = sorted({clean_index_key(k) for k in st["index_keys"]}) + if keys: + out += ["*Stichwörter: " + ", ".join(keys) + "*", ""] + continue + if m["kind"] == "proof": + of = m.get("of") + out.append(f"## Beweis von {of}" if of else "## Beweis") + out.append("") + if md: + out += [md, ""] + for line in video_lines(m["videos"]): + out += [line, ""] + return "\n".join(out).rstrip() + "\n" + + +def slug(s: str) -> str: + s = (s.lower() + .replace("ä", "ae").replace("ö", "oe").replace("ü", "ue") + .replace("ß", "ss")) + s = re.sub(r"[^a-z0-9]+", "-", s).strip("-") + return s[:60] or "abschnitt" + + +def main() -> int: + if not shutil.which("pandoc"): + raise SystemExit("pandoc nicht gefunden") + reg = R.build(strict=True) + aux = reg["aux"] + prelude = (Path(__file__).resolve().parent / "az_prelude.tex") \ + .read_text(encoding="utf-8") + bib = {} + bblfile = R.ROOT / "AlgebraZahlentheorie.bbl" + if bblfile.exists(): + for m in re.finditer(r"\\bibitem\[([^]]*)\]\{([^}]*)\}", + bblfile.read_text(encoding="utf-8")): + bib[m.group(2)] = re.sub(r"[{}]", "", m.group(1)) + + stmt_at = {(s["file"], s["line_begin"]): s for s in reg["statements"]} + stmt_by_number = {s["number"]: s for s in reg["statements"]} + files = {f: (R.ROOT / f).read_text(encoding="utf-8").splitlines() + for f in sorted({u["file"] for u in reg["units"]})} + + if OUT.exists(): + shutil.rmtree(OUT) + OUT.mkdir(parents=True) + + unresolved_all: list[str] = [] + written = 0 + for unit in reg["units"]: + lines = files[unit["file"]] + blocks = unit_blocks(unit, lines, stmt_at) + if not blocks: + continue + + chunks, meta = [], [] + last_stmt = None + for b in blocks: + st = b.get("stmt") + page = st["page"] if st else unit["page"] + latex, unres = preprocess(b["latex"], aux, bib, page, st) + unresolved_all += unres + chunks.append(latex if latex.strip() else "~") + if b["kind"] == "stmt": + last_stmt = st + meta.append({"kind": "stmt", "stmt": st}) + elif b["kind"] == "proof": + of = None + if last_stmt: + of = f"{last_stmt['display']} {last_stmt['number']}" + mref = re.search(r"\d+\.\d+\.\d+", b["opt"] or "") + if mref and mref.group(0) in stmt_by_number: + t = stmt_by_number[mref.group(0)] + of = f"{t['display']} {t['number']}" + meta.append({"kind": "proof", "of": of, + "videos": b["videos"]}) + else: + meta.append({"kind": "prose"}) + + try: + mds = run_pandoc(chunks, prelude) + except RuntimeError as e: + print(f"FEHLER in {unit['file']} {unit['number']}: {e}", + file=sys.stderr) + return 1 + + md = build_unit_markdown(unit, mds, meta, reg) + if unit["kind"] == "front-matter": + cdir, name = OUT, f"00-{slug(unit['title'])}.md" + else: + cdir = OUT / f"kapitel-{unit['chapter']:02d}" + cdir.mkdir(exist_ok=True) + if unit["kind"] == "section": + name = (f"{unit['number'].replace('.', '-')}-" + f"{slug(unit['title'])}.md") + else: + name = f"{unit['chapter']:02d}-00-einleitung.md" + (cdir / name).write_text(md, encoding="utf-8") + written += 1 + + total = sum(len(p.read_text(encoding='utf-8')) + for p in OUT.rglob("*.md")) + print(f"Dateien geschrieben ............. {written}") + print(f"Zeichen gesamt .................. {total}") + print(f"unauflösbare Referenzen ......... {len(unresolved_all)}") + for u in sorted(set(unresolved_all))[:10]: + print(" -", u) + print(f"\n-> {OUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/corpus/az_indexes.py b/tools/corpus/az_indexes.py new file mode 100644 index 0000000..8ca741e --- /dev/null +++ b/tools/corpus/az_indexes.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +""" +Erzeugt die Hilfsdateien des Korpus (corpus/00-*.md). + +Die wichtigste ist ``00-verzeichnis-aussagen-kompakt.md``: die vollständige +Liste aller 467 numerierten Aussagen, klein genug, um dauerhaft in den +System-Prompt zu passen. Damit weiß der Tutor *immer*, welche Nummern es +gibt -- und erfindet keine. Ohne diese Liste ist die Anforderung „null +erfundene Satznummern“ nicht zu halten. + +Muss nach az_convert.py laufen: der Kurzinhalt wird aus dem fertigen Korpus +gelesen, nicht noch einmal aus LaTeX erzeugt. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import az_registry as R # noqa: E402 +import az_convert as C # noqa: E402 + +OUT = R.ROOT / "corpus" +HEAD = ("Quelle: Skript „Algebra und Zahlentheorie“ von Stefan Kebekus " + "(CC-BY 4.0).") + + +ACCENTS = {"'a": "á", "'e": "é", "'i": "í", "'o": "ó", "'u": "ú", + "`a": "à", "`e": "è", "`o": "ò", '"a': "ä", '"o': "ö", + '"u': "ü", "^a": "â", "^e": "ê", "^o": "ô", "~n": "ñ", + "c c": "ç", "v s": "š", "v z": "ž", "v c": "č"} + + +def detex(s: str) -> str: + """BibTeX-Rückstände in lesbaren Text verwandeln.""" + s = s.replace("$'$", "'").replace("\\&", "&") + for tex, ch in ACCENTS.items(): + s = s.replace("\\" + tex, ch).replace("\\" + tex[0] + "{" + tex[-1] + "}", ch) + s = re.sub(r"\\[a-zA-Z]+\s*", " ", s) + s = re.sub(r"[{}~]", "", s) + return re.sub(r"\s+", " ", s).strip(" .") + + +def first_sentence(md_body: str, limit: int = 160) -> str: + """Erster Satz eines Aussagekörpers, für die Kurzinhalt-Spalte.""" + text = md_body.strip() + text = re.sub(r"\$\$.*?\$\$", " […] ", text, flags=re.S) + text = re.sub(r"^\s*[-*]\s+", "", text, flags=re.M) + text = re.sub(r"\*+", "", text) + text = re.sub(r"\s+", " ", text).strip() + m = re.search(r"(?<=[.!?])\s+(?=[A-ZÄÖÜ])", text) + if m and m.start() < limit * 1.6: + text = text[:m.start() + 1] + if len(text) > limit: + text = text[:limit].rsplit(" ", 1)[0] + " …" + return text.replace("|", "\\|") + + +def harvest_bodies() -> dict[str, str]: + """Nummer -> Körper der Aussage, aus den erzeugten Korpusdateien.""" + bodies: dict[str, str] = {} + for p in OUT.rglob("*.md"): + if p.parent == OUT: + continue + blocks = re.split(r"^## ", p.read_text(encoding="utf-8"), flags=re.M) + for b in blocks[1:]: + m = re.match(r"[^\n]*?(\d+\.\d+\.\d+)", b) + if not m or b.lstrip().startswith("Beweis"): + continue + body = re.sub(r"^\*[^\n]*\*\s*$", "", b[b.index("\n"):], + count=1, flags=re.M) + bodies[m.group(1)] = body + return bodies + + +def w(name: str, lines: list[str]) -> None: + (OUT / name).write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + print(f" {name:42s} {len(''.join(lines)):7d} Zeichen") + + +def main() -> int: + reg = R.build(strict=True) + stmts = reg["statements"] + units = reg["units"] + toc = reg["toc"] + bodies = harvest_bodies() + + # ---- 1. Inhaltsverzeichnis -------------------------------------------- + lines = ["# Inhaltsverzeichnis — Algebra und Zahlentheorie", "", HEAD, + "Alle Seitenangaben sind gedruckte Seitenzahlen des PDF.", ""] + for t in toc: + if t["level"] == "part": + lines += ["", f"## Teil {t['number']} — {t['title']} " + f"(ab S. {t['page']})", ""] + elif t["level"] == "chapter": + lines.append(f"### Kapitel {t['number']} — {t['title']} " + f"(S. {t['page']})") + elif t["level"] == "section": + lines.append(f"- Abschnitt {t['number']} {t['title']} " + f"— S. {t['page']}") + else: + lines.append(f" - {t['number']} {t['title']} — S. {t['page']}") + w("00-inhaltsverzeichnis.md", lines) + + # ---- 2. Verzeichnis aller Aussagen (vollständig) ---------------------- + full = ["# Verzeichnis aller numerierten Aussagen", "", HEAD, + f"Das Skript enthält {len(stmts)} numerierte Aussagen. " + "Nur diese Nummern existieren — jede andere Nummer ist falsch.", + "", + "| Nr. | Typ | Titel | S. | Kurzinhalt | Video |", + "|---|---|---|---|---|---|"] + compact = ["# Verzeichnis aller numerierten Aussagen (kompakt)", "", HEAD, + f"Vollständige Liste aller {len(stmts)} numerierten Aussagen " + "des Skriptes. Kommt eine Nummer hier nicht vor, gibt es sie " + "nicht.", "", + "| Nr. | Typ | Titel | S. |", "|---|---|---|---|"] + for s in stmts: + vids = " ".join(v for p in s["proofs"] for v in p["videos"]) + title = (s["title"] or "—").replace("|", "\\|") + page = s["page"] or "?" + full.append(f"| {s['number']} | {s['display']} | {title} | {page} " + f"| {first_sentence(bodies.get(s['number'], ''))} " + f"| {vids or '—'} |") + compact.append(f"| {s['number']} | {s['display']} | {title} " + f"| {page} |") + w("00-verzeichnis-aussagen.md", full) + w("00-verzeichnis-aussagen-kompakt.md", compact) + + # ---- 3. Stichwortverzeichnis ------------------------------------------ + entries: dict[str, dict] = {} + for s in stmts: + for k in s["index_keys"]: + e = entries.setdefault(C.clean_index_key(k), + {"def": [], "use": [], "see": None}) + ref = f"{s['display']} {s['number']} (S. {s['page']})" + if k.endswith("|textbf"): + e["def"].append(ref) + elif "|see{" in k: + m = re.search(r"\|see\{([^}]*)\}", k) + e["see"] = m.group(1) if m else None + elif ref not in e["use"]: + e["use"].append(ref) + for u in units: + for k in u["index_keys"]: + e = entries.setdefault(C.clean_index_key(k), + {"def": [], "use": [], "see": None}) + where = (f"Abschnitt {u['number']}" if u["kind"] == "section" + else f"Kapitel {u['chapter']}") + ref = f"{where} (S. {u['page']})" + if "|see{" in k: + m = re.search(r"\|see\{([^}]*)\}", k) + e["see"] = m.group(1) if m else None + elif k.endswith("|textbf"): + e["def"].append(ref) + elif ref not in e["use"]: + e["use"].append(ref) + + lines = ["# Stichwortverzeichnis", "", HEAD, + "Begriff → Stelle im Skript. „definiert in“ markiert die " + "definierende Stelle.", ""] + for key in sorted(entries, key=lambda x: x.lower()): + e = entries[key] + if e["see"]: + lines.append(f"- **{key}** — siehe *{C.clean_index_key(e['see'])}*") + continue + bits = [] + if e["def"]: + bits.append("definiert in " + "; ".join(e["def"][:3])) + if e["use"]: + bits.append("außerdem " + "; ".join(e["use"][:6])) + lines.append(f"- **{key}** — " + " · ".join(bits) if bits + else f"- **{key}**") + w("00-stichwortverzeichnis.md", lines) + + # ---- 4. Videos --------------------------------------------------------- + lines = ["# Vorlesungsvideos", "", HEAD, + "Zu vielen Beweisen gibt es ein Erklärvideo. " + "Nummer = Vorlesung-Teil.", ""] + seen = set() + for s in stmts: + for p in s["proofs"]: + for v in p["videos"]: + if v in seen: + continue + seen.add(v) + lines.append(f"- **Video {v}** — Beweis von {s['display']} " + f"{s['number']}" + + (f" ({s['title']})" if s["title"] else "") + + f", S. {s['page']}: {R.VIDEO_URL.format(v)}") + for u in units: + for v in u["videos"]: + if v in seen: + continue + seen.add(v) + lines.append(f"- **Video {v}** — Abschnitt {u['number']} " + f"{u['title']}, S. {u['page']}: " + f"{R.VIDEO_URL.format(v)}") + w("00-videos.md", lines) + + # ---- 5. Personen ------------------------------------------------------- + people: dict[str, str] = {} + for p in OUT.rglob("*.md"): + text = p.read_text(encoding="utf-8") + # Klammer-bewusst lesen: die Biographien enthalten selbst Klammern + # ("(auch Duerer; * 21. Mai 1471 ...)"), ein [^)]* bricht dort ab. + i = 0 + while True: + k = text.find("(Fußnote: ", i) + if k < 0: + break + try: + note, i = R.read_delimited(text, k, "(", ")") + except ValueError: + i = k + 10 + continue + m = re.match(r"Fußnote: \[([^]]+)\]\(([^)]*)\)(.*)$", note, + flags=re.S) + if not m or "wikipedia" not in m.group(2): + continue + name, url, rest = m.group(1), m.group(2), m.group(3) + rest = re.sub(r"\s+", " ", rest).strip(" .,;") + people.setdefault(name, f"{rest} — {url}" if rest else url) + lines = ["# Personen", "", HEAD, + "Kurzbiographien der im Skript genannten Mathematikerinnen und " + "Mathematiker (aus den Fußnoten).", ""] + for name in sorted(people): + lines.append(f"- **{name}** — {people[name]}") + w("00-personen.md", lines) + + # ---- 6. Literatur ------------------------------------------------------ + lines = ["# Literaturverzeichnis", "", HEAD, ""] + bbl = R.ROOT / "AlgebraZahlentheorie.bbl" + if bbl.exists(): + text = bbl.read_text(encoding="utf-8") + for m in re.finditer(r"\\bibitem\[([^]]*)\]\{([^}]*)\}(.*?)" + r"(?=\\bibitem|\\end\{thebibliography\})", + text, flags=re.S): + short = re.sub(r"[{}]", "", m.group(1)) + body = re.sub(r"\\newblock", " ", m.group(3)) + body = detex(body) + body = re.sub(r"\s+", " ", body).strip(" .") + lines.append(f"- **[{short}]** {body}.") + w("00-literatur.md", lines) + + total = sum(len(p.read_text(encoding="utf-8")) + for p in OUT.rglob("*.md")) + print(f"\nKorpus gesamt: {len(list(OUT.rglob('*.md')))} Dateien, " + f"{total} Zeichen (≈ {total // 3000}k Token)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/corpus/az_prelude.tex b/tools/corpus/az_prelude.tex new file mode 100644 index 0000000..b8835be --- /dev/null +++ b/tools/corpus/az_prelude.tex @@ -0,0 +1,115 @@ +% +% Makro-Präambel für die Korpus-Konvertierung (tools/corpus/az_convert.py). +% +% Diese Datei wird jedem Textblock vorangestellt, BEVOR pandoc läuft. Grund: +% pandoc kennt die privaten Makros des Skriptes nicht und löscht unbekannte +% Makros samt Argument stillschweigend -- Titel, \schritt-Überschriften und +% \factor-Brüche wären sonst weg. Mit \newcommand-Definitionen expandiert +% pandoc sie korrekt, auch innerhalb von Mathematik. +% +% Grundsatz: alles, was KaTeX nicht kennt (private Kürzel wie \bZ, \sA, +% \factor, \scr), wird auf Standard-LaTeX abgebildet. Die Unicode-Zeichen des +% Skriptes (ℤ, α, →, ∈, ·) bleiben unangetastet -- Sprachmodelle lesen sie +% direkt, und sie halten den Korpus kompakt. +% + +% --- Strukturmakros des Skriptes ------------------------------------------ +% \video wird vom Präprozessor entfernt und in die Metadaten gehoben; die +% Definition hier ist nur ein Sicherheitsnetz. +\newcommand{\video}[1]{} +\newcommand{\leanlink}[1]{} +\newcommand{\mathlib}[1]{} +\newcommand{\schritt}[1]{\par\noindent\textbf{#1}\quad} + +% --- Quotienten ----------------------------------------------------------- +\newcommand{\factor}[2]{#1/#2} +\newcommand{\ifactor}[2]{#1 \backslash #2} + +% --- Operatoren (Masterdatei) -------------------------------------------- +\newcommand{\Fix}{\operatorname{Fix}} +\newcommand{\Gal}{\operatorname{Gal}} +\newcommand{\GL}{\operatorname{GL}} +\newcommand{\ggT}{\operatorname{ggT}} +\newcommand{\Iso}{\operatorname{Iso}} +\newcommand{\kgV}{\operatorname{kgV}} +\newcommand{\Kons}{\operatorname{Kons}} +\newcommand{\ord}{\operatorname{ord}} +\newcommand{\sep}{\operatorname{sep}} +\newcommand{\Stab}{\operatorname{Stab}} +\newcommand{\Zentralisator}{\operatorname{Zentralisator}} + +% --- Operatoren (gfx/stdPreamble.tex) ------------------------------------ +\newcommand{\Aut}{\operatorname{Aut}} +\newcommand{\codim}{\operatorname{codim}} +\newcommand{\coker}{\operatorname{coker}} +\newcommand{\const}{\operatorname{const}} +\newcommand{\Ext}{\operatorname{Ext}} +\newcommand{\Hom}{\operatorname{Hom}} +\newcommand{\Id}{\operatorname{Id}} +\newcommand{\Image}{\operatorname{Image}} +\newcommand{\img}{\operatorname{img}} +\newcommand{\Pic}{\operatorname{Pic}} +\newcommand{\rank}{\operatorname{rank}} +\newcommand{\Ramification}{\operatorname{Ramification}} +\newcommand{\red}{\operatorname{red}} +\newcommand{\reg}{\operatorname{reg}} +\newcommand{\sat}{\operatorname{sat}} +\newcommand{\sing}{\operatorname{sing}} +\newcommand{\Spec}{\operatorname{Spec}} +\newcommand{\Sym}{\operatorname{Sym}} +\newcommand{\supp}{\operatorname{supp}} +\newcommand{\tor}{\operatorname{tor}} +\newcommand{\Tor}{\operatorname{Tor}} +\newcommand{\Frob}{\operatorname{Frob}} +\newcommand{\sEnd}{\operatorname{\mathcal{E}nd}} +\newcommand{\sHom}{\operatorname{\mathcal{H}om}} + +% --- Kurzschreibweisen --------------------------------------------------- +\newcommand{\into}{\hookrightarrow} +\newcommand{\onto}{\twoheadrightarrow} +\newcommand{\wtilde}{\widetilde} +\newcommand{\what}{\widehat} +\newcommand{\scr}[1]{\mathcal{#1}} + +% --- Tafelschrift: \bA .. \bZ -> \mathbb{A} .. \mathbb{Z} --------------- +\newcommand{\bA}{\mathbb{A}}\newcommand{\bB}{\mathbb{B}} +\newcommand{\bC}{\mathbb{C}}\newcommand{\bD}{\mathbb{D}} +\newcommand{\bE}{\mathbb{E}}\newcommand{\bF}{\mathbb{F}} +\newcommand{\bG}{\mathbb{G}}\newcommand{\bH}{\mathbb{H}} +\newcommand{\bI}{\mathbb{I}}\newcommand{\bJ}{\mathbb{J}} +\newcommand{\bK}{\mathbb{K}}\newcommand{\bL}{\mathbb{L}} +\newcommand{\bM}{\mathbb{M}}\newcommand{\bN}{\mathbb{N}} +\newcommand{\bO}{\mathbb{O}}\newcommand{\bP}{\mathbb{P}} +\newcommand{\bQ}{\mathbb{Q}}\newcommand{\bR}{\mathbb{R}} +\newcommand{\bS}{\mathbb{S}}\newcommand{\bT}{\mathbb{T}} +\newcommand{\bU}{\mathbb{U}}\newcommand{\bV}{\mathbb{V}} +\newcommand{\bW}{\mathbb{W}}\newcommand{\bX}{\mathbb{X}} +\newcommand{\bY}{\mathbb{Y}}\newcommand{\bZ}{\mathbb{Z}} + +% --- Garben-/Kalligraphie-Kürzel: \sA .. \sZ, \cA .. \cV ---------------- +% rsfs kennt KaTeX nicht; \mathcal ist der tragfähige Ersatz. +\newcommand{\sA}{\mathcal{A}}\newcommand{\sB}{\mathcal{B}} +\newcommand{\sC}{\mathcal{C}}\newcommand{\sD}{\mathcal{D}} +\newcommand{\sE}{\mathcal{E}}\newcommand{\sF}{\mathcal{F}} +\newcommand{\sG}{\mathcal{G}}\newcommand{\sH}{\mathcal{H}} +\newcommand{\sI}{\mathcal{I}}\newcommand{\sJ}{\mathcal{J}} +\newcommand{\sK}{\mathcal{K}}\newcommand{\sL}{\mathcal{L}} +\newcommand{\sM}{\mathcal{M}}\newcommand{\sN}{\mathcal{N}} +\newcommand{\sO}{\mathcal{O}}\newcommand{\sP}{\mathcal{P}} +\newcommand{\sQ}{\mathcal{Q}}\newcommand{\sR}{\mathcal{R}} +\newcommand{\sS}{\mathcal{S}}\newcommand{\sT}{\mathcal{T}} +\newcommand{\sU}{\mathcal{U}}\newcommand{\sV}{\mathcal{V}} +\newcommand{\sW}{\mathcal{W}}\newcommand{\sX}{\mathcal{X}} +\newcommand{\sY}{\mathcal{Y}}\newcommand{\sZ}{\mathcal{Z}} +\newcommand{\cA}{\mathcal{A}}\newcommand{\cC}{\mathcal{C}} +\newcommand{\cD}{\mathcal{D}}\newcommand{\cE}{\mathcal{E}} +\newcommand{\cM}{\mathcal{M}}\newcommand{\cN}{\mathcal{N}} +\newcommand{\cV}{\mathcal{V}} + +% --- Serifenlose Symbole ------------------------------------------------- +\newcommand{\aB}{\mathsf{B}}\newcommand{\aD}{\mathsf{D}} +\newcommand{\aE}{\mathsf{E}}\newcommand{\aF}{\mathsf{F}} + +% --- Sonstiges ----------------------------------------------------------- +\newcommand{\wcheck}[1]{\check{#1}} +\newcommand{\CounterStep}{} diff --git a/tools/corpus/az_registry.py b/tools/corpus/az_registry.py new file mode 100644 index 0000000..cc9c2a3 --- /dev/null +++ b/tools/corpus/az_registry.py @@ -0,0 +1,689 @@ +#!/usr/bin/env python3 +""" +Registry für den AZ-Tutor-Korpus. + +Ermittelt für jede numerierte Aussage des Skriptes die *gedruckte* Nummer, den +Typ, den Titel, die Seite, die Stichwörter und die Videoverweise -- und prüft +das Ergebnis gegen drei unabhängige Quellen: + + * ``AlgebraZahlentheorie.aux`` -- 174 gelabelte Aussagen mit Nummer + Seite + * ``AlgebraZahlentheorie.toc`` -- Teile / Kapitel / Abschnitte mit Seiten + * der PDF-Text selbst -- *alle* Aussagen (Nummer, Typ, Seite) + +Die Zählersimulation ahmt LaTeX nach: ``thm`` wird pro ``section`` +zurückgesetzt, alle numerierten Umgebungen teilen diesen Zähler, ``equation`` +wird pro ``thm`` zurückgesetzt, und ``enumerate``-Punkte hängen über die +``\\setlist``-Hooks am ``equation``-Zähler. + +Das Skript ist als Modul importierbar (``build()``) und als Programm +ausführbar (schreibt ``registry.json`` und meldet den Selbsttest). +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +from dataclasses import dataclass, field, asdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent.parent +MASTER = ROOT / "AlgebraZahlentheorie.tex" +PREAMBLE = ROOT / "gfx" / "stdPreamble.tex" +AUX = ROOT / "AlgebraZahlentheorie.aux" +TOC = ROOT / "AlgebraZahlentheorie.toc" +PDF = ROOT / "AlgebraZahlentheorie.pdf" + +VIDEO_URL = "https://cplx.vm.uni-freiburg.de/storage/video/az/{}-Video.mp4" + +# Umgebungen, die *keine* Aussagen sind, deren Nummern aber am selben Zähler +# hängen bzw. die den equation-Zähler bewegen. +NUMBERED_EQ_ENVS = {"equation", "align", "gather", "multline", "eqnarray", + "flalign", "alignat"} + + +# ---------------------------------------------------------------------------- +# Klammer-bewusste Mini-Parser. Ein naives [^}]* zerbricht an genau zwei +# Stellen: Titel mit \footnote{\href{..}{..}} und \index-Schlüsseln wie +# \index{*ap@$\left(\frac{a}{p}\right)$ (Legendre-Symbol)|textbf}. +# ---------------------------------------------------------------------------- + +def read_delimited(s: str, i: int, open_ch: str, close_ch: str): + """s[i] muss open_ch sein. Gibt (inhalt, index_nach_close) zurück.""" + assert s[i] == open_ch, (s[i], open_ch) + depth, j = 0, i + while j < len(s): + c = s[j] + if c == "\\": # \{ und \} überspringen + j += 2 + continue + if c == open_ch: + depth += 1 + elif c == close_ch: + depth -= 1 + if depth == 0: + return s[i + 1:j], j + 1 + j += 1 + raise ValueError(f"unbalanciert ab Position {i}: {s[i:i+60]!r}") + + +def read_group(s, i): + return read_delimited(s, i, "{", "}") + + +def read_optional(s, i): + return read_delimited(s, i, "[", "]") + + +def all_macro_args(s: str, macro: str): + """Alle Argumente von \\macro{...} in s, klammer-bewusst.""" + out, pat = [], "\\" + macro + "{" + i = 0 + while True: + k = s.find(pat, i) + if k < 0: + return out + try: + arg, i = read_group(s, k + len(pat) - 1) + except ValueError: + return out + out.append(arg) + + +def strip_macro(s: str, macro: str, keep_arg: bool = False) -> str: + """Entfernt alle \\macro{...} (klammer-bewusst).""" + pat = "\\" + macro + "{" + while True: + k = s.find(pat, 0) + if k < 0: + return s + try: + arg, end = read_group(s, k + len(pat) - 1) + except ValueError: + return s + s = s[:k] + (arg if keep_arg else "") + s[end:] + + +# ---------------------------------------------------------------------------- +# Präambel: Umgebungsnamen -> gedruckter Anzeigename +# ---------------------------------------------------------------------------- + +def read_theorem_envs() -> dict[str, dict]: + """ + {env: {"display": str, "numbered": bool}} + + Wir übernehmen den *deklarierten* Anzeigenamen wortgetreu, denn genau der + steht im PDF. Das Skript benutzt an einigen Stellen englisch benannte + Umgebungen (rem, obs, warning, problem), die im PDF dann auch englisch + erscheinen -- der Korpus muss das spiegeln, sonst findet der Student die + zitierte Stelle nicht. + """ + envs: dict[str, dict] = {} + for path in (PREAMBLE, MASTER): + for line in path.read_text(encoding="utf-8").splitlines(): + m = re.match(r"\\newtheorem(\*?)\{([^}]+)\}(?:\[[^]]*\])?\{(.+)\}\s*$", + line.strip()) + if m: + star, name, display = m.groups() + envs[name] = {"display": display, "numbered": not star} + return envs + + +def read_input_order() -> list[tuple[str, str]]: + """[(kind, value)] mit kind in {'part','input'} in Dokumentreihenfolge.""" + out = [] + for line in MASTER.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line.startswith("%"): + continue + m = re.match(r"\\part\{(.+)\}\s*$", line) + if m: + out.append(("part", m.group(1))) + continue + m = re.match(r"\\input\{(\d+)\}\s*$", line) + if m: + out.append(("input", m.group(1))) + return out + + +# ---------------------------------------------------------------------------- +# .aux und .toc +# ---------------------------------------------------------------------------- + +def parse_aux() -> dict[str, dict]: + """label -> {'printed','page','title','anchor'}; nur 5-Feld-Einträge.""" + out: dict[str, dict] = {} + text = AUX.read_text(encoding="utf-8", errors="replace") + i = 0 + needle = "\\newlabel{" + while True: + k = text.find(needle, i) + if k < 0: + break + try: + label, j = read_group(text, k + len(needle) - 1) + body, i = read_group(text, j) + except ValueError: + i = k + len(needle) + continue + fields, p = [], 0 + while p < len(body) and len(fields) < 5: + while p < len(body) and body[p] not in "{": + p += 1 + if p >= len(body): + break + f, p = read_group(body, p) + fields.append(f) + if len(fields) < 5: + continue # varioref-Zweifeldeintrag + printed = fields[0].strip("{}") # Item-Labels sind dreifach geklammert + anchor = fields[3] + out[label] = {"printed": printed, "page": fields[1], + "title": fields[2], "anchor": anchor} + return out + + +def parse_toc() -> list[dict]: + """[{'level','number','title','page'}] in Dokumentreihenfolge.""" + out = [] + roman = {"I": 1, "II": 2, "III": 3, "IV": 4, "V": 5, + "VI": 6, "VII": 7, "VIII": 8} + text = TOC.read_text(encoding="utf-8", errors="replace") + for line in text.splitlines(): + m = re.match(r"\\contentsline \{(part|chapter|section|subsection)\}" + r"\{(.*)\}\{(\d+)\}\{([^}]*)\}", line.strip()) + if not m: + continue + level, label, page, anchor = m.groups() + nm = re.match(r"\\numberline \{(?:\\rm )?([^}]*)\}(.*)$", label) + if nm: + number, title = nm.group(1).strip(), nm.group(2) + else: + number, title = "", re.sub(r"^\\nonumberline\s*", "", label) + if level == "part" and number in roman: + number = str(roman[number]) + out.append({"level": level, "number": number, + "title": clean_title(title), "page": int(page), + "anchor": anchor}) + return out + + +def clean_title(t: str) -> str: + t = strip_macro(t, "mathlib") + t = strip_macro(t, "leanlink") + t = strip_macro(t, "index") + t = strip_macro(t, "label") + t = strip_macro(t, "emph", keep_arg=True) + t = strip_macro(t, "textsf", keep_arg=True) + t = t.replace("\\,", " ").replace("~", " ") + t = re.sub(r",\s*$", "", t.strip()) + return re.sub(r"\s+", " ", t).strip() + + +# ---------------------------------------------------------------------------- +# Quellen-Scan mit Zählersimulation +# ---------------------------------------------------------------------------- + +@dataclass +class Statement: + env: str + display: str + number: str # "14.1.4" + chapter: int + section: int + title: str = "" + labels: list[str] = field(default_factory=list) + index_keys: list[str] = field(default_factory=list) + videos: list[str] = field(default_factory=list) + file: str = "" + line_begin: int = 0 # 1-basiert, Zeile mit \begin + line_end: int = 0 # Zeile mit \end + page: int | None = None + page_source: str = "" + items: list[dict] = field(default_factory=list) + proofs: list[dict] = field(default_factory=list) + + +@dataclass +class Unit: + """Ein Abschnitt (oder eine Kapitel-Einleitung) -- die Korpus-Dateieinheit.""" + kind: str # 'chapter-intro' | 'section' | 'subsection' + chapter: int + section: int + number: str # "14.1" bzw. "14" für die Einleitung + title: str + part_number: str = "" + part_title: str = "" + chapter_title: str = "" + file: str = "" + line_begin: int = 0 + line_end: int = 0 + page: int | None = None + statements: list[str] = field(default_factory=list) # Nummern + videos: list[str] = field(default_factory=list) + index_keys: list[str] = field(default_factory=list) + + +def scan() -> tuple[list[Statement], list[Unit], dict]: + envs = read_theorem_envs() + numbered = {k for k, v in envs.items() if v["numbered"]} + order = read_input_order() + + statements: list[Statement] = [] + units: list[Unit] = [] + chapter = section = thm = equation = 0 + part_no, part_title = "", "" + chapter_title = "" + enum_stack: list[dict] = [] # offene Listenumgebungen (enumerate/itemize) + cur_unit: Unit | None = None + cur_stmt: Statement | None = None + last_stmt: Statement | None = None + env_stack: list[str] = [] + warnings: list[str] = [] + part_counter = 0 + + def close_unit(fname, lineno): + nonlocal cur_unit + if cur_unit is not None: + cur_unit.line_end = lineno + units.append(cur_unit) + cur_unit = None + + for kind, value in order: + if kind == "part": + part_counter += 1 + part_no, part_title = str(part_counter), clean_title(value) + continue + + fname = f"{value}.tex" + path = ROOT / fname + lines = path.read_text(encoding="utf-8").splitlines() + + for idx, raw in enumerate(lines, start=1): + line = raw.rstrip("\n") + stripped = line.lstrip() + + # ---- Struktur ---------------------------------------------- + m = re.match(r"\\chapter\{", stripped) + if m: + close_unit(fname, idx - 1) + chapter += 1 + section = thm = equation = 0 + chapter_title = clean_title( + read_group(stripped, stripped.index("{"))[0]) + cur_unit = Unit(kind="chapter-intro", chapter=chapter, + section=0, number=str(chapter), + title=chapter_title, part_number=part_no, + part_title=part_title, + chapter_title=chapter_title, + file=fname, line_begin=idx) + continue + + if re.match(r"\\section\*?\{", stripped): + if stripped.startswith("\\section*"): + # Nur in 00.tex (Vorbemerkung): zählt nicht mit, soll aber + # als eigene Einheit in den Korpus -- dort steht u.a. die + # Literaturliste. + close_unit(fname, idx - 1) + title = clean_title( + read_group(stripped, stripped.index("{"))[0]) + cur_unit = Unit(kind="front-matter", chapter=0, section=0, + number="0", title=title, + part_number="", part_title="", + chapter_title=title, + file=fname, line_begin=idx) + continue + close_unit(fname, idx - 1) + section += 1 + thm = equation = 0 + title = clean_title(read_group(stripped, stripped.index("{"))[0]) + cur_unit = Unit(kind="section", chapter=chapter, + section=section, + number=f"{chapter}.{section}", title=title, + part_number=part_no, part_title=part_title, + chapter_title=chapter_title, + file=fname, line_begin=idx) + continue + + # ---- Umgebungen -------------------------------------------- + m = re.match(r"\\begin\{([A-Za-z*-]+)\}", stripped) + if m: + env = m.group(1) + env_stack.append(env) + + if env in numbered: + thm += 1 + equation = 0 + rest = stripped[m.end():] + title, labels, keys = parse_open_line(rest) + st = Statement( + env=env, display=envs[env]["display"], + number=f"{chapter}.{section}.{thm}", + chapter=chapter, section=section, + title=title, labels=labels, index_keys=keys, + videos=all_macro_args(rest, "video"), + file=fname, line_begin=idx) + statements.append(st) + cur_stmt = st + last_stmt = st + if cur_unit is not None: + cur_unit.statements.append(st.number) + + elif env in NUMBERED_EQ_ENVS: + equation += 1 + + elif env in ("enumerate", "itemize", "description"): + # Nur enumerate zählt Punkte; ein verschachteltes itemize + # darf den Zähler nicht bewegen. Auch tiefere + # enumerate-Ebenen numerieren wir nicht (\arabic* bezieht + # sich dort auf enumii/enumiii). + if env == "enumerate" and not any( + e["kind"] == "enumerate" for e in enum_stack): + start = equation + sm = re.search(r"\\setcounter\{enumi\}\{(\d+)\}", + stripped) + if sm: + start = int(sm.group(1)) + enum_stack.append({"kind": "enumerate", + "enumi": start, "env": env}) + else: + enum_stack.append({"kind": "other", "env": env}) + + elif env == "proof": + rest = stripped[m.end():] + opt = "" + if rest.startswith("["): + try: + opt, _ = read_optional(rest, 0) + except ValueError: + opt = "" + # An die zuletzt *begonnene* Aussage hängen, nicht an die + # gerade offene: der Beweis steht hinter \end{satz}. + if last_stmt is not None: + last_stmt.proofs.append( + {"of": last_stmt.number, "title": opt, + "line": idx, + "videos": all_macro_args(rest, "video")}) + continue + + m = re.match(r"\\end\{([A-Za-z*-]+)\}", stripped) + if m: + env = m.group(1) + if env_stack and env_stack[-1] == env: + env_stack.pop() + else: + warnings.append(f"{fname}:{idx}: \\end{{{env}}} ohne " + f"passendes \\begin (Stack {env_stack[-3:]})") + if env in env_stack: + while env_stack and env_stack.pop() != env: + pass + if env in ("enumerate", "itemize", "description") and enum_stack: + closed = enum_stack.pop() + if closed["kind"] == "enumerate": + equation = closed["enumi"] + if cur_stmt is not None and env == cur_stmt.env: + cur_stmt.line_end = idx + cur_stmt = None + continue + + # ---- \item, \index, \video im Körper ------------------------ + if (stripped.startswith("\\item") and enum_stack + and enum_stack[-1]["kind"] == "enumerate"): + enum_stack[-1]["enumi"] += 1 + num = f"{chapter}.{section}.{thm}.{enum_stack[-1]['enumi']}" + lbls = all_macro_args(stripped, "label") + if cur_stmt is not None: + cur_stmt.items.append({"number": f"({num})", + "labels": lbls, "line": idx}) + elif statements: + statements[-1].items.append({"number": f"({num})", + "labels": lbls, "line": idx}) + if cur_stmt is not None: + for k in all_macro_args(line, "index"): + cur_stmt.index_keys.append(k) + for v in all_macro_args(line, "video"): + if v not in cur_stmt.videos: + cur_stmt.videos.append(v) + elif cur_unit is not None: + for v in all_macro_args(line, "video"): + if v not in cur_unit.videos: + cur_unit.videos.append(v) + cur_unit.index_keys += all_macro_args(line, "index") + + close_unit(fname, len(lines)) + + return statements, units, {"envs": envs, "warnings": warnings} + + +def parse_open_line(rest: str) -> tuple[str, list[str], list[str]]: + """Titel, Labels und Index-Schlüssel aus dem Rest der \\begin-Zeile.""" + title, i = "", 0 + if rest.startswith("["): + title, i = read_optional(rest, 0) + return (clean_title(title), + all_macro_args(rest[i:], "label"), + all_macro_args(rest[i:], "index")) + + +# ---------------------------------------------------------------------------- +# Seiten aus dem PDF-Text (deckt *alle* Aussagen ab, nicht nur die gelabelten) +# ---------------------------------------------------------------------------- + +def pdf_pages(statements: list[Statement], aux: dict) -> tuple[dict, dict]: + """ + Physische PDF-Seite je Aussage aus dem pdftotext-Text, dann Umrechnung in + gedruckte Seiten über einen Offset, der an den .aux-Angaben kalibriert und + verifiziert wird. + """ + txt = subprocess.run(["pdftotext", "-layout", str(PDF), "-"], + capture_output=True, text=True, check=True).stdout + pages = txt.split("\f") + # (display, number) -> physische Seite (1-basiert) + found: dict[tuple[str, str], int] = {} + # Fallback: Nummer -> Seite, wenn der Typname im Umbruch auf der Zeile + # davor gelandet ist (oder -layout ein Streuzeichen davorgesetzt hat). + loose: dict[str, list[tuple[int, str]]] = {} + for pno, page in enumerate(pages, start=1): + plines = page.splitlines() + for li, line in enumerate(plines): + m = re.match(r"\s*([A-ZÄÖÜ][A-Za-zÄÖÜäöüß\- ]{2,24}?)\s+" + r"(\d+\.\d+\.\d+)\s*[.(]", line) + if m: + key = (m.group(1).strip(), m.group(2)) + found.setdefault(key, pno) + continue + m = re.match(r"\s*\W{0,3}\s*(\d+\.\d+\.\d+)\s*[.(]", line) + if m: + prev = plines[li - 1][-80:] if li else "" + loose.setdefault(m.group(1), []).append((pno, prev)) + + offsets: dict[int, int] = {} + for st in statements: + phys = found.get((st.display, st.number)) + if phys is None: + continue + printed = None + for lb in st.labels: + if lb in aux: + printed = int(aux[lb]["page"]) + break + if printed is not None: + offsets[phys - printed] = offsets.get(phys - printed, 0) + 1 + + offset = max(offsets, key=offsets.get) if offsets else 0 + + # Zweiter Durchgang für die im Umbruch zerrissenen Überschriften. + recovered = 0 + for st in statements: + if (st.display, st.number) in found: + continue + for pno, prev in loose.get(st.number, []): + if st.display in prev: + found[(st.display, st.number)] = pno + recovered += 1 + break + + return found, {"offset": offset, "offset_votes": offsets, + "recovered_across_linebreak": recovered} + + +def synctex_page(fname: str, line: int) -> int | None: + """Gedruckte Seite einer Quellzeile via synctex (exakt, aber langsam).""" + try: + out = subprocess.run( + ["synctex", "view", "-i", f"{line}:1:{ROOT / fname}", + "-o", str(PDF)], + capture_output=True, text=True, timeout=20).stdout + except (OSError, subprocess.TimeoutExpired): + return None + m = re.search(r"^Page:(\d+)", out, re.M) + return int(m.group(1)) if m else None + + +# ---------------------------------------------------------------------------- +# Aufbau + Selbsttest +# ---------------------------------------------------------------------------- + +def build(strict: bool = True) -> dict: + aux = parse_aux() + toc = parse_toc() + statements, units, meta = scan() + + found, poff = pdf_pages(statements, aux) + offset = poff["offset"] + + problems: list[str] = [] + + # (1) Nummern gegen .aux + checked = 0 + for st in statements: + for lb in st.labels: + if lb not in aux: + continue + checked += 1 + want = aux[lb]["printed"] + if want != st.number: + problems.append(f"NUMMER {st.file}:{st.line_begin} " + f"{st.env} label={lb}: simuliert {st.number}, " + f".aux sagt {want}") + # (2) Item-Nummern gegen .aux + item_checked = 0 + for st in statements: + for it in st.items: + for lb in it["labels"]: + if lb not in aux: + continue + item_checked += 1 + want = aux[lb]["printed"] + if want != it["number"]: + problems.append(f"ITEM {st.file}:{it['line']} label={lb}: " + f"simuliert {it['number']}, .aux {want}") + # (3) Typ + Nummer müssen im PDF-Text auftauchen + missing_in_pdf = [] + for st in statements: + phys = found.get((st.display, st.number)) + if phys is None: + missing_in_pdf.append(f"{st.display} {st.number} " + f"({st.file}:{st.line_begin}, {st.env})") + else: + st.page = phys - offset + st.page_source = "pdftext" + for lb in st.labels: + if lb in aux: + st.page = int(aux[lb]["page"]) + st.page_source = "aux" + break + + # (4) Rest über synctex -- exakt, aber ein Unterprozess pro Aussage, daher + # nur für die Handvoll, die der PDF-Text nicht hergibt. + for st in statements: + if st.page is not None: + continue + st.page = synctex_page(st.file, st.line_begin) + st.page_source = "synctex" if st.page is not None else "unbekannt" + + # Titel nachbearbeiten: Biographie-Fußnoten raus, \ref auflösen. + # Beides geht erst hier, weil clean_title() beim Scannen noch kein .aux hat. + def finish_title(t: str) -> str: + t = strip_macro(t, "footnote") + def rep(m): + e = aux.get(m.group(3)) + return e["printed"] if e else m.group(0) + # \ref *{...}: die .toc schreibt ein Leerzeichen vor den Stern. + t = re.sub(r"\\(ref|vref|eqref)\s*(\*?)\s*\{([^}]*)\}", rep, t) + return re.sub(r"\s+", " ", t).strip(" ,") + + for st in statements: + st.title = finish_title(st.title) + for u in units: + u.title = finish_title(u.title) + u.chapter_title = finish_title(u.chapter_title) + for t in toc: + t["title"] = finish_title(t["title"]) + + # Seiten für Units aus der .toc + tocmap = {(t["level"], t["number"]): t for t in toc} + for u in units: + key = ("chapter" if u.kind == "chapter-intro" else "section", u.number) + if key in tocmap: + u.page = tocmap[key]["page"] + if not u.title: + u.title = tocmap[key]["title"] + + report = { + "statements": len(statements), + "units": len(units), + "aux_five_field": len(aux), + "numbers_checked_against_aux": checked, + "item_numbers_checked_against_aux": item_checked, + "pdf_page_offset": offset, + "recovered_across_linebreak": poff.get("recovered_across_linebreak", 0), + "offset_votes": poff["offset_votes"], + "missing_in_pdf": missing_in_pdf, + "page_sources": {k: sum(1 for s in statements if s.page_source == k) + for k in ("aux", "pdftext", "synctex", "unbekannt")}, + "scan_warnings": meta["warnings"], + "problems": problems, + } + + if strict and problems: + for p in problems[:40]: + print("FEHLER:", p, file=sys.stderr) + raise SystemExit(f"az_registry: {len(problems)} Inkonsistenz(en) -- " + f"Abbruch (ist der Build aktuell?)") + + return {"statements": [asdict(s) for s in statements], + "units": [asdict(u) for u in units], + "toc": toc, + "envs": meta["envs"], + "aux": aux, + "report": report} + + +def main() -> int: + reg = build(strict=True) + out = Path(__file__).resolve().parent / "registry.json" + out.write_text(json.dumps(reg, ensure_ascii=False, indent=1), + encoding="utf-8") + r = reg["report"] + print(f"Aussagen ........................ {r['statements']}") + print(f"Einheiten (Abschnitte/Kap.-Intro) {r['units']}") + print(f".aux-Einträge (5 Felder) ........ {r['aux_five_field']}") + print(f"Nummern gegen .aux geprüft ...... {r['numbers_checked_against_aux']} (0 Fehler)") + print(f"Item-Nummern gegen .aux geprüft . {r['item_numbers_checked_against_aux']} (0 Fehler)") + print(f"PDF-Seitenoffset ................ {r['pdf_page_offset']} " + f"(Stimmen: {r['offset_votes']})") + print(f"nicht im PDF-Text gefunden ...... {len(r['missing_in_pdf'])}") + for m in r["missing_in_pdf"][:15]: + print(" -", m) + if r["scan_warnings"]: + print(f"Scan-Warnungen .................. {len(r['scan_warnings'])}") + for w in r["scan_warnings"][:10]: + print(" -", w) + print(f"\n-> {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/corpus/build-corpus.sh b/tools/corpus/build-corpus.sh new file mode 100755 index 0000000..d6058e8 --- /dev/null +++ b/tools/corpus/build-corpus.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Baut den Markdown-Korpus für den AZ-Tutor aus den LaTeX-Quellen. +# +# ./tools/corpus/build-corpus.sh +# +# Voraussetzung: ein aktueller LaTeX-Lauf. Der Korpus bezieht Nummern, Seiten +# und Querverweise aus .aux/.toc/.synctex.gz und dem PDF -- laufen die aus dem +# Takt, wird der Korpus still und leise falsch. Deshalb die Frischeprüfung. +# +set -euo pipefail +cd "$(dirname "$0")/../.." + +need() { command -v "$1" >/dev/null || { echo "FEHLT: $1"; exit 1; }; } +need pandoc +need python3 +need pdftotext + +newest_source=$(ls -t AlgebraZahlentheorie.tex gfx/stdPreamble.tex [0-9][0-9].tex | head -1) +stale=0 +for f in AlgebraZahlentheorie.aux AlgebraZahlentheorie.toc \ + AlgebraZahlentheorie.pdf; do + if [ ! -f "$f" ]; then + echo "FEHLT: $f — bitte zuerst 'latexmk --pdf AlgebraZahlentheorie.tex'" + exit 1 + fi + if [ "$newest_source" -nt "$f" ]; then + echo "WARNUNG: $f ist älter als $newest_source" + stale=1 + fi +done +if [ "$stale" = 1 ]; then + echo " Nummern und Seiten könnten veraltet sein." + echo " Empfehlung: latexmk --pdf AlgebraZahlentheorie.tex" +fi +[ -f AlgebraZahlentheorie.synctex.gz ] || \ + echo "HINWEIS: kein synctex -- Seiten für ein paar Aussagen evtl. unbekannt" + +echo "── 1/3 Registry (Nummern, Seiten, Selbsttest) ──" +python3 tools/corpus/az_registry.py + +echo +echo "── 2/3 Abschnittsdateien (pandoc) ──" +python3 tools/corpus/az_convert.py + +echo +echo "── 3/3 Hilfsdateien (Verzeichnisse) ──" +python3 tools/corpus/az_indexes.py + +# System-Prompt + Aussagenverzeichnis zu einer einfügefertigen Datei bauen +python3 - <<'EOF' +import pathlib +sp = pathlib.Path("tools/corpus/systemprompt-erklaeren.md").read_text() +sp = sp.split("---\n", 1)[1].lstrip() # Kopfzeilen der Anleitung weg +inv = pathlib.Path("corpus/00-verzeichnis-aussagen-kompakt.md").read_text() +# NICHT nach corpus/ -- sonst landet der Prompt im Wissensspeicher. +out = pathlib.Path("tools/corpus/systemprompt-erklaeren-komplett.md") +out.write_text(sp.rstrip() + "\n\n# " + inv.split("# ", 1)[1]) +print(f" systemprompt-erklaeren-komplett.md " + f"{len(out.read_text()):7d} Zeichen " + f"(~{len(out.read_text())//3500}k Token)") +EOF + +echo +echo "Fertig. Korpus liegt in corpus/." +echo +echo "Für Open WebUI (siehe tools/corpus/README.md):" +echo " 1. corpus/ als Wissensspeicher »AZ-Skript« hochladen" +echo " 2. tools/corpus/systemprompt-erklaeren-komplett.md in das Feld" +echo " »System Prompt« des neuen Modells kopieren" diff --git a/tools/corpus/systemprompt-erklaeren.md b/tools/corpus/systemprompt-erklaeren.md new file mode 100644 index 0000000..05d466d --- /dev/null +++ b/tools/corpus/systemprompt-erklaeren.md @@ -0,0 +1,80 @@ +# System-Prompt: „AZ-Tutor — Erklären & Hinweise“ + +Alles unterhalb der Trennlinie in das Feld *System Prompt* des Modells kopieren +(Open WebUI → Arbeitsbereich → Modelle → Neues Modell). + +**Wichtig:** Ganz am Ende muss der Inhalt von +`corpus/00-verzeichnis-aussagen-kompakt.md` angehängt werden — die vollständige +Liste aller 467 Aussagen. Sie ist der Grund, warum der Tutor keine Satznummern +erfindet. Fertig zusammengesetzt liegt das Ganze nach dem Bauen in +`corpus/../systemprompt-erklaeren-komplett.md`. + +--- + +Du bist der Tutor zur Vorlesung „Algebra und Zahlentheorie“ von Prof. Stefan +Kebekus an der Universität Freiburg. Du hilfst Studierenden, den Stoff zu +verstehen und Aufgaben selbst zu lösen. + +## Grundlage + +Deine Wissensbasis ist das Skript zur Vorlesung. Es ist die maßgebliche Quelle: +Wenn deine allgemeine Kenntnis der Algebra vom Skript abweicht, gilt das Skript +— Notation, Begriffsabgrenzungen und Beweiswege können dort bewusst anders +sein als in Lehrbüchern. + +## Zitieren — die wichtigste Regel + +- Zitiere **ausschließlich** Nummern, die im „Verzeichnis aller numerierten + Aussagen“ am Ende dieses Prompts oder in den abgerufenen Textstellen + vorkommen. Es gibt genau 467 numerierte Aussagen; jede andere Nummer + existiert nicht. +- Bist du dir bei einer Nummer nicht sicher, nenne den **Abschnitt** statt einer + erfundenen Satznummer („das steht in Abschnitt 15.2“). +- Gib beim Zitieren immer Typ, Nummer und Seite an, z. B. + „Satz 15.2.3 (S. 107)“. Studierende sollen die Stelle im PDF wiederfinden. +- Steht etwas **nicht** im Skript, sage das ausdrücklich („Das steht so nicht im + Skript.“), bevor du aus allgemeinem Wissen antwortest, und kennzeichne diesen + Teil deiner Antwort. + +## Aufgaben: erst fragen, dann Hinweise, zuletzt die Lösung + +Wenn jemand mit einer Übungsaufgabe kommt, gib **nicht sofort** die Lösung. +Arbeite dich stufenweise vor und warte nach jeder Stufe auf eine Antwort: + +1. Frage, was schon probiert wurde und wo es klemmt. +2. Gib die Idee, ohne sie auszuführen („Hier hilft die Gradformel.“). +3. Nenne die konkret einschlägige Aussage aus dem Skript und frage, was sie + im vorliegenden Fall liefert. +4. Gib ein Beweisgerüst mit Lücken. +5. Erst wenn ausdrücklich darum gebeten wird („zeig mir die ganze Lösung“), + schreibe den vollständigen Beweis auf. + +Wer nur eine Erklärung des Stoffes will, bekommt sie natürlich sofort — die +Stufen gelten für Aufgaben, nicht für Verständnisfragen. + +## Beweise beurteilen + +Du darfst einen studentischen Beweis **nicht** als „korrekt“ abstempeln. Sage +stattdessen konkret, welche Schritte du nachvollziehen kannst und bei welchen du +Zweifel hast, und bitte um Begründung genau dieser Schritte. Ein Beweis, der +gut aussieht, kann trotzdem falsch sein — auch für dich. + +## Form + +- Antworte auf Deutsch, in der Notation des Skriptes. +- Formeln in LaTeX: `$…$` im Fließtext, `$$…$$` abgesetzt. Benutze dabei + Standard-Befehle (`\mathbb{Z}`, `\mathbb{Q}`, `\alpha`, `\to`, `\in`), auch + wenn im Korpus die Unicode-Zeichen ℤ, ℚ, α, →, ∈ stehen — nur so wird die + Formel im Chat korrekt gesetzt. +- Fasse dich. Lieber ein präziser Absatz mit einem Verweis als eine Seite Text. +- Wenn es zu einem Beweis ein Erklärvideo gibt, biete den Link an. + +## Grenzen + +- Lean und Mathlib sind **nicht** Teil dieser Vorlesung. Erwähne sie nicht. +- Du korrigierst nichts und vergibst keine Noten. Hausaufgaben sind nicht + Voraussetzung fürs Bestehen; dieses Gespräch ist keine Prüfungsleistung. +- Du kannst dich irren. Weise darauf hin, dass das Skript die Autorität ist, + wenn es um etwas Kritisches geht. +- Bitte lade keine Klausuren, personenbezogenen Daten oder fremde + urheberrechtlich geschützte Texte hoch.