From 27ecf5d4ba769ddddbd2f4d2a19f7ca083021897 Mon Sep 17 00:00:00 2001 From: "David M. Klein" <4+leander19961@noreply.localhost> Date: Wed, 26 Aug 2026 15:18:52 +0200 Subject: [PATCH] Dateien nach "docker/syncer" hochladen --- docker/syncer/Dockerfile | 32 +++ docker/syncer/appstream-gen.py | 328 +++++++++++++++++++++++++++++ docker/syncer/entrypoint.sh | 43 ++++ docker/syncer/sync.sh | 191 +++++++++++++++++ docker/syncer/sync_helpers.py | 367 +++++++++++++++++++++++++++++++++ 5 files changed, 961 insertions(+) create mode 100644 docker/syncer/Dockerfile create mode 100644 docker/syncer/appstream-gen.py create mode 100644 docker/syncer/entrypoint.sh create mode 100644 docker/syncer/sync.sh create mode 100644 docker/syncer/sync_helpers.py diff --git a/docker/syncer/Dockerfile b/docker/syncer/Dockerfile new file mode 100644 index 0000000..a4f7eb4 --- /dev/null +++ b/docker/syncer/Dockerfile @@ -0,0 +1,32 @@ +# Build-Kontext ist das Repo-Root (siehe docker-compose.yml: context: ..), +# deshalb sind alle COPY-Pfade relativ zum Repo-Root angegeben. +FROM almalinux:latest + +# dnf-plugins-core -> "dnf download" und "dnf repomanage" +# createrepo_c -> createrepo_c und modifyrepo_c +# appstream -> appstreamcli fuer die optionale Katalog-Validierung +# python3 -> Interpreter fuer appstream-gen.py +RUN dnf install -y \ + dnf-plugins-core \ + createrepo_c \ + appstream \ + python3 \ + git \ + jq \ + openssh-clients \ + wget \ + && dnf clean all + +RUN wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 \ + -O /usr/local/bin/yq \ + && chmod +x /usr/local/bin/yq + +COPY docker/syncer/sync.sh /usr/local/bin/sync.sh +COPY docker/syncer/sync_helpers.py /usr/local/bin/sync_helpers.py +COPY docker/syncer/appstream-gen.py /usr/local/bin/appstream-gen.py +COPY docker/syncer/entrypoint.sh /entrypoint.sh +RUN chmod +x /usr/local/bin/sync.sh /entrypoint.sh + +# Bewusst kein VOLUME und kein EXPOSE: die Volumes definiert docker-compose.yml, +# und ausgeliefert wird ueber den separaten nginx-Container. +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/docker/syncer/appstream-gen.py b/docker/syncer/appstream-gen.py new file mode 100644 index 0000000..4742fd3 --- /dev/null +++ b/docker/syncer/appstream-gen.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +""" +Erzeugt einen AppStream-Katalog (Collection-XML) aus packages.yaml. + +Der Katalog referenziert die gespiegelten RPMs direkt ueber . +Es werden keine Wrapper-Pakete gebaut. Das Ergebnis wird per +`modifyrepo_c --mdtype=appstream` in die Repo-Metadaten eingehaengt, +damit Discover / GNOME Software die Pakete findet. + +Aufruf: + appstream-gen.py packages.yaml -o appstream.xml.gz \ + [--available-packages vorhanden.txt] [--origin firmen-repo] +""" + +import argparse +import gzip +import re +import sys +import xml.etree.ElementTree as ET +from xml.dom import minidom + +import yaml + +# AppStream-Spezifikationsversion des Katalogformats. +CATALOG_VERSION = "0.14" + +DEFAULT_TYPE = "console-application" +DEFAULT_ICON = "application-x-executable" +DEFAULT_NOTE = ( + "Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. " + "Fragen und Paketwünsche bitte an den IT-Service-Desk." +) + + +def die(msg): + sys.stdout.flush() + print(f"[appstream-gen] FEHLER: {msg}", file=sys.stderr, flush=True) + sys.exit(1) + + +def warn(msg): + sys.stdout.flush() + print(f"[appstream-gen] WARNUNG: {msg}", file=sys.stderr, flush=True) + + +def info(msg): + print(f"[appstream-gen] {msg}", flush=True) + + +def sanitize_id_part(value): + """Macht aus einem Paketnamen ein gueltiges AppStream-ID-Segment.""" + part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-") + if not part: + die(f"Paketname ergibt kein gueltiges ID-Segment: {value!r}") + if part[0].isdigit(): + part = "pkg-" + part + return part + + +def prettify_name(pkgname): + """Fallback-Anzeigename, wenn in der YAML keiner gesetzt ist. + + dotnet-sdk-9.0 -> Dotnet Sdk 9.0 + """ + words = re.split(r"[-_]", pkgname) + return " ".join(w[:1].upper() + w[1:] if w else w for w in words) + + +def as_list(value): + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + return [value] + + +def normalize_entry(entry, source_name): + """Akzeptiert sowohl 'paketname' als auch ein Mapping mit Metadaten.""" + if isinstance(entry, str): + return {"name": entry, "_source": source_name} + if isinstance(entry, dict): + if "name" not in entry: + die(f"Paketeintrag in Quelle {source_name!r} ohne 'name': {entry!r}") + data = dict(entry) + data["_source"] = source_name + return data + die(f"Unerwarteter Paketeintrag in Quelle {source_name!r}: {entry!r}") + + +def pick(entry, defaults, key, fallback=None): + """Wert aus dem Paketeintrag, sonst aus defaults, sonst fallback.""" + if key in entry and entry[key] is not None: + return entry[key] + if key in defaults and defaults[key] is not None: + return defaults[key] + return fallback + + +def build_component(entry, defaults, id_prefix): + pkgname = entry["name"] + + comp_type = pick(entry, defaults, "type", DEFAULT_TYPE) + component = ET.Element("component", {"type": comp_type}) + + comp_id = entry.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}" + ET.SubElement(component, "id").text = comp_id + ET.SubElement(component, "pkgname").text = pkgname + + display_name = entry.get("display_name") or entry.get("name_display") + if not display_name: + display_name = prettify_name(pkgname) + ET.SubElement(component, "name").text = display_name + + note = pick(entry, defaults, "note", DEFAULT_NOTE) + summary = entry.get("summary") or note + ET.SubElement(component, "summary").text = summary + + # Beschreibung: eigener Text plus Hinweis auf die Herkunft. + description = ET.SubElement(component, "description") + body = entry.get("description") + if body: + for paragraph in [p.strip() for p in str(body).split("\n\n") if p.strip()]: + ET.SubElement(description, "p").text = paragraph + if note and note != summary: + ET.SubElement(description, "p").text = note + elif not body and note == summary: + ET.SubElement(description, "p").text = note + + icon = pick(entry, defaults, "icon", DEFAULT_ICON) + if icon: + ET.SubElement(component, "icon", {"type": "stock"}).text = icon + + categories = as_list(pick(entry, defaults, "categories")) + if categories: + cat_el = ET.SubElement(component, "categories") + for category in categories: + ET.SubElement(cat_el, "category").text = str(category) + + keywords = as_list(entry.get("keywords")) + if keywords: + kw_el = ET.SubElement(component, "keywords") + for keyword in keywords: + ET.SubElement(kw_el, "keyword").text = str(keyword) + + binaries = as_list(entry.get("binary") or entry.get("binaries")) + if binaries: + provides = ET.SubElement(component, "provides") + for binary in binaries: + ET.SubElement(provides, "binary").text = str(binary) + + homepage = entry.get("homepage") or defaults.get("homepage") + if homepage: + ET.SubElement(component, "url", {"type": "homepage"}).text = str(homepage) + + help_url = entry.get("help_url") or defaults.get("help_url") + if help_url: + ET.SubElement(component, "url", {"type": "help"}).text = str(help_url) + + developer = entry.get("developer") or defaults.get("developer") + if developer: + # Das id-Attribut ist seit AppStream 1.0 erwartet; ohne es meldet + # appstreamcli 'developer-id-missing'. + dev_id = entry.get("developer_id") or defaults.get("developer_id") + attrs = {"id": str(dev_id)} if dev_id else {} + dev_el = ET.SubElement(component, "developer", attrs) + ET.SubElement(dev_el, "name").text = str(developer) + + project_license = entry.get("project_license") or defaults.get("project_license") + if project_license: + ET.SubElement(component, "project_license").text = str(project_license) + + # desktop-application braucht einen Launchable-Eintrag, sonst ignoriert + # Discover die Komponente beim Starten der installierten App. + if comp_type == "desktop-application": + desktop_id = entry.get("desktop_id") + if desktop_id: + ET.SubElement( + component, "launchable", {"type": "desktop-id"} + ).text = str(desktop_id) + else: + warn( + f"{pkgname}: type=desktop-application ohne 'desktop_id'. " + "Discover kann die Anwendung dann nicht starten." + ) + + return comp_id, component + + +def load_available(path): + if not path: + return None + try: + with open(path, "r", encoding="utf-8") as handle: + names = {line.strip() for line in handle if line.strip()} + except OSError as exc: + die(f"Konnte Paketliste {path!r} nicht lesen: {exc}") + info(f"{len(names)} tatsaechlich vorhandene Pakete eingelesen") + return names + + +def main(): + parser = argparse.ArgumentParser( + description="Erzeugt einen AppStream-Katalog aus packages.yaml" + ) + parser.add_argument("packages_yaml", help="Pfad zur packages.yaml") + parser.add_argument( + "-o", "--output", default="appstream.xml.gz", + help="Zieldatei (.xml.gz oder .xml), Standard: appstream.xml.gz", + ) + parser.add_argument( + "--origin", + help="Origin des Katalogs. Ueberschreibt 'appstream.origin' aus der YAML.", + ) + parser.add_argument( + "--available-packages", + help="Datei mit den tatsaechlich gespiegelten Paketnamen (einer pro Zeile). " + "Nicht vorhandene Pakete werden dann uebersprungen.", + ) + parser.add_argument( + "--source", + help="Nur Pakete dieser einen Quelle beruecksichtigen. Fuer Setups " + "mit einem Repo-Verzeichnis pro Quelle.", + ) + parser.add_argument( + "--strict", action="store_true", + help="Mit Fehler abbrechen, wenn ein gelistetes Paket nicht vorhanden ist.", + ) + args = parser.parse_args() + + try: + with open(args.packages_yaml, "r", encoding="utf-8") as handle: + config = yaml.safe_load(handle) or {} + except OSError as exc: + die(f"Konnte {args.packages_yaml!r} nicht lesen: {exc}") + except yaml.YAMLError as exc: + die(f"YAML-Fehler in {args.packages_yaml!r}: {exc}") + + appstream_cfg = config.get("appstream") or {} + defaults = appstream_cfg.get("defaults") or {} + origin = args.origin or appstream_cfg.get("origin") or "firmen-repo" + id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo" + + # Falls kein developer_id gesetzt ist, die ersten beiden Segmente des + # id_prefix verwenden: de.firma.repo -> de.firma + if "developer_id" not in defaults or not defaults.get("developer_id"): + segments = id_prefix.split(".") + if len(segments) >= 2: + defaults = dict(defaults) + defaults["developer_id"] = ".".join(segments[:2]) + + available = load_available(args.available_packages) + + sources = config.get("sources") or [] + if not sources: + die("Keine 'sources' in der packages.yaml gefunden.") + + if args.source: + sources = [s for s in sources if s.get("name") == args.source] + if not sources: + die(f"Keine Quelle mit dem Namen {args.source!r} in der YAML gefunden.") + info(f"Beschraenke auf Quelle {args.source!r}") + + root = ET.Element("components", {"version": CATALOG_VERSION, "origin": origin}) + + seen_ids = {} + seen_pkgs = set() + skipped = [] + count = 0 + + for source in sources: + source_name = source.get("name", "") + for raw_entry in source.get("packages") or []: + entry = normalize_entry(raw_entry, source_name) + pkgname = entry["name"] + + if entry.get("appstream") is False or entry.get("hidden") is True: + info(f"{pkgname}: per Konfiguration von AppStream ausgenommen") + continue + + if available is not None and pkgname not in available: + skipped.append(pkgname) + continue + + if pkgname in seen_pkgs: + warn(f"{pkgname}: mehrfach gelistet, spaetere Eintraege ignoriert") + continue + seen_pkgs.add(pkgname) + + comp_id, component = build_component(entry, defaults, id_prefix) + if comp_id in seen_ids: + die( + f"Doppelte AppStream-ID {comp_id!r} " + f"({seen_ids[comp_id]} und {pkgname}). " + "Setze 'id:' fuer eines der beiden Pakete." + ) + seen_ids[comp_id] = pkgname + root.append(component) + count += 1 + + if skipped: + message = ( + f"{len(skipped)} gelistete Pakete sind nicht im Repo vorhanden " + f"und wurden uebersprungen: {', '.join(sorted(skipped))}" + ) + if args.strict: + die(message) + warn(message) + + raw = ET.tostring(root, encoding="utf-8") + pretty = minidom.parseString(raw).toprettyxml(indent=" ", encoding="UTF-8") + + try: + if args.output.endswith(".gz"): + with gzip.open(args.output, "wb") as handle: + handle.write(pretty) + else: + with open(args.output, "wb") as handle: + handle.write(pretty) + except OSError as exc: + die(f"Konnte {args.output!r} nicht schreiben: {exc}") + + info(f"{count} Komponente(n) nach {args.output} geschrieben (origin={origin})") + if count == 0: + warn("Katalog ist leer. Discover wird nichts anzeigen.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docker/syncer/entrypoint.sh b/docker/syncer/entrypoint.sh new file mode 100644 index 0000000..40d9ff5 --- /dev/null +++ b/docker/syncer/entrypoint.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Container entrypoint: run sync.sh once, or in a loop every +# SYNC_INTERVAL_SECONDS. There's no systemd/cron inside the container, so a +# simple sleep-loop is the pragmatic choice. +set -euo pipefail + +: "${SYNC_INTERVAL_SECONDS:=3600}" +: "${RUN_ONCE:=false}" + +log() { echo "[entrypoint] $(date '+%Y-%m-%d %H:%M:%S') $*"; } + +run_once() { + /usr/local/bin/sync.sh +} + +if [[ "$RUN_ONCE" == "true" ]]; then + log "RUN_ONCE=true - running sync.sh once and exiting" + run_once + exit $? +fi + +log "Starting sync loop (interval: ${SYNC_INTERVAL_SECONDS}s)" + +# Graceful shutdown on docker stop / compose down +term_handler() { + log "Received SIGTERM, shutting down" + exit 0 +} +trap term_handler SIGTERM SIGINT + +while true; do + # NOTE: don't inline `run_once` into `if ! run_once; then ...` - inside + # that then-branch, $? would reflect the (always-0) negated test itself, + # not run_once's real exit code. Capture it explicitly instead. + rc=0 + run_once || rc=$? + if [[ $rc -ne 0 ]]; then + log "sync.sh failed (exit ${rc}) - will retry after the next interval" + fi + log "Sleeping ${SYNC_INTERVAL_SECONDS}s until next sync..." + sleep "${SYNC_INTERVAL_SECONDS}" & + wait $! +done \ No newline at end of file diff --git a/docker/syncer/sync.sh b/docker/syncer/sync.sh new file mode 100644 index 0000000..b14efff --- /dev/null +++ b/docker/syncer/sync.sh @@ -0,0 +1,191 @@ +#!/bin/bash +# Synct die Paket-Whitelist aus Git (oder aus einer lokal gemounteten Datei) +# und laedt die selektierten Pakete aus den jeweils angegebenen Fremd-Repos. +# Erzeugt anschliessend RPM- und - sofern der Generator vorhanden ist - +# AppStream-Metadaten. +set -euo pipefail + +# createrepo_c uebernimmt die umask des Aufrufers. Ohne "umask 022" kann +# repodata/ mit 0700 landen; der nginx-Worker (User nginx) kommt dann nicht +# hinein und liefert 403 - bzw. 404, falls die nginx-Config try_files nutzt. +umask 022 + +# --- Pfade ---------------------------------------------------------------- +# Muessen zu den Volumes in docker-compose.yml passen: +# repo-data -> /srv/repo (im nginx-Container: /usr/share/nginx/html/repo) +# sync-cache -> /var/cache/rpm-mirror-sync +# Daraus ergibt sich die oeffentliche URL: /repo/pkgs/repodata/repomd.xml +WORK_DIR="${WORK_DIR:-/var/cache/rpm-mirror-sync}" +REPO_ROOT="${REPO_ROOT:-/srv/repo}" +DEST="$REPO_ROOT/pkgs" + +CONFIG_DIR="$WORK_DIR/config" +LOCAL_PACKAGES_FILE=/config/packages.yaml +TMP_REPO_DIR=/etc/yum.repos.d +LOCKFILE="$WORK_DIR/sync.lock" +APPSTREAM_GEN="${APPSTREAM_GEN:-/usr/local/bin/appstream-gen.py}" + +mkdir -p "$WORK_DIR" "$DEST" + +exec 200>"$LOCKFILE" +if ! flock -n 200; then + echo "[sync] Ein anderer Sync-Lauf ist bereits aktiv, breche ab." + exit 0 +fi + +echo "[sync] $(date -Iseconds) Starte Sync-Lauf" + +# --- 1. Whitelist besorgen ------------------------------------------------ +# Eine gemountete /config/packages.yaml hat Vorrang. Nur wenn sie fehlt, +# wird SOURCES_GIT_REPO gebraucht. +if [ -f "$LOCAL_PACKAGES_FILE" ]; then + PACKAGES_FILE="$LOCAL_PACKAGES_FILE" + echo "[sync] Nutze lokal gemountete $PACKAGES_FILE" +else + : "${SOURCES_GIT_REPO:?weder SOURCES_GIT_REPO gesetzt noch /config/packages.yaml gemountet}" + CONFIG_REPO_BRANCH="${CONFIG_REPO_BRANCH:-main}" + PACKAGES_FILE="$CONFIG_DIR/packages.yaml" + + if [ -n "${GIT_SSH_KEY_PATH:-}" ] && [ -f "$GIT_SSH_KEY_PATH" ]; then + export GIT_SSH_COMMAND="ssh -i ${GIT_SSH_KEY_PATH} -o StrictHostKeyChecking=accept-new" + fi + + if [ -d "$CONFIG_DIR/.git" ]; then + git -C "$CONFIG_DIR" fetch --depth=1 origin "$CONFIG_REPO_BRANCH" + git -C "$CONFIG_DIR" reset --hard "origin/$CONFIG_REPO_BRANCH" + else + # Reste eines abgebrochenen Clones wegraeumen, sonst scheitert git clone. + rm -rf "$CONFIG_DIR" + git clone --depth=1 --branch "$CONFIG_REPO_BRANCH" "$SOURCES_GIT_REPO" "$CONFIG_DIR" + fi +fi + +if [ ! -f "$PACKAGES_FILE" ]; then + echo "[sync] FEHLER: $PACKAGES_FILE nicht gefunden." + exit 1 +fi + +# --- 2. Pro Quelle: temporaeres Repo aktivieren, Pakete gezielt ziehen ----- +source_count=$(yq -o=json '.sources | length' "$PACKAGES_FILE") +download_failed=0 + +for i in $(seq 0 $((source_count - 1))); do + src=$(yq -o=json ".sources[$i]" "$PACKAGES_FILE") + name=$(echo "$src" | jq -r '.name') + baseurl=$(echo "$src" | jq -r '.baseurl') + gpgkey=$(echo "$src" | jq -r '.gpgkey // empty') + + # Standard: false. Nur bei echten Fremd-Repos aktivieren, deren + # Abhaengigkeiten die Clients nicht ueber ihre Basis-Repos bekommen. + resolve_deps=$(echo "$src" | jq -r '.resolve_deps // false') + + # Paketeintraege koennen Strings ODER Mappings mit Metadaten sein. + mapfile -t packages < <( + echo "$src" | jq -r '.packages[] | if type == "string" then . else .name end' + ) + + if [ "${#packages[@]}" -eq 0 ]; then + echo "[sync] Quelle '$name': keine Pakete gelistet, ueberspringe." + continue + fi + + echo "[sync] Quelle '$name': ${#packages[@]} Paket(e) -> $baseurl" + + repofile="$TMP_REPO_DIR/tmp-${name}.repo" + { + echo "[tmp-${name}]" + echo "name=${name}" + echo "baseurl=${baseurl}" + echo "enabled=0" + if [ -n "$gpgkey" ]; then + echo "gpgcheck=1" + echo "gpgkey=${gpgkey}" + else + echo "gpgcheck=0" + fi + } > "$repofile" + + # Die Basis-Repos des Containers bleiben aktiviert, damit dnf + # Abhaengigkeiten wie glibc oder libstdc++ ueberhaupt aufloesen kann. + # Ohne sie scheitert bereits die Aufloesung, nicht erst der Download. + dl_args=(--destdir="$DEST" --enablerepo="tmp-${name}") + + if [ "$resolve_deps" = "true" ]; then + echo "[sync] Quelle '$name': Abhaengigkeiten werden mitgespiegelt" + # Bewusst OHNE --alldeps: bereits im Container installierte Pakete + # (glibc & Co.) haben die Clients ohnehin und muessen nicht ins Repo. + dl_args+=(--resolve) + fi + + if ! dnf download "${dl_args[@]}" "${packages[@]}"; then + echo "[sync] WARNUNG: Download fuer Quelle '$name' fehlgeschlagen." + download_failed=1 + fi + + rm -f "$repofile" +done + +# --- 3. Alte Paketversionen aufraeumen ------------------------------------ +# repomanage braucht bereits vorhandene Metadaten. Beim allerersten Lauf +# gibt es noch keine, deshalb der Existenztest. +if [ -f "$DEST/repodata/repomd.xml" ]; then + echo "[sync] Raeume veraltete Paketversionen auf" + if ! dnf repomanage --old "$DEST" | xargs -r rm -f; then + echo "[sync] WARNUNG: repomanage fehlgeschlagen, ueberspringe Aufraeumen." + fi +else + echo "[sync] Erster Lauf, kein Aufraeumen noetig." +fi + +# --- 4. RPM-Metadaten neu erzeugen ---------------------------------------- +echo "[sync] Aktualisiere Repo-Metadaten" +createrepo_c --update "$DEST" + +# --- 5. AppStream-Katalog erzeugen und einhaengen -------------------------- +# Ohne diesen Schritt sind die Pakete zwar per dnf installierbar, tauchen +# aber weder in Discover noch in GNOME Software auf. Der Generator ist +# optional: fehlt er, laeuft der Sync trotzdem sauber durch. +if [ -x "$APPSTREAM_GEN" ]; then + echo "[sync] Erzeuge AppStream-Katalog" + + ASWORK=$(mktemp -d) + trap 'rm -rf "$ASWORK"' EXIT + + # Welche Pakete liegen tatsaechlich im Repo? Verhindert, dass Discover + # Eintraege anzeigt, deren RPM gar nicht da ist. + if compgen -G "$DEST"/*.rpm > /dev/null; then + rpm -qp --qf '%{NAME}\n' "$DEST"/*.rpm 2>/dev/null \ + | sort -u > "$ASWORK/vorhanden.txt" + else + echo "[sync] WARNUNG: Keine RPMs in $DEST gefunden." + : > "$ASWORK/vorhanden.txt" + fi + + python3 "$APPSTREAM_GEN" "$PACKAGES_FILE" \ + --output "$ASWORK/appstream.xml.gz" \ + --available-packages "$ASWORK/vorhanden.txt" + + # Optionale Validierung, falls appstreamcli im Image vorhanden ist. + if command -v appstreamcli > /dev/null 2>&1; then + if ! appstreamcli validate --no-net "$ASWORK/appstream.xml.gz"; then + echo "[sync] WARNUNG: AppStream-Katalog hat Validierungshinweise." + fi + fi + + # modifyrepo_c MUSS nach createrepo_c laufen, da createrepo_c die + # repomd.xml komplett neu schreibt. + modifyrepo_c --mdtype=appstream "$ASWORK/appstream.xml.gz" "$DEST/repodata/" +else + echo "[sync] HINWEIS: $APPSTREAM_GEN nicht vorhanden - AppStream wird uebersprungen." +fi + +# --- 6. Leserechte fuer den Webserver sicherstellen ------------------------ +# a+rX = Leserecht auf alles, Betretungsrecht nur auf Verzeichnisse. +chmod -R a+rX "$REPO_ROOT" + +if [ "$download_failed" -ne 0 ]; then + echo "[sync] $(date -Iseconds) Sync-Lauf mit Fehlern abgeschlossen" + exit 1 +fi + +echo "[sync] $(date -Iseconds) Sync-Lauf abgeschlossen" \ No newline at end of file diff --git a/docker/syncer/sync_helpers.py b/docker/syncer/sync_helpers.py new file mode 100644 index 0000000..0cd96f8 --- /dev/null +++ b/docker/syncer/sync_helpers.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +""" +sync_helpers.py - small helper used by sync.sh + +Reliable YAML parsing and XML escaping in pure bash/sed is a losing game, so +these two jobs live here: + + parse-yaml + Reads the "sources:" list from the YAML file, NORMALISES it and prints + it as JSON on stdout, so sync.sh can loop over it safely. + + Normalisation: every entry under "packages:" becomes an object with at + least a "name" key, whether it was written as a plain string or as a + mapping with extra AppStream metadata. sync.sh can therefore always do + p["name"] without caring which form the YAML used. + + gen-appstream --origin O --yaml F --entries-dir D + Builds the AppStream catalog for the combined repo. + + Metadata precedence, highest first: + 1. what packages.yaml says about the package + 2. the upstream RPM summary collected by sync.sh into the + appstream-entries.tsv files under --entries-dir + 3. a generated fallback + + Only packages that actually made it into the mirror (i.e. that appear + in one of the TSV files) get a component - never pulled-in + dependencies, and never packages whose download failed. + + NOTE ON DISCOVER/GNOME SOFTWARE: + Components without an icon/screenshots/.desktop file make a package + installable and findable via SEARCH, but they will not appear on the + curated "Explore" front page or when browsing categories. That is a + frontend behaviour for non-desktop-app components and cannot be fixed + at the repo-metadata level. Only a real desktop-application component + (type: desktop-application + desktop_id: in packages.yaml, which + requires the package to ship a .desktop file) shows up there. +""" +import argparse +import glob +import json +import os +import re +import sys +from xml.sax.saxutils import escape, quoteattr + +CATALOG_VERSION = "0.14" +DEFAULT_NOTE = ( + "Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. " + "Fragen und Paketwünsche bitte an den IT-Service-Desk." +) + + +def die(msg): + sys.stdout.flush() + sys.stderr.write(f"ERROR: {msg}\n") + sys.exit(1) + + +def warn(msg): + sys.stdout.flush() + sys.stderr.write(f"WARNING: {msg}\n") + + +def load_yaml(path): + try: + import yaml + except ImportError: + die( + "python3-pyyaml (PyYAML) is not installed.\n" + "Install it with: dnf install -y python3-pyyaml" + ) + try: + with open(path, "r", encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + except OSError as exc: + die(f"could not read {path!r}: {exc}") + except Exception as exc: # yaml.YAMLError and friends + die(f"could not parse {path!r}: {exc}") + + +def normalise_package(entry, source_name, index): + """Accept both 'pkgname' and {name: pkgname, display_name: ...}.""" + if isinstance(entry, str): + return {"name": entry} + if isinstance(entry, dict): + if not entry.get("name"): + die( + f"package #{index} in source {source_name!r} has no 'name' key: " + f"{entry!r}" + ) + return dict(entry) + die( + f"package #{index} in source {source_name!r} must be a string or a " + f"mapping, got {type(entry).__name__}" + ) + + +def normalise_sources(data): + sources = data.get("sources") or [] + if not isinstance(sources, list): + die("'sources' in YAML is not a list") + + for i, src in enumerate(sources): + if not isinstance(src, dict): + die(f"source #{i} is not a mapping") + for field in ("name", "baseurl", "packages"): + if field not in src: + die(f"source #{i} is missing required field '{field}'") + if not isinstance(src["packages"], list) or not src["packages"]: + die( + f"source '{src.get('name')}' has an empty/invalid packages list" + ) + src["packages"] = [ + normalise_package(p, src.get("name"), j) + for j, p in enumerate(src["packages"]) + ] + return sources + + +def cmd_parse_yaml(path): + data = load_yaml(path) + json.dump(normalise_sources(data), sys.stdout) + + +# --------------------------------------------------------------------------- +# AppStream catalog generation +# --------------------------------------------------------------------------- + +def sanitize_id_part(value): + part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-") + if not part: + die(f"package name yields no usable AppStream id segment: {value!r}") + if part[0].isdigit(): + part = "pkg-" + part + return part + + +def as_list(value): + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + return [value] + + +def pick(meta, defaults, key, fallback=None): + if meta.get(key) is not None: + return meta[key] + if defaults.get(key) is not None: + return defaults[key] + return fallback + + +def read_entries(entries_dir): + """Merge every source's appstream-entries.tsv into {name: summary}.""" + entries = {} + pattern = os.path.join(entries_dir, "*", "appstream-entries.tsv") + for path in sorted(glob.glob(pattern)): + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.rstrip("\n") + if not line: + continue + parts = line.split("\t", 1) + name = parts[0].strip() + summary = parts[1].strip() if len(parts) > 1 else "" + if not name: + continue + if name in entries and entries[name] and summary \ + and entries[name] != summary: + warn( + f"package '{name}' was contributed by more than one " + "source with differing summaries - keeping the first" + ) + continue + entries.setdefault(name, summary) + except OSError: + continue + return entries + + +def collect_metadata(yaml_path): + """Return (defaults, {pkgname: metadata}, id_prefix, yaml_origin).""" + data = load_yaml(yaml_path) + appstream_cfg = data.get("appstream") or {} + defaults = dict(appstream_cfg.get("defaults") or {}) + id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo" + yaml_origin = appstream_cfg.get("origin") + + if not defaults.get("developer_id"): + segments = id_prefix.split(".") + if len(segments) >= 2: + defaults["developer_id"] = ".".join(segments[:2]) + + by_name = {} + for src in normalise_sources(data): + for pkg in src["packages"]: + name = pkg["name"] + if name in by_name: + warn(f"package '{name}' listed more than once - keeping the first") + continue + by_name[name] = pkg + return defaults, by_name, id_prefix, yaml_origin + + +def emit(out, depth, text): + out.append(" " * depth + text) + + +def build_component(pkgname, meta, upstream_summary, defaults, id_prefix, out): + if meta.get("appstream") is False or meta.get("hidden") is True: + return False + + binaries = as_list(meta.get("binary") or meta.get("binaries")) + + # A console-application component without trips + # appstreamcli's 'console-app-no-binary' check, so only claim that type + # when a binary is actually declared. Runtimes and libraries stay + # 'generic', which Discover still finds via search. + comp_type = pick(meta, defaults, "type") + if not comp_type: + comp_type = "console-application" if binaries else "generic" + + comp_id = meta.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}" + + display_name = meta.get("display_name") or pkgname + + note = pick(meta, defaults, "note", DEFAULT_NOTE) + summary = meta.get("summary") or upstream_summary or note + + emit(out, 1, f"") + emit(out, 2, f"{escape(comp_id)}") + emit(out, 2, f"{escape(pkgname)}") + emit(out, 2, f"{escape(display_name)}") + emit(out, 2, f"{escape(summary)}") + + emit(out, 2, "") + body = meta.get("description") + wrote_paragraph = False + if body: + for para in [p.strip() for p in str(body).split("\n\n") if p.strip()]: + emit(out, 3, f"

{escape(para)}

") + wrote_paragraph = True + if note and (not wrote_paragraph or note != summary): + emit(out, 3, f"

{escape(note)}

") + elif not wrote_paragraph: + emit(out, 3, f"

{escape(summary)}

") + emit(out, 2, "") + + icon = pick(meta, defaults, "icon", "application-x-executable") + if icon: + emit(out, 2, f'{escape(str(icon))}') + + categories = as_list(pick(meta, defaults, "categories")) + if categories: + emit(out, 2, "") + for cat in categories: + emit(out, 3, f"{escape(str(cat))}") + emit(out, 2, "") + + keywords = as_list(meta.get("keywords")) + if keywords: + emit(out, 2, "") + for kw in keywords: + emit(out, 3, f"{escape(str(kw))}") + emit(out, 2, "") + + if binaries: + emit(out, 2, "") + for binary in binaries: + emit(out, 3, f"{escape(str(binary))}") + emit(out, 2, "") + + homepage = meta.get("homepage") or defaults.get("homepage") + if homepage: + emit(out, 2, f'{escape(str(homepage))}') + help_url = meta.get("help_url") or defaults.get("help_url") + if help_url: + emit(out, 2, f'{escape(str(help_url))}') + + developer = meta.get("developer") or defaults.get("developer") + if developer: + dev_id = meta.get("developer_id") or defaults.get("developer_id") + attr = f" id={quoteattr(str(dev_id))}" if dev_id else "" + emit(out, 2, f"") + emit(out, 3, f"{escape(str(developer))}") + emit(out, 2, "") + + project_license = meta.get("project_license") or defaults.get("project_license") + if project_license: + emit(out, 2, f"{escape(str(project_license))}") + + # A desktop-application component that Discover cannot launch is worse + # than a generic one, so insist on the desktop file id. + if comp_type == "desktop-application": + desktop_id = meta.get("desktop_id") + if desktop_id: + emit( + out, 2, + f'{escape(str(desktop_id))}', + ) + else: + warn( + f"{pkgname}: type=desktop-application without 'desktop_id' - " + "Discover will not be able to launch it after installation" + ) + + emit(out, 1, "") + return True + + +def cmd_gen_appstream(origin, yaml_path, entries_dir): + defaults, by_name, id_prefix, yaml_origin = collect_metadata(yaml_path) + entries = read_entries(entries_dir) + + if not entries: + warn( + f"no appstream-entries.tsv found under {entries_dir} - the catalog " + "will be empty and nothing will show up in Discover" + ) + + effective_origin = origin or yaml_origin or "rpm-mirror" + + out = [] + out.append('') + out.append( + f'' + ) + + written = 0 + for pkgname in sorted(entries): + meta = by_name.get(pkgname, {}) + if build_component(pkgname, meta, entries[pkgname], defaults, + id_prefix, out): + written += 1 + + out.append("") + sys.stdout.write("\n".join(out) + "\n") + sys.stderr.write( + f"gen-appstream: wrote {written} component(s), origin={effective_origin}\n" + ) + + +def main(): + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + + p1 = sub.add_parser("parse-yaml") + p1.add_argument("path") + + p2 = sub.add_parser("gen-appstream") + p2.add_argument("--origin", default="") + p2.add_argument("--yaml", required=True) + p2.add_argument("--entries-dir", required=True) + + args = ap.parse_args() + + if args.cmd == "parse-yaml": + cmd_parse_yaml(args.path) + elif args.cmd == "gen-appstream": + cmd_gen_appstream(args.origin, args.yaml, args.entries_dir) + + +if __name__ == "__main__": + main() \ No newline at end of file