diff --git a/docker/.env-example b/docker/.env-example deleted file mode 100644 index 87b465e..0000000 --- a/docker/.env-example +++ /dev/null @@ -1,80 +0,0 @@ -# Copy this file to .env. docker-compose.yml loads it via env_file for both -# containers, and docker compose itself also uses it to resolve ${VAR} -# references inside docker-compose.yml (e.g. NGINX_PORT, LOCAL_SOURCES_YAML). -# This file is the single place to configure everything - nothing is -# hardcoded in docker-compose.yml itself. - -# ============================================================================ -# Where the sources.yaml comes from -# ============================================================================ - -# Option A (default): local file, bind-mounted into the syncer container. -# Point LOCAL_SOURCES_YAML at your real file (path is relative to -# docker-compose.yml, defaults to the bundled sources.example.yaml) and -# leave SOURCES_GIT_REPO empty. SOURCES_YAML_FILE must then be the absolute -# in-container path the bind mount uses (leave as-is unless you also change -# the mount target in docker-compose.yml). -#LOCAL_SOURCES_YAML=./sources.example.yaml -CONFIG_REPO_URL=https://gitea.leanderserver.de/leander19961/dnf-repo-test -CONFIG_REPO_BRANCH=main -PACKAGES_FILE=packages.yaml - -# Option B: pull the list from git instead. Set SOURCES_GIT_REPO, and -# SOURCES_YAML_FILE to the path of the YAML file *inside* that git repo -# (relative), e.g.: -# SOURCES_GIT_REPO=https://git.example.com/infra/rpm-sources.git -# SOURCES_GIT_BRANCH=main -# SOURCES_YAML_FILE=sources.yaml -# (LOCAL_SOURCES_YAML / the bind mount is then simply unused.) - -# ============================================================================ -# Paths inside the syncer container -# ============================================================================ -# These map to the volumes defined in docker-compose.yml - only change them -# if you also change the corresponding volume mounts. -WORK_DIR=/var/cache/rpm-mirror-sync -REPO_BASE_DIR=/srv/repo -CLIENT_REPO_DIR=/srv/repo/client-repos - -# ============================================================================ -# Client-facing URL -# ============================================================================ -# Public URL under which the nginx container serves the repo. Used to -# generate the .repo files under /repo/client-repos/. Point this at -# wherever the host running nginx is reachable from your AlmaLinux clients -# (should match NGINX_PORT below). -CLIENT_BASE_URL=http://repo.leanderserver.de/repo - -# ============================================================================ -# repoids of AlmaLinux's own repos inside the syncer container -# ============================================================================ -# Check with: docker compose exec syncer dnf repolist -ALMA_REPO_IDS=baseos appstream extras crb - -# ============================================================================ -# Sync schedule -# ============================================================================ -# How often (seconds) the syncer container re-syncs. 3600 = hourly. -SYNC_INTERVAL_SECONDS=3600 -# Set to "true" to sync once and exit instead of looping (e.g. if you'd -# rather trigger this container from an external scheduler). -RUN_ONCE=false - -# ============================================================================ -# Mirror signing key -# ============================================================================ -# Identity used for the ONE GPG key the mirror signs every mirrored package -# with (regardless of how many upstream sources there are). Generated once -# on first run - together with a revocation certificate - and reused -# forever. Both live under GPG_HOME on the sync-cache volume, so don't run -# "docker compose down -v" unless you're fine with clients having to -# re-trust a brand new key afterwards. -GPG_HOME=/var/cache/rpm-mirror-sync/gnupg -GPG_KEY_NAME=RPM Mirror -GPG_KEY_EMAIL=rpm-mirror@example.local - -# ============================================================================ -# Ports -# ============================================================================ -# Host port nginx publishes the repo on. -NGINX_PORT=8081 diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 6d6dfc8..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -FROM almalinux:10 - -# Ohne UTF-8-Locale verstuemmeln appstreamcli und andere Tools Umlaute in -# ihrer Ausgabe ("?ber" statt "über"). -ENV LANG=C.UTF-8 \ - LC_ALL=C.UTF-8 - -# createrepo_c bringt auch modifyrepo_c mit. -# python3 + PyYAML werden von appstream-gen.py gebraucht - nicht darauf -# verlassen, dass das Basisimage sie zufaellig mitbringt. -# gnupg2/rpm-sign sind fuer die spaetere Signierung vorgesehen. -RUN dnf install -y \ - dnf-plugins-core \ - createrepo_c \ - appstream \ - git \ - jq \ - openssh-clients \ - python3 \ - python3-pyyaml \ - gnupg2 \ - rpm-sign \ - findutils \ - util-linux-core \ - curl \ - && dnf clean all - -# yq (Go-Variante) fuer das YAML-Parsing in sync.sh. -# Version gepinnt: "latest" macht Builds nicht reproduzierbar und kann bei -# einem Breaking Change der yq-Syntax den Sync stillschweigend zerlegen. -ARG YQ_VERSION=v4.44.3 -RUN curl -fsSL -o /usr/local/bin/yq \ - "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \ - && chmod +x /usr/local/bin/yq - -COPY sync.sh /usr/local/bin/sync.sh -COPY appstream-gen.py /usr/local/bin/appstream-gen.py -COPY sync_helpers.py /usr/local/bin/sync_helpers.py -COPY entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/sync.sh \ - /usr/local/bin/appstream-gen.py \ - /usr/local/bin/entrypoint.sh - -# Bewusst KEIN "VOLUME /srv/repo": -# Ein VOLUME im Dockerfile legt beim Start ein anonymes Volume an. Wird in -# docker-compose.yml ein benanntes Volume auf einen ANDEREN Pfad gemountet, -# schreibt der Sync ins anonyme Volume, waehrend nginx das benannte (leere) -# ausliefert. Genau dieser Fall ist hier schon einmal aufgetreten. -# Das Mount kommt ausschliesslich aus docker-compose.yml. - -# Ebenfalls bewusst nicht enthalten: nginx und cronie. -# nginx laeuft als eigener Container (siehe docker-compose.yml), das -# Sync-Intervall steuert entrypoint.sh per Sleep-Schleife. - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] \ No newline at end of file diff --git a/docker/appstream-gen.py b/docker/appstream-gen.py deleted file mode 100644 index 2ea9e16..0000000 --- a/docker/appstream-gen.py +++ /dev/null @@ -1,302 +0,0 @@ -#!/usr/bin/env python3 -""" -Erzeugt einen AppStream-Katalog (Collection-XML) aus packages.yaml. - -Der Katalog referenziert die gespiegelten RPMs direkt ueber . -Es werden keine Wrapper-Pakete gebaut. Das Ergebnis wird per -`modifyrepo_c --mdtype=appstream` in die Repo-Metadaten eingehaengt, -damit Discover / GNOME Software die Pakete findet. - -Aufruf: - appstream-gen.py packages.yaml -o appstream.xml.gz \ - [--available-packages vorhanden.txt] [--origin firmen-repo] -""" - -import argparse -import gzip -import re -import sys -import xml.etree.ElementTree as ET -from xml.dom import minidom - -import yaml - -# AppStream-Spezifikationsversion des Katalogformats. -CATALOG_VERSION = "0.14" - -DEFAULT_TYPE = "console-application" -DEFAULT_ICON = "application-x-executable" -DEFAULT_NOTE = "Über Unternehmens-Repo bereitgestellt." - - -def die(msg): - sys.stdout.flush() - print(f"[appstream-gen] FEHLER: {msg}", file=sys.stderr, flush=True) - sys.exit(1) - - -def warn(msg): - sys.stdout.flush() - print(f"[appstream-gen] WARNUNG: {msg}", file=sys.stderr, flush=True) - - -def info(msg): - print(f"[appstream-gen] {msg}", flush=True) - - -def sanitize_id_part(value): - """Macht aus einem Paketnamen ein gueltiges AppStream-ID-Segment.""" - part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-") - if not part: - die(f"Paketname ergibt kein gueltiges ID-Segment: {value!r}") - if part[0].isdigit(): - part = "pkg-" + part - return part - - -def prettify_name(pkgname): - """Fallback-Anzeigename, wenn in der YAML keiner gesetzt ist. - - dotnet-sdk-9.0 -> Dotnet Sdk 9.0 - """ - words = re.split(r"[-_]", pkgname) - return " ".join(w[:1].upper() + w[1:] if w else w for w in words) - - -def as_list(value): - if value is None: - return [] - if isinstance(value, (list, tuple)): - return list(value) - return [value] - - -def normalize_entry(entry, source_name): - """Akzeptiert sowohl 'paketname' als auch ein Mapping mit Metadaten.""" - if isinstance(entry, str): - return {"name": entry, "_source": source_name} - if isinstance(entry, dict): - if "name" not in entry: - die(f"Paketeintrag in Quelle {source_name!r} ohne 'name': {entry!r}") - data = dict(entry) - data["_source"] = source_name - return data - die(f"Unerwarteter Paketeintrag in Quelle {source_name!r}: {entry!r}") - - -def pick(entry, defaults, key, fallback=None): - """Wert aus dem Paketeintrag, sonst aus defaults, sonst fallback.""" - if key in entry and entry[key] is not None: - return entry[key] - if key in defaults and defaults[key] is not None: - return defaults[key] - return fallback - - -def build_component(entry, defaults, id_prefix): - pkgname = entry["name"] - - comp_type = pick(entry, defaults, "type", DEFAULT_TYPE) - component = ET.Element("component", {"type": comp_type}) - - comp_id = entry.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}" - ET.SubElement(component, "id").text = comp_id - ET.SubElement(component, "pkgname").text = pkgname - - display_name = entry.get("display_name") or entry.get("name_display") - if not display_name: - display_name = prettify_name(pkgname) - ET.SubElement(component, "name").text = display_name - - note = pick(entry, defaults, "note", DEFAULT_NOTE) - summary = entry.get("summary") or note - ET.SubElement(component, "summary").text = summary - - # Beschreibung: eigener Text plus Hinweis auf die Herkunft. - description = ET.SubElement(component, "description") - body = entry.get("description") - if body: - for paragraph in [p.strip() for p in str(body).split("\n\n") if p.strip()]: - ET.SubElement(description, "p").text = paragraph - if note and note != summary: - ET.SubElement(description, "p").text = note - elif not body and note == summary: - ET.SubElement(description, "p").text = note - - icon = pick(entry, defaults, "icon", DEFAULT_ICON) - if icon: - ET.SubElement(component, "icon", {"type": "stock"}).text = icon - - categories = as_list(pick(entry, defaults, "categories")) - if categories: - cat_el = ET.SubElement(component, "categories") - for category in categories: - ET.SubElement(cat_el, "category").text = str(category) - - keywords = as_list(entry.get("keywords")) - if keywords: - kw_el = ET.SubElement(component, "keywords") - for keyword in keywords: - ET.SubElement(kw_el, "keyword").text = str(keyword) - - binaries = as_list(entry.get("binary") or entry.get("binaries")) - if binaries: - provides = ET.SubElement(component, "provides") - for binary in binaries: - ET.SubElement(provides, "binary").text = str(binary) - - homepage = entry.get("homepage") or defaults.get("homepage") - if homepage: - ET.SubElement(component, "url", {"type": "homepage"}).text = str(homepage) - - help_url = entry.get("help_url") or defaults.get("help_url") - if help_url: - ET.SubElement(component, "url", {"type": "help"}).text = str(help_url) - - developer = entry.get("developer") or defaults.get("developer") - if developer: - dev_el = ET.SubElement(component, "developer") - ET.SubElement(dev_el, "name").text = str(developer) - - project_license = entry.get("project_license") or defaults.get("project_license") - if project_license: - ET.SubElement(component, "project_license").text = str(project_license) - - # desktop-application braucht einen Launchable-Eintrag, sonst ignoriert - # Discover die Komponente beim Starten der installierten App. - if comp_type == "desktop-application": - desktop_id = entry.get("desktop_id") - if desktop_id: - ET.SubElement( - component, "launchable", {"type": "desktop-id"} - ).text = str(desktop_id) - else: - warn( - f"{pkgname}: type=desktop-application ohne 'desktop_id'. " - "Discover kann die Anwendung dann nicht starten." - ) - - return comp_id, component - - -def load_available(path): - if not path: - return None - try: - with open(path, "r", encoding="utf-8") as handle: - names = {line.strip() for line in handle if line.strip()} - except OSError as exc: - die(f"Konnte Paketliste {path!r} nicht lesen: {exc}") - info(f"{len(names)} tatsaechlich vorhandene Pakete eingelesen") - return names - - -def main(): - parser = argparse.ArgumentParser( - description="Erzeugt einen AppStream-Katalog aus packages.yaml" - ) - parser.add_argument("packages_yaml", help="Pfad zur packages.yaml") - parser.add_argument( - "-o", "--output", default="appstream.xml.gz", - help="Zieldatei (.xml.gz oder .xml), Standard: appstream.xml.gz", - ) - parser.add_argument( - "--origin", - help="Origin des Katalogs. Ueberschreibt 'appstream.origin' aus der YAML.", - ) - parser.add_argument( - "--available-packages", - help="Datei mit den tatsaechlich gespiegelten Paketnamen (einer pro Zeile). " - "Nicht vorhandene Pakete werden dann uebersprungen.", - ) - parser.add_argument( - "--strict", action="store_true", - help="Mit Fehler abbrechen, wenn ein gelistetes Paket nicht vorhanden ist.", - ) - args = parser.parse_args() - - try: - with open(args.packages_yaml, "r", encoding="utf-8") as handle: - config = yaml.safe_load(handle) or {} - except OSError as exc: - die(f"Konnte {args.packages_yaml!r} nicht lesen: {exc}") - except yaml.YAMLError as exc: - die(f"YAML-Fehler in {args.packages_yaml!r}: {exc}") - - appstream_cfg = config.get("appstream") or {} - defaults = appstream_cfg.get("defaults") or {} - origin = args.origin or appstream_cfg.get("origin") or "firmen-repo" - id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo" - - available = load_available(args.available_packages) - - sources = config.get("sources") or [] - if not sources: - die("Keine 'sources' in der packages.yaml gefunden.") - - root = ET.Element("components", {"version": CATALOG_VERSION, "origin": origin}) - - seen_ids = {} - seen_pkgs = set() - skipped = [] - count = 0 - - for source in sources: - source_name = source.get("name", "") - for raw_entry in source.get("packages") or []: - entry = normalize_entry(raw_entry, source_name) - pkgname = entry["name"] - - if entry.get("appstream") is False or entry.get("hidden") is True: - info(f"{pkgname}: per Konfiguration von AppStream ausgenommen") - continue - - if available is not None and pkgname not in available: - skipped.append(pkgname) - continue - - if pkgname in seen_pkgs: - warn(f"{pkgname}: mehrfach gelistet, spaetere Eintraege ignoriert") - continue - seen_pkgs.add(pkgname) - - comp_id, component = build_component(entry, defaults, id_prefix) - if comp_id in seen_ids: - die( - f"Doppelte AppStream-ID {comp_id!r} " - f"({seen_ids[comp_id]} und {pkgname}). " - "Setze 'id:' fuer eines der beiden Pakete." - ) - seen_ids[comp_id] = pkgname - root.append(component) - count += 1 - - if skipped: - message = ( - f"{len(skipped)} gelistete Pakete sind nicht im Repo vorhanden " - f"und wurden uebersprungen: {', '.join(sorted(skipped))}" - ) - if args.strict: - die(message) - warn(message) - - raw = ET.tostring(root, encoding="utf-8") - pretty = minidom.parseString(raw).toprettyxml(indent=" ", encoding="UTF-8") - - try: - if args.output.endswith(".gz"): - with gzip.open(args.output, "wb") as handle: - handle.write(pretty) - else: - with open(args.output, "wb") as handle: - handle.write(pretty) - except OSError as exc: - die(f"Konnte {args.output!r} nicht schreiben: {exc}") - - info(f"{count} Komponente(n) nach {args.output} geschrieben (origin={origin})") - if count == 0: - warn("Katalog ist leer. Discover wird nichts anzeigen.") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 3768d4b..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Aus DIESEM Verzeichnis heraus starten: -# cd docker && docker compose up -d --build -services: - syncer: - build: - # Kontext ist das docker/-Verzeichnis, in dem auch diese Datei liegt. - context: . - dockerfile: Dockerfile - image: rpm-mirror-syncer:latest - container_name: rpm-mirror-syncer - restart: unless-stopped - env_file: - - .env - volumes: - # repo-data enthaelt ausschliesslich oeffentliche Dateien und wird - # deshalb auch vom nginx-Container gemountet. - - repo-data:/srv/repo - # sync-cache haelt den Git-Checkout der Paketliste und spaeter den - # privaten GPG-Signierschluessel - absichtlich NICHT in nginx. - - sync-cache:/var/cache/rpm-mirror-sync - # Nur noetig, wenn die Paketliste NICHT aus Git kommt - # (SOURCES_GIT_REPO leer lassen). Sonst auskommentiert lassen. - #- ${LOCAL_SOURCES_YAML:-./packages.example.yaml}:/config/sources.yaml:ro - - nginx: - image: nginx:alpine - container_name: rpm-mirror-nginx - restart: unless-stopped - depends_on: - - syncer - ports: - - "${NGINX_PORT:-8081}:80" - volumes: - # Mount-Ziel und der Root in nginx.conf muessen zusammenpassen: - # root /usr/share/nginx/html + Mount auf .../html/repo => URL /repo/ - - repo-data:/usr/share/nginx/html/repo:ro - - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro - -volumes: - repo-data: - sync-cache: \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100644 index 816e59e..0000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Container entrypoint: run sync.sh once, or in a loop every -# SYNC_INTERVAL_SECONDS. There's no systemd/cron inside the container, so a -# simple sleep-loop is the pragmatic choice. -set -euo pipefail - -: "${SYNC_INTERVAL_SECONDS:=3600}" -: "${RUN_ONCE:=false}" - -log() { echo "[entrypoint] $(date '+%Y-%m-%d %H:%M:%S') $*"; } - -run_once() { - /usr/local/bin/sync.sh -} - -if [[ "$RUN_ONCE" == "true" ]]; then - log "RUN_ONCE=true - running sync.sh once and exiting" - run_once - exit $? -fi - -log "Starting sync loop (interval: ${SYNC_INTERVAL_SECONDS}s)" - -# Graceful shutdown on docker stop / compose down -term_handler() { - log "Received SIGTERM, shutting down" - exit 0 -} -trap term_handler SIGTERM SIGINT - -while true; do - # NOTE: don't inline `run_once` into `if ! run_once; then ...` - inside - # that then-branch, $? would reflect the (always-0) negated test itself, - # not run_once's real exit code. Capture it explicitly instead. - rc=0 - run_once || rc=$? - if [[ $rc -ne 0 ]]; then - log "sync.sh failed (exit ${rc}) - will retry after the next interval" - fi - log "Sleeping ${SYNC_INTERVAL_SECONDS}s until next sync..." - sleep "${SYNC_INTERVAL_SECONDS}" & - wait $! -done \ No newline at end of file diff --git a/docker/nginx.conf b/docker/nginx.conf deleted file mode 100644 index 7d5e759..0000000 --- a/docker/nginx.conf +++ /dev/null @@ -1,38 +0,0 @@ -server { - listen 80; - # Declare IPv6 explicitly ourselves - the stock nginx image otherwise - # tries to inject this line into default.conf on startup - # (10-listen-on-ipv6-by-default.sh), which fails with a "read-only file - # system?" message because we mount this file read-only. Harmless, but - # noisy; setting it here avoids the entrypoint script touching the file - # at all. - listen [::]:80; - server_name _; - - # Small landing response at the root so a bare hit doesn't 404. - location = / { - default_type text/plain; - return 200 "RPM mirror - browse /repo/\n"; - } - - # The actual mirrored repos (one subdirectory per source name) and the - # generated client .repo files under /repo/client-repos/. - location /repo/ { - alias /usr/share/nginx/html/repo/; - autoindex on; - autoindex_exact_size off; - autoindex_localtime on; - - # repomd.xml is what dnf checks first to see if a repo changed - - # keep it uncached so clients notice updates immediately. Everything - # else (rpms, *.xml.gz) is content-addressed enough to cache freely. - if ($request_uri ~* /repodata/repomd\.xml$) { - add_header Cache-Control "no-cache"; - } - } - - location /healthz { - default_type text/plain; - return 200 "ok\n"; - } -} \ No newline at end of file diff --git a/docker/sync.sh b/docker/sync.sh deleted file mode 100644 index 3c01bce..0000000 --- a/docker/sync.sh +++ /dev/null @@ -1,523 +0,0 @@ -#!/usr/bin/env bash -# -# sync.sh - Mirror selected packages from third-party RPM repos into ONE -# combined local repo consumable by AlmaLinux clients (dnf / -# PackageKit / GNOME Software / Discover) - WITHOUT pulling in or -# exposing packages that AlmaLinux itself already ships -# (AppStream, BaseOS, Extras, CRB/"Extras for Enterprise"). -# -# ALL sources defined in the YAML end up in a SINGLE repo -# (REPO_BASE_DIR/pkgs/) with a SINGLE client-facing .repo file -# (CLIENT_REPO_DIR/mirror.repo) - clients only ever add one repo, no matter -# how many sources you configure. -# -# What it does, per entry in the YAML source list: -# 1. Registers the source repo temporarily and resolves the full -# dependency closure of the requested packages, using the source repo -# AND the local AlmaLinux repos (needed so resolution succeeds, e.g. -# glibc-style deps that only exist in AlmaLinux). -# 2. Throws away every RPM in that closure that AlmaLinux itself provides. -# Only RPMs that genuinely only exist in the third-party repo are kept. -# 3. Verifies each kept RPM against the source's own gpgkey (if given in -# the YAML), then STRIPS the original signature and RE-SIGNS it with -# the mirror's own GPG key. Clients therefore only ever need to trust -# one key (this mirror's), never the individual upstream repos' keys. -# 4. Moves the signed RPMs into the single shared repo directory -# (REPO_BASE_DIR/pkgs/). -# Once all sources have been processed, the script rebuilds repo metadata -# for that ONE directory with createrepo_c and generates AppStream "generic -# component" metadata for ONLY the packages explicitly listed under -# "packages:" across all sources (never for pulled-in dependencies), -# injecting it once into that same repo. Only those show up as installable -# items in GNOME Software/Discover. -# -# Set "metadata_only: true" on a source to skip steps 2-4 entirely for that -# source: no RPMs are downloaded/mirrored, only curated AppStream visibility -# is contributed to the combined repo. Use this when baseurl already points -# at a repo your clients have enabled directly (e.g. AlmaLinux's own -# AppStream repo, which on AlmaLinux 10 ships dotnet-* itself) - you just -# want to narrow down what shows up in Discover without duplicating RPM -# content clients can already get natively. -# -# NOTE ON NAME COLLISIONS: if two different sources define an RPM with the -# exact same NEVRA (name-version-release.arch), the later one silently wins -# in the shared directory (harmless - it's the same package). If two -# sources define the SAME PACKAGE NAME with genuinely different content, -# that's not supported here - keep package names distinct across sources. -# -# IMPORTANT LIMITATION: -# GNOME Software's curated "Explore" front page only lists packages that -# ship a real desktop-application AppStream component (icon, screenshots, -# .desktop file). A "generic" component (what this script generates for -# CLI tools/SDKs/runtimes) makes the package installable from Discover and -# shows up in search/details, but it will not appear on that curated front -# page. This is GNOME Software/appstreamcli behaviour and cannot be fixed -# by repo metadata alone. -# -# SECURITY NOTE ON RE-SIGNING: -# The mirror's private signing key is generated once (unattended, without -# a passphrase - standard practice for automated repo signing) and kept -# under GPG_HOME. Only the exported PUBLIC key ends up in REPO_BASE_DIR -# where clients can fetch it. GPG_HOME itself must never be exposed via -# nginx/the webserver and should live on storage only the sync process can -# read. Anyone who can read GPG_HOME can sign packages as your mirror - -# treat it like any other private key material. -# -# A revocation certificate is generated alongside the key on first run -# (GPG_HOME/revocation-cert.asc) - since the key never expires, this -# certificate is the ONLY way to invalidate it later if it's ever -# compromised. Copy it to secure, offline storage right after first -# startup; see the log output / README for the exact command. -# -# Requires: dnf, dnf-plugins-core (for "dnf download"), createrepo_c, -# modifyrepo_c, rpm-sign (rpmsign/rpmkeys), gnupg2, python3, -# python3-pyyaml, git (if fetching the source list from git), curl. -# -# Run as root (needs to write a temporary .repo file to /etc/yum.repos.d). - -set -euo pipefail - -# --------------------------------------------------------------------------- -# Configuration (override via environment, e.g. in the systemd unit/cron job) -# --------------------------------------------------------------------------- - -# Where the YAML source list lives. Either point SOURCES_GIT_REPO at a git -# repo (SOURCES_YAML_FILE is the path *inside* that repo), or leave -# SOURCES_GIT_REPO empty and point SOURCES_YAML_FILE at a local file. -SOURCES_GIT_REPO="${SOURCES_GIT_REPO:-}" -SOURCES_GIT_BRANCH="${SOURCES_GIT_BRANCH:-main}" -SOURCES_YAML_FILE="${SOURCES_YAML_FILE:-sources.yaml}" - -# Local working/output paths. -WORK_DIR="${WORK_DIR:-/var/cache/rpm-mirror-sync}" -REPO_BASE_DIR="${REPO_BASE_DIR:-/srv/repo}" -CLIENT_REPO_DIR="${CLIENT_REPO_DIR:-${REPO_BASE_DIR}/client-repos}" -# The ONE combined repo directory all sources' packages end up in. -PKG_DIR="${REPO_BASE_DIR}/pkgs" - -# Base URL under which REPO_BASE_DIR is actually served to clients (web -# server / reverse proxy in front of REPO_BASE_DIR). Used only to generate -# ready-to-use .repo files for clients. -CLIENT_BASE_URL="${CLIENT_BASE_URL:-http://mirror.example.local/repo}" - -# repoids of AlmaLinux's own repos on THIS machine, as shown by -# `dnf repolist`. Adjust to match your system if they differ. -# ("Extras for Enterprise" is the "extras" repo in AlmaLinux 10 naming; crb -# is CodeReady Builder / "extras-common" on some setups - check your -# `dnf repolist` output and adjust below.) -ALMA_REPO_IDS="${ALMA_REPO_IDS:-baseos appstream extras crb}" - -# Origin of the generated AppStream catalog. Must be unique - if it collides -# with a distribution's own origin (e.g. "almalinux"), the icon/metadata -# caches on the clients overwrite each other. Can also be set via -# "appstream: origin:" in the YAML; this variable wins. -APPSTREAM_ORIGIN="${APPSTREAM_ORIGIN:-rpm-mirror}" - -# The mirror's own signing identity. A key is generated once (on first run) -# under GPG_HOME and reused on every subsequent run - make sure GPG_HOME -# points at persistent storage (a volume), or you'll get a new key (and -# therefore a trust-breaking change for clients) on every restart. -GPG_HOME="${GPG_HOME:-${WORK_DIR}/gnupg}" -GPG_KEY_NAME="${GPG_KEY_NAME:-RPM Mirror}" -GPG_KEY_EMAIL="${GPG_KEY_EMAIL:-rpm-mirror@example.local}" -MIRROR_GPG_KEY_FILE="${REPO_BASE_DIR}/RPM-GPG-KEY-mirror" - -LOG_TAG="rpm-mirror-sync" - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${WORK_DIR}/sync.log" >&2; } -die() { log "ERROR: $*"; exit 1; } - -require_cmd() { - command -v "$1" >/dev/null 2>&1 || die "required command '$1' not found (install it first)" -} - -cleanup() { - # Remove any temp source .repo files we created, even on failure. - if [[ -n "${TEMP_REPO_FILES:-}" ]]; then - for f in "${TEMP_REPO_FILES[@]}"; do - [[ -n "$f" ]] && rm -f "$f" - done - fi -} - -# Remove one entry from TEMP_REPO_FILES. -# NOTE: "${ARRAY[@]/pattern}" does SUBSTRING REPLACEMENT, so the element is -# left behind as an empty string rather than removed - which then makes -# cleanup() call `rm -f ""`. Rebuild the array instead. -forget_repo_file() { - local drop="$1" keep=() f - for f in "${TEMP_REPO_FILES[@]}"; do - [[ "$f" == "$drop" ]] || keep+=( "$f" ) - done - TEMP_REPO_FILES=( "${keep[@]}" ) -} -trap cleanup EXIT - -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -HELPER="${SCRIPT_DIR}/sync_helpers.py" - -# --------------------------------------------------------------------------- -# Preconditions -# --------------------------------------------------------------------------- - -[[ $EUID -eq 0 ]] || die "must run as root (writes to /etc/yum.repos.d and ${REPO_BASE_DIR})" - -for c in dnf createrepo_c modifyrepo_c rpm rpmsign rpmkeys gpg python3 curl gzip; do - require_cmd "$c" -done -[[ -f "$HELPER" ]] || die "sync_helpers.py not found next to sync.sh (expected: $HELPER)" -python3 -c "import yaml" 2>/dev/null || die "python3-pyyaml not installed (dnf install -y python3-pyyaml)" - -mkdir -p "$WORK_DIR" "$REPO_BASE_DIR" "$CLIENT_REPO_DIR" "$PKG_DIR" - -# --------------------------------------------------------------------------- -# Mirror signing key: generate once, reuse forever, export the public part -# --------------------------------------------------------------------------- - -MIRROR_GPG_KEY_ID="" -REVOCATION_CERT_FILE="${GPG_HOME}/revocation-cert.asc" - -init_mirror_gpg_key() { - mkdir -p "$GPG_HOME" - chmod 700 "$GPG_HOME" - - if [[ -z "$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons 2>/dev/null)" ]]; then - log "No mirror signing key found under ${GPG_HOME} - generating one (one-time)..." - gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \ - --quick-generate-key "${GPG_KEY_NAME} <${GPG_KEY_EMAIL}>" rsa4096 sign never \ - || die "failed to generate mirror GPG signing key" - - # GnuPG >= 2.1 automatically writes a revocation certificate under - # openpgp-revocs.d/.rev as part of key generation itself - # (visible in the gpg output above: "revocation certificate stored - # as ..."). We just need to copy it to a known, stable path - no - # need to (fragile-ly) drive gpg's interactive --gen-revoke menu - # ourselves, which behaves differently across gpg versions. - local auto_revoc - auto_revoc="$(find "${GPG_HOME}/openpgp-revocs.d" -maxdepth 1 -name '*.rev' 2>/dev/null | head -n1)" - [[ -n "$auto_revoc" && -s "$auto_revoc" ]] \ - || die "expected GnuPG to auto-generate a revocation certificate under ${GPG_HOME}/openpgp-revocs.d/ but none was found" - - cp "$auto_revoc" "$REVOCATION_CERT_FILE" - chmod 600 "$REVOCATION_CERT_FILE" - - log "!!! Revocation certificate written to: ${REVOCATION_CERT_FILE}" - log "!!! Copy it to secure, OFFLINE storage now, e.g.:" - log "!!! docker compose cp syncer:${REVOCATION_CERT_FILE} ./mirror-key-revocation-cert.asc" - log "!!! It is the only way to invalidate this key if it is ever compromised -" - log "!!! it never expires, so it needs the same protection as the private key." - fi - - MIRROR_GPG_KEY_ID="$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons \ - | awk -F: '/^sec:/ {print $5; exit}')" - [[ -n "$MIRROR_GPG_KEY_ID" ]] || die "could not determine mirror GPG key id" - - # rpmsign/rpm --addsign etc. read the signing identity from macros. - cat > "${HOME:-/root}/.rpmmacros" < "$MIRROR_GPG_KEY_FILE" - - log "Mirror signing key ready: ${MIRROR_GPG_KEY_ID} (public key: ${MIRROR_GPG_KEY_FILE})" -} - -init_mirror_gpg_key - - - -# --------------------------------------------------------------------------- -# Fetch the YAML source list -# --------------------------------------------------------------------------- - -resolve_yaml_path() { - if [[ -n "$SOURCES_GIT_REPO" ]]; then - if [[ "$SOURCES_YAML_FILE" == /* ]]; then - die "SOURCES_GIT_REPO is set, but SOURCES_YAML_FILE ('${SOURCES_YAML_FILE}') looks like an absolute path. When pulling the source list from git, SOURCES_YAML_FILE must be a path RELATIVE to that repo's root (e.g. 'sources.yaml' or 'config/sources.yaml') - not the '/config/...' path used for the local bind-mount option. Fix SOURCES_YAML_FILE in .env." - fi - local clone_dir="${WORK_DIR}/sources-repo" - if [[ -d "${clone_dir}/.git" ]]; then - log "Updating source list git repo..." - git -C "$clone_dir" fetch --depth 1 origin "$SOURCES_GIT_BRANCH" - git -C "$clone_dir" reset --hard "origin/${SOURCES_GIT_BRANCH}" - else - log "Cloning source list git repo..." - rm -rf "$clone_dir" - git clone --depth 1 --branch "$SOURCES_GIT_BRANCH" "$SOURCES_GIT_REPO" "$clone_dir" - fi - echo "${clone_dir}/${SOURCES_YAML_FILE}" - else - echo "$SOURCES_YAML_FILE" - fi -} - -YAML_PATH="$(resolve_yaml_path)" -[[ -f "$YAML_PATH" ]] || die "source YAML not found: $YAML_PATH" - -SOURCES_JSON="$(python3 "$HELPER" parse-yaml "$YAML_PATH")" || die "failed to parse $YAML_PATH" -SOURCE_COUNT="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$SOURCES_JSON")" -log "Loaded $SOURCE_COUNT source(s) from $YAML_PATH" - -# --------------------------------------------------------------------------- -# Per-source sync -# --------------------------------------------------------------------------- - -declare -a TEMP_REPO_FILES=() - -# Build --enablerepo flags for the AlmaLinux repos as a bash array once. -declare -a ALMA_ENABLE_FLAGS=() -for r in $ALMA_REPO_IDS; do - ALMA_ENABLE_FLAGS+=( "--enablerepo=${r}" ) -done - -sync_one_source() { - local name="$1" baseurl="$2" gpgkey="$3" packages_json="$4" metadata_only="${5:-false}" - - local repoid="mirror-src-${name}" - local repo_file="/etc/yum.repos.d/${repoid}.repo" - local tmp_dir="${WORK_DIR}/${name}/download" - local list_dir="${WORK_DIR}/${name}" - # This source's contribution to the combined AppStream metadata: one - # "namesummary" line per explicitly requested package. Collected - # here, merged across all sources, and turned into ONE appstream.xml - # after the whole source loop finishes. - local entries_tsv="${list_dir}/appstream-entries.tsv" - - log "=== Syncing source '${name}' (metadata_only=${metadata_only}) ===" - mkdir -p "$tmp_dir" "$list_dir" - : > "$entries_tsv" - - # readarray of requested top-level package names for this source. - # parse-yaml normalises every entry to an object, so a package written - # as a plain string and one written as a mapping with AppStream metadata - # both arrive here as {"name": ...}. - mapfile -t packages < <(python3 -c ' -import json,sys -for p in json.load(sys.stdin): - print(p["name"]) -' <<<"$packages_json") - [[ ${#packages[@]} -gt 0 ]] || die "source '${name}' has no packages" - - # --- 1. Register the source repo temporarily ------------------------- - { - echo "[${repoid}]" - echo "name=Mirror source: ${name}" - echo "baseurl=${baseurl}" - echo "enabled=1" - if [[ -n "$gpgkey" ]]; then - echo "gpgcheck=1" - echo "gpgkey=${gpgkey}" - else - echo "gpgcheck=0" - fi - } > "$repo_file" - TEMP_REPO_FILES+=( "$repo_file" ) - dnf clean expire-cache --disablerepo='*' --enablerepo="$repoid" >/dev/null 2>&1 || true - - # --- metadata_only sources: curate Discover/GNOME Software visibility - # WITHOUT mirroring any RPM content. Use this when baseurl already - # points at a repo your clients have enabled directly (e.g. AlmaLinux's - # own AppStream repo, which ships dotnet-* itself on AlmaLinux 10) - the - # actual package install is then resolved by dnf from that repo as - # normal; this source only contributes AppStream "generic component" - # data for the packages listed to the combined repo built after the - # source loop, so ONLY those show up as installable in Discover instead - # of AlmaLinux's entire (huge) AppStream catalog. - # PRECONDITION: clients must have the repo(s) that actually provide - # these packages enabled already (true for any stock AlmaLinux install - - # baseos/appstream/extras/crb are on by default). - if [[ "$metadata_only" == "true" ]]; then - for pkg in "${packages[@]}"; do - local summary - summary="$(dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \ - --qf '%{summary}' "$pkg" 2>/dev/null | head -n1)" - printf '%s\t%s\n' "$pkg" "$summary" >> "$entries_tsv" - done - - log "Source '${name}': metadata_only - queued AppStream visibility for ${#packages[@]} package(s), no RPMs mirrored" - - rm -f "$repo_file" - forget_repo_file "$repo_file" - return - fi - - # --- 2. List everything the source repo itself provides -------------- - # (used later to decide "does this RPM genuinely come from the source, - # or did AlmaLinux already have it".) - local source_nevra_list="${list_dir}/source-nevra.txt" - dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \ - --qf '%{name}-%{version}-%{release}.%{arch}' -a \ - > "$source_nevra_list" \ - || die "failed to query source repo '${name}' (check baseurl/gpgkey)" - [[ -s "$source_nevra_list" ]] || die "source repo '${name}' returned no packages - check baseurl" - - # --- 3. Resolve full dependency closure for the requested packages --- - # Enable the source repo AND the local AlmaLinux repos so dependency - # resolution succeeds even for deps AlmaLinux normally provides - - # we filter those back out in step 4. - rm -rf "$tmp_dir"; mkdir -p "$tmp_dir" - log "Resolving + downloading dependency closure for: ${packages[*]}" - dnf download -y --resolve --alldeps \ - --destdir="$tmp_dir" \ - --disablerepo='*' \ - --enablerepo="$repoid" \ - "${ALMA_ENABLE_FLAGS[@]}" \ - "${packages[@]}" \ - || die "dnf download failed for source '${name}'" - - # --- 4. Drop everything AlmaLinux already provides, verify + re-sign -- - # the rest with the mirror's own key ----------------------------------- - local verify_dbpath="${list_dir}/verify-rpmdb" - if [[ -n "$gpgkey" ]]; then - local source_gpgkey_file="${list_dir}/source-gpgkey.asc" - curl -fsSL "$gpgkey" -o "$source_gpgkey_file" \ - || die "could not fetch gpgkey for source '${name}' from ${gpgkey}" - rm -rf "$verify_dbpath"; mkdir -p "$verify_dbpath" - rpm --dbpath "$verify_dbpath" --initdb - rpm --dbpath "$verify_dbpath" --import "$source_gpgkey_file" \ - || die "could not import gpgkey for source '${name}'" - else - log " WARNING: source '${name}' has no gpgkey configured - skipping upstream signature verification" - fi - - local kept=0 dropped=0 kept_requested=0 kept_dependency=0 - for f in "$tmp_dir"/*.rpm; do - [[ -e "$f" ]] || continue - local base; base="$(basename "$f" .rpm)" - - if ! grep -qxF "$base" "$source_nevra_list"; then - log " dropping (provided by AlmaLinux): $base" - rm -f "$f" - dropped=$((dropped + 1)) - continue - fi - - if [[ -n "$gpgkey" ]]; then - rpmkeys --dbpath "$verify_dbpath" --checksig "$f" >/dev/null 2>&1 \ - || die "signature verification FAILED for '${base}' from source '${name}' - refusing to mirror it (tampered download or wrong gpgkey configured)" - fi - - # Strip the upstream signature and sign with the mirror's own key, - # so clients only ever need to trust MIRROR_GPG_KEY_FILE. - rpmsign --resign "$f" >/dev/null \ - || die "failed to re-sign '${base}' with the mirror key" - - # Label each kept package as explicitly "requested" (listed under - # packages: in the YAML) or a pulled-in "dependency" - packages the - # requested ones need but AlmaLinux doesn't provide, so they have to - # be mirrored too or `dnf install ` breaks on clients. - # This is purely informational (log output); both kinds are kept. - local pkg_name tag - pkg_name="$(rpm -qp --qf '%{NAME}' "$f" 2>/dev/null)" - tag="dependency" - for req in "${packages[@]}"; do - if [[ "$pkg_name" == "$req" ]]; then - tag="requested" - break - fi - done - if [[ "$tag" == "requested" ]]; then - kept_requested=$((kept_requested + 1)) - local summary - summary="$(rpm -qp --qf '%{SUMMARY}' "$f" 2>/dev/null)" - printf '%s\t%s\n' "$pkg_name" "$summary" >> "$entries_tsv" - else - kept_dependency=$((kept_dependency + 1)) - fi - log " keeping (${tag}): $base" - - mv "$f" "${PKG_DIR}/" - kept=$((kept + 1)) - done - log "Source '${name}': kept ${kept} package(s) - ${kept_requested} explicitly requested + ${kept_dependency} pulled-in third-party dependencies (re-signed with mirror key), dropped ${dropped} AlmaLinux-provided package(s)" - [[ $kept -gt 0 ]] || die "nothing kept for source '${name}' - check baseurl/package names" - - # --- cleanup temp repo file for this source --------------------------- - rm -f "$repo_file" - forget_repo_file "$repo_file" -} - -# Iterate over sources (JSON array -> one JSON object per line via jq-less python) -while IFS= read -r src_json; do - name="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["name"])' "$src_json")" - baseurl="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["baseurl"])' "$src_json")" - gpgkey="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1]).get("gpgkey",""))' "$src_json")" - pkgs_json="$(python3 -c 'import json,sys;print(json.dumps(json.loads(sys.argv[1])["packages"]))' "$src_json")" - metadata_only="$(python3 -c 'import json,sys;print(str(bool(json.loads(sys.argv[1]).get("metadata_only", False))).lower())' "$src_json")" - sync_one_source "$name" "$baseurl" "$gpgkey" "$pkgs_json" "$metadata_only" -done < <(python3 -c ' -import json,sys -for s in json.load(sys.stdin): - print(json.dumps(s)) -' <<<"$SOURCES_JSON") - -# --------------------------------------------------------------------------- -# Build the ONE combined repo from everything all sources contributed -# --------------------------------------------------------------------------- - -createrepo_c --update "$PKG_DIR" >/dev/null - -# Build ONE AppStream catalog for the single shared repo. The helper reads -# every source's appstream-entries.tsv (written during its own processing -# above) to know WHICH packages made it into the mirror, and packages.yaml -# to know HOW to present them (display name, description, categories, -# keywords). YAML metadata wins; the upstream RPM summary from the TSV is -# the fallback. -APPSTREAM_XML="${WORK_DIR}/appstream.xml" -python3 "$HELPER" gen-appstream \ - --origin "$APPSTREAM_ORIGIN" \ - --yaml "$YAML_PATH" \ - --entries-dir "$WORK_DIR" \ - > "$APPSTREAM_XML" \ - || die "failed to generate AppStream catalog" - -# Purely informational: appstreamcli's checks target the style expectations -# for public app stores (content ratings, screenshots, ...). They do not -# affect whether Discover shows the packages. -if command -v appstreamcli >/dev/null 2>&1; then - appstreamcli validate --no-net "$APPSTREAM_XML" >/dev/null 2>&1 \ - || log "note: AppStream catalog has style hints (harmless) - run 'appstreamcli validate ${APPSTREAM_XML}' to see them" -fi - -gzip -fk "$APPSTREAM_XML" -modifyrepo_c --mdtype=appstream "${APPSTREAM_XML}.gz" "${PKG_DIR}/repodata" >/dev/null - -# Detached-sign repomd.xml itself with the mirror key, so clients can verify -# the METADATA (which packages/versions exist, their checksums) hasn't been -# tampered with - not just each individual RPM. Must happen last, after -# createrepo_c/modifyrepo_c are done touching repomd.xml (they update its -# checksums), otherwise the signature would be over stale content. -gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \ - --detach-sign --armor \ - --output "${PKG_DIR}/repodata/repomd.xml.asc" \ - "${PKG_DIR}/repodata/repomd.xml" \ - || die "failed to sign repomd.xml with the mirror key" - -# --- The ONE client-facing .repo file - this is all clients ever need ----- -cat > "${CLIENT_REPO_DIR}/mirror.repo" < - Reads the "sources:" list from the YAML file, NORMALISES it and prints - it as JSON on stdout, so sync.sh can loop over it safely. - - Normalisation: every entry under "packages:" becomes an object with at - least a "name" key, whether it was written as a plain string or as a - mapping with extra AppStream metadata. sync.sh can therefore always do - p["name"] without caring which form the YAML used. - - gen-appstream --origin O --yaml F --entries-dir D - Builds the AppStream catalog for the combined repo. - - Metadata precedence, highest first: - 1. what packages.yaml says about the package - 2. the upstream RPM summary collected by sync.sh into the - appstream-entries.tsv files under --entries-dir - 3. a generated fallback - - Only packages that actually made it into the mirror (i.e. that appear - in one of the TSV files) get a component - never pulled-in - dependencies, and never packages whose download failed. - - NOTE ON DISCOVER/GNOME SOFTWARE: - Components without an icon/screenshots/.desktop file make a package - installable and findable via SEARCH, but they will not appear on the - curated "Explore" front page or when browsing categories. That is a - frontend behaviour for non-desktop-app components and cannot be fixed - at the repo-metadata level. Only a real desktop-application component - (type: desktop-application + desktop_id: in packages.yaml, which - requires the package to ship a .desktop file) shows up there. -""" -import argparse -import glob -import json -import os -import re -import sys -from xml.sax.saxutils import escape, quoteattr - -CATALOG_VERSION = "0.14" -DEFAULT_NOTE = ( - "Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. " - "Fragen und Paketwünsche bitte an den IT-Service-Desk." -) - - -def die(msg): - sys.stdout.flush() - sys.stderr.write(f"ERROR: {msg}\n") - sys.exit(1) - - -def warn(msg): - sys.stdout.flush() - sys.stderr.write(f"WARNING: {msg}\n") - - -def load_yaml(path): - try: - import yaml - except ImportError: - die( - "python3-pyyaml (PyYAML) is not installed.\n" - "Install it with: dnf install -y python3-pyyaml" - ) - try: - with open(path, "r", encoding="utf-8") as fh: - return yaml.safe_load(fh) or {} - except OSError as exc: - die(f"could not read {path!r}: {exc}") - except Exception as exc: # yaml.YAMLError and friends - die(f"could not parse {path!r}: {exc}") - - -def normalise_package(entry, source_name, index): - """Accept both 'pkgname' and {name: pkgname, display_name: ...}.""" - if isinstance(entry, str): - return {"name": entry} - if isinstance(entry, dict): - if not entry.get("name"): - die( - f"package #{index} in source {source_name!r} has no 'name' key: " - f"{entry!r}" - ) - return dict(entry) - die( - f"package #{index} in source {source_name!r} must be a string or a " - f"mapping, got {type(entry).__name__}" - ) - - -def normalise_sources(data): - sources = data.get("sources") or [] - if not isinstance(sources, list): - die("'sources' in YAML is not a list") - - for i, src in enumerate(sources): - if not isinstance(src, dict): - die(f"source #{i} is not a mapping") - for field in ("name", "baseurl", "packages"): - if field not in src: - die(f"source #{i} is missing required field '{field}'") - if not isinstance(src["packages"], list) or not src["packages"]: - die( - f"source '{src.get('name')}' has an empty/invalid packages list" - ) - src["packages"] = [ - normalise_package(p, src.get("name"), j) - for j, p in enumerate(src["packages"]) - ] - return sources - - -def cmd_parse_yaml(path): - data = load_yaml(path) - json.dump(normalise_sources(data), sys.stdout) - - -# --------------------------------------------------------------------------- -# AppStream catalog generation -# --------------------------------------------------------------------------- - -def sanitize_id_part(value): - part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-") - if not part: - die(f"package name yields no usable AppStream id segment: {value!r}") - if part[0].isdigit(): - part = "pkg-" + part - return part - - -def as_list(value): - if value is None: - return [] - if isinstance(value, (list, tuple)): - return list(value) - return [value] - - -def pick(meta, defaults, key, fallback=None): - if meta.get(key) is not None: - return meta[key] - if defaults.get(key) is not None: - return defaults[key] - return fallback - - -def read_entries(entries_dir): - """Merge every source's appstream-entries.tsv into {name: summary}.""" - entries = {} - pattern = os.path.join(entries_dir, "*", "appstream-entries.tsv") - for path in sorted(glob.glob(pattern)): - try: - with open(path, encoding="utf-8") as fh: - for line in fh: - line = line.rstrip("\n") - if not line: - continue - parts = line.split("\t", 1) - name = parts[0].strip() - summary = parts[1].strip() if len(parts) > 1 else "" - if not name: - continue - if name in entries and entries[name] and summary \ - and entries[name] != summary: - warn( - f"package '{name}' was contributed by more than one " - "source with differing summaries - keeping the first" - ) - continue - entries.setdefault(name, summary) - except OSError: - continue - return entries - - -def collect_metadata(yaml_path): - """Return (defaults, {pkgname: metadata}, id_prefix, yaml_origin).""" - data = load_yaml(yaml_path) - appstream_cfg = data.get("appstream") or {} - defaults = dict(appstream_cfg.get("defaults") or {}) - id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo" - yaml_origin = appstream_cfg.get("origin") - - if not defaults.get("developer_id"): - segments = id_prefix.split(".") - if len(segments) >= 2: - defaults["developer_id"] = ".".join(segments[:2]) - - by_name = {} - for src in normalise_sources(data): - for pkg in src["packages"]: - name = pkg["name"] - if name in by_name: - warn(f"package '{name}' listed more than once - keeping the first") - continue - by_name[name] = pkg - return defaults, by_name, id_prefix, yaml_origin - - -def emit(out, depth, text): - out.append(" " * depth + text) - - -def build_component(pkgname, meta, upstream_summary, defaults, id_prefix, out): - if meta.get("appstream") is False or meta.get("hidden") is True: - return False - - binaries = as_list(meta.get("binary") or meta.get("binaries")) - - # A console-application component without trips - # appstreamcli's 'console-app-no-binary' check, so only claim that type - # when a binary is actually declared. Runtimes and libraries stay - # 'generic', which Discover still finds via search. - comp_type = pick(meta, defaults, "type") - if not comp_type: - comp_type = "console-application" if binaries else "generic" - - comp_id = meta.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}" - - display_name = meta.get("display_name") or pkgname - - note = pick(meta, defaults, "note", DEFAULT_NOTE) - summary = meta.get("summary") or upstream_summary or note - - emit(out, 1, f"") - emit(out, 2, f"{escape(comp_id)}") - emit(out, 2, f"{escape(pkgname)}") - emit(out, 2, f"{escape(display_name)}") - emit(out, 2, f"{escape(summary)}") - - emit(out, 2, "") - body = meta.get("description") - wrote_paragraph = False - if body: - for para in [p.strip() for p in str(body).split("\n\n") if p.strip()]: - emit(out, 3, f"

{escape(para)}

") - wrote_paragraph = True - if note and (not wrote_paragraph or note != summary): - emit(out, 3, f"

{escape(note)}

") - elif not wrote_paragraph: - emit(out, 3, f"

{escape(summary)}

") - emit(out, 2, "") - - icon = pick(meta, defaults, "icon", "application-x-executable") - if icon: - emit(out, 2, f'{escape(str(icon))}') - - categories = as_list(pick(meta, defaults, "categories")) - if categories: - emit(out, 2, "") - for cat in categories: - emit(out, 3, f"{escape(str(cat))}") - emit(out, 2, "") - - keywords = as_list(meta.get("keywords")) - if keywords: - emit(out, 2, "") - for kw in keywords: - emit(out, 3, f"{escape(str(kw))}") - emit(out, 2, "") - - if binaries: - emit(out, 2, "") - for binary in binaries: - emit(out, 3, f"{escape(str(binary))}") - emit(out, 2, "") - - homepage = meta.get("homepage") or defaults.get("homepage") - if homepage: - emit(out, 2, f'{escape(str(homepage))}') - help_url = meta.get("help_url") or defaults.get("help_url") - if help_url: - emit(out, 2, f'{escape(str(help_url))}') - - developer = meta.get("developer") or defaults.get("developer") - if developer: - dev_id = meta.get("developer_id") or defaults.get("developer_id") - attr = f" id={quoteattr(str(dev_id))}" if dev_id else "" - emit(out, 2, f"") - emit(out, 3, f"{escape(str(developer))}") - emit(out, 2, "") - - project_license = meta.get("project_license") or defaults.get("project_license") - if project_license: - emit(out, 2, f"{escape(str(project_license))}") - - # A desktop-application component that Discover cannot launch is worse - # than a generic one, so insist on the desktop file id. - if comp_type == "desktop-application": - desktop_id = meta.get("desktop_id") - if desktop_id: - emit( - out, 2, - f'{escape(str(desktop_id))}', - ) - else: - warn( - f"{pkgname}: type=desktop-application without 'desktop_id' - " - "Discover will not be able to launch it after installation" - ) - - emit(out, 1, "") - return True - - -def cmd_gen_appstream(origin, yaml_path, entries_dir): - defaults, by_name, id_prefix, yaml_origin = collect_metadata(yaml_path) - entries = read_entries(entries_dir) - - if not entries: - warn( - f"no appstream-entries.tsv found under {entries_dir} - the catalog " - "will be empty and nothing will show up in Discover" - ) - - effective_origin = origin or yaml_origin or "rpm-mirror" - - out = [] - out.append('') - out.append( - f'' - ) - - written = 0 - for pkgname in sorted(entries): - meta = by_name.get(pkgname, {}) - if build_component(pkgname, meta, entries[pkgname], defaults, - id_prefix, out): - written += 1 - - out.append("") - sys.stdout.write("\n".join(out) + "\n") - sys.stderr.write( - f"gen-appstream: wrote {written} component(s), origin={effective_origin}\n" - ) - - -def main(): - ap = argparse.ArgumentParser() - sub = ap.add_subparsers(dest="cmd", required=True) - - p1 = sub.add_parser("parse-yaml") - p1.add_argument("path") - - p2 = sub.add_parser("gen-appstream") - p2.add_argument("--origin", default="") - p2.add_argument("--yaml", required=True) - p2.add_argument("--entries-dir", required=True) - - args = ap.parse_args() - - if args.cmd == "parse-yaml": - cmd_parse_yaml(args.path) - elif args.cmd == "gen-appstream": - cmd_gen_appstream(args.origin, args.yaml, args.entries_dir) - - -if __name__ == "__main__": - main() \ No newline at end of file