leander19961/dnf-repo-test
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Verzeichnis 'docker ' löschen

Browse Source
This commit is contained in:
leander19961
2026-08-26 15:17:02 +02:00
parent c5d1c2d3f9
commit ba7974d1cb
8 changed files with 0 additions and 1449 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/.env-example
-80
View File
@@ -1,80 +0,0 @@
# Copy this file to .env. docker-compose.yml loads it via env_file for both
# containers, and docker compose itself also uses it to resolve ${VAR}
# references inside docker-compose.yml (e.g. NGINX_PORT, LOCAL_SOURCES_YAML).
# This file is the single place to configure everything - nothing is
# hardcoded in docker-compose.yml itself.
# ============================================================================
# Where the sources.yaml comes from
# ============================================================================
# Option A (default): local file, bind-mounted into the syncer container.
# Point LOCAL_SOURCES_YAML at your real file (path is relative to
# docker-compose.yml, defaults to the bundled sources.example.yaml) and
# leave SOURCES_GIT_REPO empty. SOURCES_YAML_FILE must then be the absolute
# in-container path the bind mount uses (leave as-is unless you also change
# the mount target in docker-compose.yml).
#LOCAL_SOURCES_YAML=./sources.example.yaml
CONFIG_REPO_URL=https://gitea.leanderserver.de/leander19961/dnf-repo-test
CONFIG_REPO_BRANCH=main
PACKAGES_FILE=packages.yaml
# Option B: pull the list from git instead. Set SOURCES_GIT_REPO, and
# SOURCES_YAML_FILE to the path of the YAML file *inside* that git repo
# (relative), e.g.:
# SOURCES_GIT_REPO=https://git.example.com/infra/rpm-sources.git
# SOURCES_GIT_BRANCH=main
# SOURCES_YAML_FILE=sources.yaml
# (LOCAL_SOURCES_YAML / the bind mount is then simply unused.)
# ============================================================================
# Paths inside the syncer container
# ============================================================================
# These map to the volumes defined in docker-compose.yml - only change them
# if you also change the corresponding volume mounts.
WORK_DIR=/var/cache/rpm-mirror-sync
REPO_BASE_DIR=/srv/repo
CLIENT_REPO_DIR=/srv/repo/client-repos
# ============================================================================
# Client-facing URL
# ============================================================================
# Public URL under which the nginx container serves the repo. Used to
# generate the .repo files under /repo/client-repos/. Point this at
# wherever the host running nginx is reachable from your AlmaLinux clients
# (should match NGINX_PORT below).
CLIENT_BASE_URL=http://repo.leanderserver.de/repo
# ============================================================================
# repoids of AlmaLinux's own repos inside the syncer container
# ============================================================================
# Check with: docker compose exec syncer dnf repolist
ALMA_REPO_IDS=baseos appstream extras crb
# ============================================================================
# Sync schedule
# ============================================================================
# How often (seconds) the syncer container re-syncs. 3600 = hourly.
SYNC_INTERVAL_SECONDS=3600
# Set to "true" to sync once and exit instead of looping (e.g. if you'd
# rather trigger this container from an external scheduler).
RUN_ONCE=false
# ============================================================================
# Mirror signing key
# ============================================================================
# Identity used for the ONE GPG key the mirror signs every mirrored package
# with (regardless of how many upstream sources there are). Generated once
# on first run - together with a revocation certificate - and reused
# forever. Both live under GPG_HOME on the sync-cache volume, so don't run
# "docker compose down -v" unless you're fine with clients having to
# re-trust a brand new key afterwards.
GPG_HOME=/var/cache/rpm-mirror-sync/gnupg
GPG_KEY_NAME=RPM Mirror
GPG_KEY_EMAIL=rpm-mirror@example.local
# ============================================================================
# Ports
# ============================================================================
# Host port nginx publishes the repo on.
NGINX_PORT=8081
docker/Dockerfile
-55
View File
@@ -1,55 +0,0 @@
FROM almalinux:10
# Ohne UTF-8-Locale verstuemmeln appstreamcli und andere Tools Umlaute in
# ihrer Ausgabe ("?ber" statt "über").
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8
# createrepo_c bringt auch modifyrepo_c mit.
# python3 + PyYAML werden von appstream-gen.py gebraucht - nicht darauf
# verlassen, dass das Basisimage sie zufaellig mitbringt.
# gnupg2/rpm-sign sind fuer die spaetere Signierung vorgesehen.
RUN dnf install -y \
dnf-plugins-core \
createrepo_c \
appstream \
git \
jq \
openssh-clients \
python3 \
python3-pyyaml \
gnupg2 \
rpm-sign \
findutils \
util-linux-core \
curl \
&& dnf clean all
# yq (Go-Variante) fuer das YAML-Parsing in sync.sh.
# Version gepinnt: "latest" macht Builds nicht reproduzierbar und kann bei
# einem Breaking Change der yq-Syntax den Sync stillschweigend zerlegen.
ARG YQ_VERSION=v4.44.3
RUN curl -fsSL -o /usr/local/bin/yq \
"https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" \
&& chmod +x /usr/local/bin/yq
COPY sync.sh /usr/local/bin/sync.sh
COPY appstream-gen.py /usr/local/bin/appstream-gen.py
COPY sync_helpers.py /usr/local/bin/sync_helpers.py
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/sync.sh \
/usr/local/bin/appstream-gen.py \
/usr/local/bin/entrypoint.sh
# Bewusst KEIN "VOLUME /srv/repo":
# Ein VOLUME im Dockerfile legt beim Start ein anonymes Volume an. Wird in
# docker-compose.yml ein benanntes Volume auf einen ANDEREN Pfad gemountet,
# schreibt der Sync ins anonyme Volume, waehrend nginx das benannte (leere)
# ausliefert. Genau dieser Fall ist hier schon einmal aufgetreten.
# Das Mount kommt ausschliesslich aus docker-compose.yml.
# Ebenfalls bewusst nicht enthalten: nginx und cronie.
# nginx laeuft als eigener Container (siehe docker-compose.yml), das
# Sync-Intervall steuert entrypoint.sh per Sleep-Schleife.
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
docker/appstream-gen.py
-302
View File
@@ -1,302 +0,0 @@
#!/usr/bin/env python3
"""
Erzeugt einen AppStream-Katalog (Collection-XML) aus packages.yaml.
Der Katalog referenziert die gespiegelten RPMs direkt ueber <pkgname>.
Es werden keine Wrapper-Pakete gebaut. Das Ergebnis wird per
`modifyrepo_c --mdtype=appstream` in die Repo-Metadaten eingehaengt,
damit Discover / GNOME Software die Pakete findet.
Aufruf:
appstream-gen.py packages.yaml -o appstream.xml.gz \
[--available-packages vorhanden.txt] [--origin firmen-repo]
"""
import argparse
import gzip
import re
import sys
import xml.etree.ElementTree as ET
from xml.dom import minidom
import yaml
# AppStream-Spezifikationsversion des Katalogformats.
CATALOG_VERSION = "0.14"
DEFAULT_TYPE = "console-application"
DEFAULT_ICON = "application-x-executable"
DEFAULT_NOTE = "Über Unternehmens-Repo bereitgestellt."
def die(msg):
sys.stdout.flush()
print(f"[appstream-gen] FEHLER: {msg}", file=sys.stderr, flush=True)
sys.exit(1)
def warn(msg):
sys.stdout.flush()
print(f"[appstream-gen] WARNUNG: {msg}", file=sys.stderr, flush=True)
def info(msg):
print(f"[appstream-gen] {msg}", flush=True)
def sanitize_id_part(value):
"""Macht aus einem Paketnamen ein gueltiges AppStream-ID-Segment."""
part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-")
if not part:
die(f"Paketname ergibt kein gueltiges ID-Segment: {value!r}")
if part[0].isdigit():
part = "pkg-" + part
return part
def prettify_name(pkgname):
"""Fallback-Anzeigename, wenn in der YAML keiner gesetzt ist.
dotnet-sdk-9.0 -> Dotnet Sdk 9.0
"""
words = re.split(r"[-_]", pkgname)
return " ".join(w[:1].upper() + w[1:] if w else w for w in words)
def as_list(value):
if value is None:
return []
if isinstance(value, (list, tuple)):
return list(value)
return [value]
def normalize_entry(entry, source_name):
"""Akzeptiert sowohl 'paketname' als auch ein Mapping mit Metadaten."""
if isinstance(entry, str):
return {"name": entry, "_source": source_name}
if isinstance(entry, dict):
if "name" not in entry:
die(f"Paketeintrag in Quelle {source_name!r} ohne 'name': {entry!r}")
data = dict(entry)
data["_source"] = source_name
return data
die(f"Unerwarteter Paketeintrag in Quelle {source_name!r}: {entry!r}")
def pick(entry, defaults, key, fallback=None):
"""Wert aus dem Paketeintrag, sonst aus defaults, sonst fallback."""
if key in entry and entry[key] is not None:
return entry[key]
if key in defaults and defaults[key] is not None:
return defaults[key]
return fallback
def build_component(entry, defaults, id_prefix):
pkgname = entry["name"]
comp_type = pick(entry, defaults, "type", DEFAULT_TYPE)
component = ET.Element("component", {"type": comp_type})
comp_id = entry.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}"
ET.SubElement(component, "id").text = comp_id
ET.SubElement(component, "pkgname").text = pkgname
display_name = entry.get("display_name") or entry.get("name_display")
if not display_name:
display_name = prettify_name(pkgname)
ET.SubElement(component, "name").text = display_name
note = pick(entry, defaults, "note", DEFAULT_NOTE)
summary = entry.get("summary") or note
ET.SubElement(component, "summary").text = summary
# Beschreibung: eigener Text plus Hinweis auf die Herkunft.
description = ET.SubElement(component, "description")
body = entry.get("description")
if body:
for paragraph in [p.strip() for p in str(body).split("\n\n") if p.strip()]:
ET.SubElement(description, "p").text = paragraph
if note and note != summary:
ET.SubElement(description, "p").text = note
elif not body and note == summary:
ET.SubElement(description, "p").text = note
icon = pick(entry, defaults, "icon", DEFAULT_ICON)
if icon:
ET.SubElement(component, "icon", {"type": "stock"}).text = icon
categories = as_list(pick(entry, defaults, "categories"))
if categories:
cat_el = ET.SubElement(component, "categories")
for category in categories:
ET.SubElement(cat_el, "category").text = str(category)
keywords = as_list(entry.get("keywords"))
if keywords:
kw_el = ET.SubElement(component, "keywords")
for keyword in keywords:
ET.SubElement(kw_el, "keyword").text = str(keyword)
binaries = as_list(entry.get("binary") or entry.get("binaries"))
if binaries:
provides = ET.SubElement(component, "provides")
for binary in binaries:
ET.SubElement(provides, "binary").text = str(binary)
homepage = entry.get("homepage") or defaults.get("homepage")
if homepage:
ET.SubElement(component, "url", {"type": "homepage"}).text = str(homepage)
help_url = entry.get("help_url") or defaults.get("help_url")
if help_url:
ET.SubElement(component, "url", {"type": "help"}).text = str(help_url)
developer = entry.get("developer") or defaults.get("developer")
if developer:
dev_el = ET.SubElement(component, "developer")
ET.SubElement(dev_el, "name").text = str(developer)
project_license = entry.get("project_license") or defaults.get("project_license")
if project_license:
ET.SubElement(component, "project_license").text = str(project_license)
# desktop-application braucht einen Launchable-Eintrag, sonst ignoriert
# Discover die Komponente beim Starten der installierten App.
if comp_type == "desktop-application":
desktop_id = entry.get("desktop_id")
if desktop_id:
ET.SubElement(
component, "launchable", {"type": "desktop-id"}
).text = str(desktop_id)
else:
warn(
f"{pkgname}: type=desktop-application ohne 'desktop_id'. "
"Discover kann die Anwendung dann nicht starten."
)
return comp_id, component
def load_available(path):
if not path:
return None
try:
with open(path, "r", encoding="utf-8") as handle:
names = {line.strip() for line in handle if line.strip()}
except OSError as exc:
die(f"Konnte Paketliste {path!r} nicht lesen: {exc}")
info(f"{len(names)} tatsaechlich vorhandene Pakete eingelesen")
return names
def main():
parser = argparse.ArgumentParser(
description="Erzeugt einen AppStream-Katalog aus packages.yaml"
)
parser.add_argument("packages_yaml", help="Pfad zur packages.yaml")
parser.add_argument(
"-o", "--output", default="appstream.xml.gz",
help="Zieldatei (.xml.gz oder .xml), Standard: appstream.xml.gz",
)
parser.add_argument(
"--origin",
help="Origin des Katalogs. Ueberschreibt 'appstream.origin' aus der YAML.",
)
parser.add_argument(
"--available-packages",
help="Datei mit den tatsaechlich gespiegelten Paketnamen (einer pro Zeile). "
"Nicht vorhandene Pakete werden dann uebersprungen.",
)
parser.add_argument(
"--strict", action="store_true",
help="Mit Fehler abbrechen, wenn ein gelistetes Paket nicht vorhanden ist.",
)
args = parser.parse_args()
try:
with open(args.packages_yaml, "r", encoding="utf-8") as handle:
config = yaml.safe_load(handle) or {}
except OSError as exc:
die(f"Konnte {args.packages_yaml!r} nicht lesen: {exc}")
except yaml.YAMLError as exc:
die(f"YAML-Fehler in {args.packages_yaml!r}: {exc}")
appstream_cfg = config.get("appstream") or {}
defaults = appstream_cfg.get("defaults") or {}
origin = args.origin or appstream_cfg.get("origin") or "firmen-repo"
id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo"
available = load_available(args.available_packages)
sources = config.get("sources") or []
if not sources:
die("Keine 'sources' in der packages.yaml gefunden.")
root = ET.Element("components", {"version": CATALOG_VERSION, "origin": origin})
seen_ids = {}
seen_pkgs = set()
skipped = []
count = 0
for source in sources:
source_name = source.get("name", "<unbenannt>")
for raw_entry in source.get("packages") or []:
entry = normalize_entry(raw_entry, source_name)
pkgname = entry["name"]
if entry.get("appstream") is False or entry.get("hidden") is True:
info(f"{pkgname}: per Konfiguration von AppStream ausgenommen")
continue
if available is not None and pkgname not in available:
skipped.append(pkgname)
continue
if pkgname in seen_pkgs:
warn(f"{pkgname}: mehrfach gelistet, spaetere Eintraege ignoriert")
continue
seen_pkgs.add(pkgname)
comp_id, component = build_component(entry, defaults, id_prefix)
if comp_id in seen_ids:
die(
f"Doppelte AppStream-ID {comp_id!r} "
f"({seen_ids[comp_id]} und {pkgname}). "
"Setze 'id:' fuer eines der beiden Pakete."
)
seen_ids[comp_id] = pkgname
root.append(component)
count += 1
if skipped:
message = (
f"{len(skipped)} gelistete Pakete sind nicht im Repo vorhanden "
f"und wurden uebersprungen: {', '.join(sorted(skipped))}"
)
if args.strict:
die(message)
warn(message)
raw = ET.tostring(root, encoding="utf-8")
pretty = minidom.parseString(raw).toprettyxml(indent=" ", encoding="UTF-8")
try:
if args.output.endswith(".gz"):
with gzip.open(args.output, "wb") as handle:
handle.write(pretty)
else:
with open(args.output, "wb") as handle:
handle.write(pretty)
except OSError as exc:
die(f"Konnte {args.output!r} nicht schreiben: {exc}")
info(f"{count} Komponente(n) nach {args.output} geschrieben (origin={origin})")
if count == 0:
warn("Katalog ist leer. Discover wird nichts anzeigen.")
if __name__ == "__main__":
main()
docker/docker-compose.yml
-41
View File
@@ -1,41 +0,0 @@
# Aus DIESEM Verzeichnis heraus starten:
# cd docker && docker compose up -d --build
services:
syncer:
build:
# Kontext ist das docker/-Verzeichnis, in dem auch diese Datei liegt.
context: .
dockerfile: Dockerfile
image: rpm-mirror-syncer:latest
container_name: rpm-mirror-syncer
restart: unless-stopped
env_file:
- .env
volumes:
# repo-data enthaelt ausschliesslich oeffentliche Dateien und wird
# deshalb auch vom nginx-Container gemountet.
- repo-data:/srv/repo
# sync-cache haelt den Git-Checkout der Paketliste und spaeter den
# privaten GPG-Signierschluessel - absichtlich NICHT in nginx.
- sync-cache:/var/cache/rpm-mirror-sync
# Nur noetig, wenn die Paketliste NICHT aus Git kommt
# (SOURCES_GIT_REPO leer lassen). Sonst auskommentiert lassen.
#- ${LOCAL_SOURCES_YAML:-./packages.example.yaml}:/config/sources.yaml:ro
nginx:
image: nginx:alpine
container_name: rpm-mirror-nginx
restart: unless-stopped
depends_on:
- syncer
ports:
- "${NGINX_PORT:-8081}:80"
volumes:
# Mount-Ziel und der Root in nginx.conf muessen zusammenpassen:
# root /usr/share/nginx/html + Mount auf .../html/repo => URL /repo/
- repo-data:/usr/share/nginx/html/repo:ro
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
volumes:
repo-data:
sync-cache:
docker/entrypoint.sh
-43
View File
@@ -1,43 +0,0 @@
#!/usr/bin/env bash
# Container entrypoint: run sync.sh once, or in a loop every
# SYNC_INTERVAL_SECONDS. There's no systemd/cron inside the container, so a
# simple sleep-loop is the pragmatic choice.
set -euo pipefail
: "${SYNC_INTERVAL_SECONDS:=3600}"
: "${RUN_ONCE:=false}"
log() { echo "[entrypoint] $(date '+%Y-%m-%d %H:%M:%S') $*"; }
run_once() {
/usr/local/bin/sync.sh
}
if [[ "$RUN_ONCE" == "true" ]]; then
log "RUN_ONCE=true - running sync.sh once and exiting"
run_once
exit $?
fi
log "Starting sync loop (interval: ${SYNC_INTERVAL_SECONDS}s)"
# Graceful shutdown on docker stop / compose down
term_handler() {
log "Received SIGTERM, shutting down"
exit 0
}
trap term_handler SIGTERM SIGINT
while true; do
# NOTE: don't inline `run_once` into `if ! run_once; then ...` - inside
# that then-branch, $? would reflect the (always-0) negated test itself,
# not run_once's real exit code. Capture it explicitly instead.
rc=0
run_once || rc=$?
if [[ $rc -ne 0 ]]; then
log "sync.sh failed (exit ${rc}) - will retry after the next interval"
fi
log "Sleeping ${SYNC_INTERVAL_SECONDS}s until next sync..."
sleep "${SYNC_INTERVAL_SECONDS}" &
wait $!
done
docker/nginx.conf
-38
View File
@@ -1,38 +0,0 @@
server {
listen 80;
# Declare IPv6 explicitly ourselves - the stock nginx image otherwise
# tries to inject this line into default.conf on startup
# (10-listen-on-ipv6-by-default.sh), which fails with a "read-only file
# system?" message because we mount this file read-only. Harmless, but
# noisy; setting it here avoids the entrypoint script touching the file
# at all.
listen [::]:80;
server_name _;
# Small landing response at the root so a bare hit doesn't 404.
location = / {
default_type text/plain;
return 200 "RPM mirror - browse /repo/\n";
}
# The actual mirrored repos (one subdirectory per source name) and the
# generated client .repo files under /repo/client-repos/.
location /repo/ {
alias /usr/share/nginx/html/repo/;
autoindex on;
autoindex_exact_size off;
autoindex_localtime on;
# repomd.xml is what dnf checks first to see if a repo changed -
# keep it uncached so clients notice updates immediately. Everything
# else (rpms, *.xml.gz) is content-addressed enough to cache freely.
if ($request_uri ~* /repodata/repomd\.xml$) {
add_header Cache-Control "no-cache";
}
}
location /healthz {
default_type text/plain;
return 200 "ok\n";
}
}
docker/sync.sh
-523
View File
@@ -1,523 +0,0 @@
#!/usr/bin/env bash
#
# sync.sh - Mirror selected packages from third-party RPM repos into ONE
# combined local repo consumable by AlmaLinux clients (dnf /
# PackageKit / GNOME Software / Discover) - WITHOUT pulling in or
# exposing packages that AlmaLinux itself already ships
# (AppStream, BaseOS, Extras, CRB/"Extras for Enterprise").
#
# ALL sources defined in the YAML end up in a SINGLE repo
# (REPO_BASE_DIR/pkgs/) with a SINGLE client-facing .repo file
# (CLIENT_REPO_DIR/mirror.repo) - clients only ever add one repo, no matter
# how many sources you configure.
#
# What it does, per entry in the YAML source list:
# 1. Registers the source repo temporarily and resolves the full
# dependency closure of the requested packages, using the source repo
# AND the local AlmaLinux repos (needed so resolution succeeds, e.g.
# glibc-style deps that only exist in AlmaLinux).
# 2. Throws away every RPM in that closure that AlmaLinux itself provides.
# Only RPMs that genuinely only exist in the third-party repo are kept.
# 3. Verifies each kept RPM against the source's own gpgkey (if given in
# the YAML), then STRIPS the original signature and RE-SIGNS it with
# the mirror's own GPG key. Clients therefore only ever need to trust
# one key (this mirror's), never the individual upstream repos' keys.
# 4. Moves the signed RPMs into the single shared repo directory
# (REPO_BASE_DIR/pkgs/).
# Once all sources have been processed, the script rebuilds repo metadata
# for that ONE directory with createrepo_c and generates AppStream "generic
# component" metadata for ONLY the packages explicitly listed under
# "packages:" across all sources (never for pulled-in dependencies),
# injecting it once into that same repo. Only those show up as installable
# items in GNOME Software/Discover.
#
# Set "metadata_only: true" on a source to skip steps 2-4 entirely for that
# source: no RPMs are downloaded/mirrored, only curated AppStream visibility
# is contributed to the combined repo. Use this when baseurl already points
# at a repo your clients have enabled directly (e.g. AlmaLinux's own
# AppStream repo, which on AlmaLinux 10 ships dotnet-* itself) - you just
# want to narrow down what shows up in Discover without duplicating RPM
# content clients can already get natively.
#
# NOTE ON NAME COLLISIONS: if two different sources define an RPM with the
# exact same NEVRA (name-version-release.arch), the later one silently wins
# in the shared directory (harmless - it's the same package). If two
# sources define the SAME PACKAGE NAME with genuinely different content,
# that's not supported here - keep package names distinct across sources.
#
# IMPORTANT LIMITATION:
# GNOME Software's curated "Explore" front page only lists packages that
# ship a real desktop-application AppStream component (icon, screenshots,
# .desktop file). A "generic" component (what this script generates for
# CLI tools/SDKs/runtimes) makes the package installable from Discover and
# shows up in search/details, but it will not appear on that curated front
# page. This is GNOME Software/appstreamcli behaviour and cannot be fixed
# by repo metadata alone.
#
# SECURITY NOTE ON RE-SIGNING:
# The mirror's private signing key is generated once (unattended, without
# a passphrase - standard practice for automated repo signing) and kept
# under GPG_HOME. Only the exported PUBLIC key ends up in REPO_BASE_DIR
# where clients can fetch it. GPG_HOME itself must never be exposed via
# nginx/the webserver and should live on storage only the sync process can
# read. Anyone who can read GPG_HOME can sign packages as your mirror -
# treat it like any other private key material.
#
# A revocation certificate is generated alongside the key on first run
# (GPG_HOME/revocation-cert.asc) - since the key never expires, this
# certificate is the ONLY way to invalidate it later if it's ever
# compromised. Copy it to secure, offline storage right after first
# startup; see the log output / README for the exact command.
#
# Requires: dnf, dnf-plugins-core (for "dnf download"), createrepo_c,
# modifyrepo_c, rpm-sign (rpmsign/rpmkeys), gnupg2, python3,
# python3-pyyaml, git (if fetching the source list from git), curl.
#
# Run as root (needs to write a temporary .repo file to /etc/yum.repos.d).
set -euo pipefail
# ---------------------------------------------------------------------------
# Configuration (override via environment, e.g. in the systemd unit/cron job)
# ---------------------------------------------------------------------------
# Where the YAML source list lives. Either point SOURCES_GIT_REPO at a git
# repo (SOURCES_YAML_FILE is the path *inside* that repo), or leave
# SOURCES_GIT_REPO empty and point SOURCES_YAML_FILE at a local file.
SOURCES_GIT_REPO="${SOURCES_GIT_REPO:-}"
SOURCES_GIT_BRANCH="${SOURCES_GIT_BRANCH:-main}"
SOURCES_YAML_FILE="${SOURCES_YAML_FILE:-sources.yaml}"
# Local working/output paths.
WORK_DIR="${WORK_DIR:-/var/cache/rpm-mirror-sync}"
REPO_BASE_DIR="${REPO_BASE_DIR:-/srv/repo}"
CLIENT_REPO_DIR="${CLIENT_REPO_DIR:-${REPO_BASE_DIR}/client-repos}"
# The ONE combined repo directory all sources' packages end up in.
PKG_DIR="${REPO_BASE_DIR}/pkgs"
# Base URL under which REPO_BASE_DIR is actually served to clients (web
# server / reverse proxy in front of REPO_BASE_DIR). Used only to generate
# ready-to-use .repo files for clients.
CLIENT_BASE_URL="${CLIENT_BASE_URL:-http://mirror.example.local/repo}"
# repoids of AlmaLinux's own repos on THIS machine, as shown by
# `dnf repolist`. Adjust to match your system if they differ.
# ("Extras for Enterprise" is the "extras" repo in AlmaLinux 10 naming; crb
# is CodeReady Builder / "extras-common" on some setups - check your
# `dnf repolist` output and adjust below.)
ALMA_REPO_IDS="${ALMA_REPO_IDS:-baseos appstream extras crb}"
# Origin of the generated AppStream catalog. Must be unique - if it collides
# with a distribution's own origin (e.g. "almalinux"), the icon/metadata
# caches on the clients overwrite each other. Can also be set via
# "appstream: origin:" in the YAML; this variable wins.
APPSTREAM_ORIGIN="${APPSTREAM_ORIGIN:-rpm-mirror}"
# The mirror's own signing identity. A key is generated once (on first run)
# under GPG_HOME and reused on every subsequent run - make sure GPG_HOME
# points at persistent storage (a volume), or you'll get a new key (and
# therefore a trust-breaking change for clients) on every restart.
GPG_HOME="${GPG_HOME:-${WORK_DIR}/gnupg}"
GPG_KEY_NAME="${GPG_KEY_NAME:-RPM Mirror}"
GPG_KEY_EMAIL="${GPG_KEY_EMAIL:-rpm-mirror@example.local}"
MIRROR_GPG_KEY_FILE="${REPO_BASE_DIR}/RPM-GPG-KEY-mirror"
LOG_TAG="rpm-mirror-sync"
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${WORK_DIR}/sync.log" >&2; }
die() { log "ERROR: $*"; exit 1; }
require_cmd() {
command -v "$1" >/dev/null 2>&1 || die "required command '$1' not found (install it first)"
}
cleanup() {
# Remove any temp source .repo files we created, even on failure.
if [[ -n "${TEMP_REPO_FILES:-}" ]]; then
for f in "${TEMP_REPO_FILES[@]}"; do
[[ -n "$f" ]] && rm -f "$f"
done
fi
}
# Remove one entry from TEMP_REPO_FILES.
# NOTE: "${ARRAY[@]/pattern}" does SUBSTRING REPLACEMENT, so the element is
# left behind as an empty string rather than removed - which then makes
# cleanup() call `rm -f ""`. Rebuild the array instead.
forget_repo_file() {
local drop="$1" keep=() f
for f in "${TEMP_REPO_FILES[@]}"; do
[[ "$f" == "$drop" ]] || keep+=( "$f" )
done
TEMP_REPO_FILES=( "${keep[@]}" )
}
trap cleanup EXIT
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
HELPER="${SCRIPT_DIR}/sync_helpers.py"
# ---------------------------------------------------------------------------
# Preconditions
# ---------------------------------------------------------------------------
[[ $EUID -eq 0 ]] || die "must run as root (writes to /etc/yum.repos.d and ${REPO_BASE_DIR})"
for c in dnf createrepo_c modifyrepo_c rpm rpmsign rpmkeys gpg python3 curl gzip; do
require_cmd "$c"
done
[[ -f "$HELPER" ]] || die "sync_helpers.py not found next to sync.sh (expected: $HELPER)"
python3 -c "import yaml" 2>/dev/null || die "python3-pyyaml not installed (dnf install -y python3-pyyaml)"
mkdir -p "$WORK_DIR" "$REPO_BASE_DIR" "$CLIENT_REPO_DIR" "$PKG_DIR"
# ---------------------------------------------------------------------------
# Mirror signing key: generate once, reuse forever, export the public part
# ---------------------------------------------------------------------------
MIRROR_GPG_KEY_ID=""
REVOCATION_CERT_FILE="${GPG_HOME}/revocation-cert.asc"
init_mirror_gpg_key() {
mkdir -p "$GPG_HOME"
chmod 700 "$GPG_HOME"
if [[ -z "$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons 2>/dev/null)" ]]; then
log "No mirror signing key found under ${GPG_HOME} - generating one (one-time)..."
gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \
--quick-generate-key "${GPG_KEY_NAME} <${GPG_KEY_EMAIL}>" rsa4096 sign never \
|| die "failed to generate mirror GPG signing key"
# GnuPG >= 2.1 automatically writes a revocation certificate under
# openpgp-revocs.d/<fingerprint>.rev as part of key generation itself
# (visible in the gpg output above: "revocation certificate stored
# as ..."). We just need to copy it to a known, stable path - no
# need to (fragile-ly) drive gpg's interactive --gen-revoke menu
# ourselves, which behaves differently across gpg versions.
local auto_revoc
auto_revoc="$(find "${GPG_HOME}/openpgp-revocs.d" -maxdepth 1 -name '*.rev' 2>/dev/null | head -n1)"
[[ -n "$auto_revoc" && -s "$auto_revoc" ]] \
|| die "expected GnuPG to auto-generate a revocation certificate under ${GPG_HOME}/openpgp-revocs.d/ but none was found"
cp "$auto_revoc" "$REVOCATION_CERT_FILE"
chmod 600 "$REVOCATION_CERT_FILE"
log "!!! Revocation certificate written to: ${REVOCATION_CERT_FILE}"
log "!!! Copy it to secure, OFFLINE storage now, e.g.:"
log "!!! docker compose cp syncer:${REVOCATION_CERT_FILE} ./mirror-key-revocation-cert.asc"
log "!!! It is the only way to invalidate this key if it is ever compromised -"
log "!!! it never expires, so it needs the same protection as the private key."
fi
MIRROR_GPG_KEY_ID="$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons \
| awk -F: '/^sec:/ {print $5; exit}')"
[[ -n "$MIRROR_GPG_KEY_ID" ]] || die "could not determine mirror GPG key id"
# rpmsign/rpm --addsign etc. read the signing identity from macros.
cat > "${HOME:-/root}/.rpmmacros" <<EOF
%_gpg_name ${MIRROR_GPG_KEY_ID}
%_gpg_path ${GPG_HOME}
%_signature gpg
EOF
# Publish the PUBLIC key once at the repo root. This is the only key
# clients will ever need, no matter how many upstream sources exist.
gpg --homedir "$GPG_HOME" --armor --export "$MIRROR_GPG_KEY_ID" > "$MIRROR_GPG_KEY_FILE"
log "Mirror signing key ready: ${MIRROR_GPG_KEY_ID} (public key: ${MIRROR_GPG_KEY_FILE})"
}
init_mirror_gpg_key
# ---------------------------------------------------------------------------
# Fetch the YAML source list
# ---------------------------------------------------------------------------
resolve_yaml_path() {
if [[ -n "$SOURCES_GIT_REPO" ]]; then
if [[ "$SOURCES_YAML_FILE" == /* ]]; then
die "SOURCES_GIT_REPO is set, but SOURCES_YAML_FILE ('${SOURCES_YAML_FILE}') looks like an absolute path. When pulling the source list from git, SOURCES_YAML_FILE must be a path RELATIVE to that repo's root (e.g. 'sources.yaml' or 'config/sources.yaml') - not the '/config/...' path used for the local bind-mount option. Fix SOURCES_YAML_FILE in .env."
fi
local clone_dir="${WORK_DIR}/sources-repo"
if [[ -d "${clone_dir}/.git" ]]; then
log "Updating source list git repo..."
git -C "$clone_dir" fetch --depth 1 origin "$SOURCES_GIT_BRANCH"
git -C "$clone_dir" reset --hard "origin/${SOURCES_GIT_BRANCH}"
else
log "Cloning source list git repo..."
rm -rf "$clone_dir"
git clone --depth 1 --branch "$SOURCES_GIT_BRANCH" "$SOURCES_GIT_REPO" "$clone_dir"
fi
echo "${clone_dir}/${SOURCES_YAML_FILE}"
else
echo "$SOURCES_YAML_FILE"
fi
}
YAML_PATH="$(resolve_yaml_path)"
[[ -f "$YAML_PATH" ]] || die "source YAML not found: $YAML_PATH"
SOURCES_JSON="$(python3 "$HELPER" parse-yaml "$YAML_PATH")" || die "failed to parse $YAML_PATH"
SOURCE_COUNT="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$SOURCES_JSON")"
log "Loaded $SOURCE_COUNT source(s) from $YAML_PATH"
# ---------------------------------------------------------------------------
# Per-source sync
# ---------------------------------------------------------------------------
declare -a TEMP_REPO_FILES=()
# Build --enablerepo flags for the AlmaLinux repos as a bash array once.
declare -a ALMA_ENABLE_FLAGS=()
for r in $ALMA_REPO_IDS; do
ALMA_ENABLE_FLAGS+=( "--enablerepo=${r}" )
done
sync_one_source() {
local name="$1" baseurl="$2" gpgkey="$3" packages_json="$4" metadata_only="${5:-false}"
local repoid="mirror-src-${name}"
local repo_file="/etc/yum.repos.d/${repoid}.repo"
local tmp_dir="${WORK_DIR}/${name}/download"
local list_dir="${WORK_DIR}/${name}"
# This source's contribution to the combined AppStream metadata: one
# "name<TAB>summary" line per explicitly requested package. Collected
# here, merged across all sources, and turned into ONE appstream.xml
# after the whole source loop finishes.
local entries_tsv="${list_dir}/appstream-entries.tsv"
log "=== Syncing source '${name}' (metadata_only=${metadata_only}) ==="
mkdir -p "$tmp_dir" "$list_dir"
: > "$entries_tsv"
# readarray of requested top-level package names for this source.
# parse-yaml normalises every entry to an object, so a package written
# as a plain string and one written as a mapping with AppStream metadata
# both arrive here as {"name": ...}.
mapfile -t packages < <(python3 -c '
import json,sys
for p in json.load(sys.stdin):
print(p["name"])
' <<<"$packages_json")
[[ ${#packages[@]} -gt 0 ]] || die "source '${name}' has no packages"
# --- 1. Register the source repo temporarily -------------------------
{
echo "[${repoid}]"
echo "name=Mirror source: ${name}"
echo "baseurl=${baseurl}"
echo "enabled=1"
if [[ -n "$gpgkey" ]]; then
echo "gpgcheck=1"
echo "gpgkey=${gpgkey}"
else
echo "gpgcheck=0"
fi
} > "$repo_file"
TEMP_REPO_FILES+=( "$repo_file" )
dnf clean expire-cache --disablerepo='*' --enablerepo="$repoid" >/dev/null 2>&1 || true
# --- metadata_only sources: curate Discover/GNOME Software visibility
# WITHOUT mirroring any RPM content. Use this when baseurl already
# points at a repo your clients have enabled directly (e.g. AlmaLinux's
# own AppStream repo, which ships dotnet-* itself on AlmaLinux 10) - the
# actual package install is then resolved by dnf from that repo as
# normal; this source only contributes AppStream "generic component"
# data for the packages listed to the combined repo built after the
# source loop, so ONLY those show up as installable in Discover instead
# of AlmaLinux's entire (huge) AppStream catalog.
# PRECONDITION: clients must have the repo(s) that actually provide
# these packages enabled already (true for any stock AlmaLinux install -
# baseos/appstream/extras/crb are on by default).
if [[ "$metadata_only" == "true" ]]; then
for pkg in "${packages[@]}"; do
local summary
summary="$(dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \
--qf '%{summary}' "$pkg" 2>/dev/null | head -n1)"
printf '%s\t%s\n' "$pkg" "$summary" >> "$entries_tsv"
done
log "Source '${name}': metadata_only - queued AppStream visibility for ${#packages[@]} package(s), no RPMs mirrored"
rm -f "$repo_file"
forget_repo_file "$repo_file"
return
fi
# --- 2. List everything the source repo itself provides --------------
# (used later to decide "does this RPM genuinely come from the source,
# or did AlmaLinux already have it".)
local source_nevra_list="${list_dir}/source-nevra.txt"
dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \
--qf '%{name}-%{version}-%{release}.%{arch}' -a \
> "$source_nevra_list" \
|| die "failed to query source repo '${name}' (check baseurl/gpgkey)"
[[ -s "$source_nevra_list" ]] || die "source repo '${name}' returned no packages - check baseurl"
# --- 3. Resolve full dependency closure for the requested packages ---
# Enable the source repo AND the local AlmaLinux repos so dependency
# resolution succeeds even for deps AlmaLinux normally provides -
# we filter those back out in step 4.
rm -rf "$tmp_dir"; mkdir -p "$tmp_dir"
log "Resolving + downloading dependency closure for: ${packages[*]}"
dnf download -y --resolve --alldeps \
--destdir="$tmp_dir" \
--disablerepo='*' \
--enablerepo="$repoid" \
"${ALMA_ENABLE_FLAGS[@]}" \
"${packages[@]}" \
|| die "dnf download failed for source '${name}'"
# --- 4. Drop everything AlmaLinux already provides, verify + re-sign --
# the rest with the mirror's own key -----------------------------------
local verify_dbpath="${list_dir}/verify-rpmdb"
if [[ -n "$gpgkey" ]]; then
local source_gpgkey_file="${list_dir}/source-gpgkey.asc"
curl -fsSL "$gpgkey" -o "$source_gpgkey_file" \
|| die "could not fetch gpgkey for source '${name}' from ${gpgkey}"
rm -rf "$verify_dbpath"; mkdir -p "$verify_dbpath"
rpm --dbpath "$verify_dbpath" --initdb
rpm --dbpath "$verify_dbpath" --import "$source_gpgkey_file" \
|| die "could not import gpgkey for source '${name}'"
else
log " WARNING: source '${name}' has no gpgkey configured - skipping upstream signature verification"
fi
local kept=0 dropped=0 kept_requested=0 kept_dependency=0
for f in "$tmp_dir"/*.rpm; do
[[ -e "$f" ]] || continue
local base; base="$(basename "$f" .rpm)"
if ! grep -qxF "$base" "$source_nevra_list"; then
log " dropping (provided by AlmaLinux): $base"
rm -f "$f"
dropped=$((dropped + 1))
continue
fi
if [[ -n "$gpgkey" ]]; then
rpmkeys --dbpath "$verify_dbpath" --checksig "$f" >/dev/null 2>&1 \
|| die "signature verification FAILED for '${base}' from source '${name}' - refusing to mirror it (tampered download or wrong gpgkey configured)"
fi
# Strip the upstream signature and sign with the mirror's own key,
# so clients only ever need to trust MIRROR_GPG_KEY_FILE.
rpmsign --resign "$f" >/dev/null \
|| die "failed to re-sign '${base}' with the mirror key"
# Label each kept package as explicitly "requested" (listed under
# packages: in the YAML) or a pulled-in "dependency" - packages the
# requested ones need but AlmaLinux doesn't provide, so they have to
# be mirrored too or `dnf install <requested>` breaks on clients.
# This is purely informational (log output); both kinds are kept.
local pkg_name tag
pkg_name="$(rpm -qp --qf '%{NAME}' "$f" 2>/dev/null)"
tag="dependency"
for req in "${packages[@]}"; do
if [[ "$pkg_name" == "$req" ]]; then
tag="requested"
break
fi
done
if [[ "$tag" == "requested" ]]; then
kept_requested=$((kept_requested + 1))
local summary
summary="$(rpm -qp --qf '%{SUMMARY}' "$f" 2>/dev/null)"
printf '%s\t%s\n' "$pkg_name" "$summary" >> "$entries_tsv"
else
kept_dependency=$((kept_dependency + 1))
fi
log " keeping (${tag}): $base"
mv "$f" "${PKG_DIR}/"
kept=$((kept + 1))
done
log "Source '${name}': kept ${kept} package(s) - ${kept_requested} explicitly requested + ${kept_dependency} pulled-in third-party dependencies (re-signed with mirror key), dropped ${dropped} AlmaLinux-provided package(s)"
[[ $kept -gt 0 ]] || die "nothing kept for source '${name}' - check baseurl/package names"
# --- cleanup temp repo file for this source ---------------------------
rm -f "$repo_file"
forget_repo_file "$repo_file"
}
# Iterate over sources (JSON array -> one JSON object per line via jq-less python)
while IFS= read -r src_json; do
name="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["name"])' "$src_json")"
baseurl="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["baseurl"])' "$src_json")"
gpgkey="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1]).get("gpgkey",""))' "$src_json")"
pkgs_json="$(python3 -c 'import json,sys;print(json.dumps(json.loads(sys.argv[1])["packages"]))' "$src_json")"
metadata_only="$(python3 -c 'import json,sys;print(str(bool(json.loads(sys.argv[1]).get("metadata_only", False))).lower())' "$src_json")"
sync_one_source "$name" "$baseurl" "$gpgkey" "$pkgs_json" "$metadata_only"
done < <(python3 -c '
import json,sys
for s in json.load(sys.stdin):
print(json.dumps(s))
' <<<"$SOURCES_JSON")
# ---------------------------------------------------------------------------
# Build the ONE combined repo from everything all sources contributed
# ---------------------------------------------------------------------------
createrepo_c --update "$PKG_DIR" >/dev/null
# Build ONE AppStream catalog for the single shared repo. The helper reads
# every source's appstream-entries.tsv (written during its own processing
# above) to know WHICH packages made it into the mirror, and packages.yaml
# to know HOW to present them (display name, description, categories,
# keywords). YAML metadata wins; the upstream RPM summary from the TSV is
# the fallback.
APPSTREAM_XML="${WORK_DIR}/appstream.xml"
python3 "$HELPER" gen-appstream \
--origin "$APPSTREAM_ORIGIN" \
--yaml "$YAML_PATH" \
--entries-dir "$WORK_DIR" \
> "$APPSTREAM_XML" \
|| die "failed to generate AppStream catalog"
# Purely informational: appstreamcli's checks target the style expectations
# for public app stores (content ratings, screenshots, ...). They do not
# affect whether Discover shows the packages.
if command -v appstreamcli >/dev/null 2>&1; then
appstreamcli validate --no-net "$APPSTREAM_XML" >/dev/null 2>&1 \
|| log "note: AppStream catalog has style hints (harmless) - run 'appstreamcli validate ${APPSTREAM_XML}' to see them"
fi
gzip -fk "$APPSTREAM_XML"
modifyrepo_c --mdtype=appstream "${APPSTREAM_XML}.gz" "${PKG_DIR}/repodata" >/dev/null
# Detached-sign repomd.xml itself with the mirror key, so clients can verify
# the METADATA (which packages/versions exist, their checksums) hasn't been
# tampered with - not just each individual RPM. Must happen last, after
# createrepo_c/modifyrepo_c are done touching repomd.xml (they update its
# checksums), otherwise the signature would be over stale content.
gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \
--detach-sign --armor \
--output "${PKG_DIR}/repodata/repomd.xml.asc" \
"${PKG_DIR}/repodata/repomd.xml" \
|| die "failed to sign repomd.xml with the mirror key"
# --- The ONE client-facing .repo file - this is all clients ever need -----
cat > "${CLIENT_REPO_DIR}/mirror.repo" <<EOF
[rpm-mirror]
name=RPM mirror
baseurl=${CLIENT_BASE_URL}/pkgs/
enabled=1
gpgcheck=1
gpgkey=${CLIENT_BASE_URL}/RPM-GPG-KEY-mirror
# gpgcheck=1 verifies each RPM's own signature (re-signed with the mirror
# key). repo_gpgcheck=1 additionally verifies repomd.xml itself against
# repomd.xml.asc (generated above) - together these confirm both the
# package contents AND the package listing/checksums came from this
# mirror, unmodified.
repo_gpgcheck=1
EOF
log "All sources synced into a single repo at ${PKG_DIR}/"
log "All mirrored packages are signed with mirror key ${MIRROR_GPG_KEY_ID} - clients only need ${MIRROR_GPG_KEY_FILE}."
log "Client setup (one command, one repo, done):"
log " curl -fsSL ${CLIENT_BASE_URL}/client-repos/mirror.repo -o /etc/yum.repos.d/rpm-mirror.repo && dnf makecache"
docker/sync_helpers.py
-367
View File
@@ -1,367 +0,0 @@
#!/usr/bin/env python3
"""
sync_helpers.py - small helper used by sync.sh
Reliable YAML parsing and XML escaping in pure bash/sed is a losing game, so
these two jobs live here:
parse-yaml <file>
Reads the "sources:" list from the YAML file, NORMALISES it and prints
it as JSON on stdout, so sync.sh can loop over it safely.
Normalisation: every entry under "packages:" becomes an object with at
least a "name" key, whether it was written as a plain string or as a
mapping with extra AppStream metadata. sync.sh can therefore always do
p["name"] without caring which form the YAML used.
gen-appstream --origin O --yaml F --entries-dir D
Builds the AppStream catalog for the combined repo.
Metadata precedence, highest first:
1. what packages.yaml says about the package
2. the upstream RPM summary collected by sync.sh into the
appstream-entries.tsv files under --entries-dir
3. a generated fallback
Only packages that actually made it into the mirror (i.e. that appear
in one of the TSV files) get a component - never pulled-in
dependencies, and never packages whose download failed.
NOTE ON DISCOVER/GNOME SOFTWARE:
Components without an icon/screenshots/.desktop file make a package
installable and findable via SEARCH, but they will not appear on the
curated "Explore" front page or when browsing categories. That is a
frontend behaviour for non-desktop-app components and cannot be fixed
at the repo-metadata level. Only a real desktop-application component
(type: desktop-application + desktop_id: in packages.yaml, which
requires the package to ship a .desktop file) shows up there.
"""
import argparse
import glob
import json
import os
import re
import sys
from xml.sax.saxutils import escape, quoteattr
CATALOG_VERSION = "0.14"
DEFAULT_NOTE = (
"Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. "
"Fragen und Paketwünsche bitte an den IT-Service-Desk."
)
def die(msg):
sys.stdout.flush()
sys.stderr.write(f"ERROR: {msg}\n")
sys.exit(1)
def warn(msg):
sys.stdout.flush()
sys.stderr.write(f"WARNING: {msg}\n")
def load_yaml(path):
try:
import yaml
except ImportError:
die(
"python3-pyyaml (PyYAML) is not installed.\n"
"Install it with: dnf install -y python3-pyyaml"
)
try:
with open(path, "r", encoding="utf-8") as fh:
return yaml.safe_load(fh) or {}
except OSError as exc:
die(f"could not read {path!r}: {exc}")
except Exception as exc: # yaml.YAMLError and friends
die(f"could not parse {path!r}: {exc}")
def normalise_package(entry, source_name, index):
"""Accept both 'pkgname' and {name: pkgname, display_name: ...}."""
if isinstance(entry, str):
return {"name": entry}
if isinstance(entry, dict):
if not entry.get("name"):
die(
f"package #{index} in source {source_name!r} has no 'name' key: "
f"{entry!r}"
)
return dict(entry)
die(
f"package #{index} in source {source_name!r} must be a string or a "
f"mapping, got {type(entry).__name__}"
)
def normalise_sources(data):
sources = data.get("sources") or []
if not isinstance(sources, list):
die("'sources' in YAML is not a list")
for i, src in enumerate(sources):
if not isinstance(src, dict):
die(f"source #{i} is not a mapping")
for field in ("name", "baseurl", "packages"):
if field not in src:
die(f"source #{i} is missing required field '{field}'")
if not isinstance(src["packages"], list) or not src["packages"]:
die(
f"source '{src.get('name')}' has an empty/invalid packages list"
)
src["packages"] = [
normalise_package(p, src.get("name"), j)
for j, p in enumerate(src["packages"])
]
return sources
def cmd_parse_yaml(path):
data = load_yaml(path)
json.dump(normalise_sources(data), sys.stdout)
# ---------------------------------------------------------------------------
# AppStream catalog generation
# ---------------------------------------------------------------------------
def sanitize_id_part(value):
part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-")
if not part:
die(f"package name yields no usable AppStream id segment: {value!r}")
if part[0].isdigit():
part = "pkg-" + part
return part
def as_list(value):
if value is None:
return []
if isinstance(value, (list, tuple)):
return list(value)
return [value]
def pick(meta, defaults, key, fallback=None):
if meta.get(key) is not None:
return meta[key]
if defaults.get(key) is not None:
return defaults[key]
return fallback
def read_entries(entries_dir):
"""Merge every source's appstream-entries.tsv into {name: summary}."""
entries = {}
pattern = os.path.join(entries_dir, "*", "appstream-entries.tsv")
for path in sorted(glob.glob(pattern)):
try:
with open(path, encoding="utf-8") as fh:
for line in fh:
line = line.rstrip("\n")
if not line:
continue
parts = line.split("\t", 1)
name = parts[0].strip()
summary = parts[1].strip() if len(parts) > 1 else ""
if not name:
continue
if name in entries and entries[name] and summary \
and entries[name] != summary:
warn(
f"package '{name}' was contributed by more than one "
"source with differing summaries - keeping the first"
)
continue
entries.setdefault(name, summary)
except OSError:
continue
return entries
def collect_metadata(yaml_path):
"""Return (defaults, {pkgname: metadata}, id_prefix, yaml_origin)."""
data = load_yaml(yaml_path)
appstream_cfg = data.get("appstream") or {}
defaults = dict(appstream_cfg.get("defaults") or {})
id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo"
yaml_origin = appstream_cfg.get("origin")
if not defaults.get("developer_id"):
segments = id_prefix.split(".")
if len(segments) >= 2:
defaults["developer_id"] = ".".join(segments[:2])
by_name = {}
for src in normalise_sources(data):
for pkg in src["packages"]:
name = pkg["name"]
if name in by_name:
warn(f"package '{name}' listed more than once - keeping the first")
continue
by_name[name] = pkg
return defaults, by_name, id_prefix, yaml_origin
def emit(out, depth, text):
out.append(" " * depth + text)
def build_component(pkgname, meta, upstream_summary, defaults, id_prefix, out):
if meta.get("appstream") is False or meta.get("hidden") is True:
return False
binaries = as_list(meta.get("binary") or meta.get("binaries"))
# A console-application component without <provides><binary> trips
# appstreamcli's 'console-app-no-binary' check, so only claim that type
# when a binary is actually declared. Runtimes and libraries stay
# 'generic', which Discover still finds via search.
comp_type = pick(meta, defaults, "type")
if not comp_type:
comp_type = "console-application" if binaries else "generic"
comp_id = meta.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}"
display_name = meta.get("display_name") or pkgname
note = pick(meta, defaults, "note", DEFAULT_NOTE)
summary = meta.get("summary") or upstream_summary or note
emit(out, 1, f"<component type={quoteattr(comp_type)}>")
emit(out, 2, f"<id>{escape(comp_id)}</id>")
emit(out, 2, f"<pkgname>{escape(pkgname)}</pkgname>")
emit(out, 2, f"<name>{escape(display_name)}</name>")
emit(out, 2, f"<summary>{escape(summary)}</summary>")
emit(out, 2, "<description>")
body = meta.get("description")
wrote_paragraph = False
if body:
for para in [p.strip() for p in str(body).split("\n\n") if p.strip()]:
emit(out, 3, f"<p>{escape(para)}</p>")
wrote_paragraph = True
if note and (not wrote_paragraph or note != summary):
emit(out, 3, f"<p>{escape(note)}</p>")
elif not wrote_paragraph:
emit(out, 3, f"<p>{escape(summary)}</p>")
emit(out, 2, "</description>")
icon = pick(meta, defaults, "icon", "application-x-executable")
if icon:
emit(out, 2, f'<icon type="stock">{escape(str(icon))}</icon>')
categories = as_list(pick(meta, defaults, "categories"))
if categories:
emit(out, 2, "<categories>")
for cat in categories:
emit(out, 3, f"<category>{escape(str(cat))}</category>")
emit(out, 2, "</categories>")
keywords = as_list(meta.get("keywords"))
if keywords:
emit(out, 2, "<keywords>")
for kw in keywords:
emit(out, 3, f"<keyword>{escape(str(kw))}</keyword>")
emit(out, 2, "</keywords>")
if binaries:
emit(out, 2, "<provides>")
for binary in binaries:
emit(out, 3, f"<binary>{escape(str(binary))}</binary>")
emit(out, 2, "</provides>")
homepage = meta.get("homepage") or defaults.get("homepage")
if homepage:
emit(out, 2, f'<url type="homepage">{escape(str(homepage))}</url>')
help_url = meta.get("help_url") or defaults.get("help_url")
if help_url:
emit(out, 2, f'<url type="help">{escape(str(help_url))}</url>')
developer = meta.get("developer") or defaults.get("developer")
if developer:
dev_id = meta.get("developer_id") or defaults.get("developer_id")
attr = f" id={quoteattr(str(dev_id))}" if dev_id else ""
emit(out, 2, f"<developer{attr}>")
emit(out, 3, f"<name>{escape(str(developer))}</name>")
emit(out, 2, "</developer>")
project_license = meta.get("project_license") or defaults.get("project_license")
if project_license:
emit(out, 2, f"<project_license>{escape(str(project_license))}</project_license>")
# A desktop-application component that Discover cannot launch is worse
# than a generic one, so insist on the desktop file id.
if comp_type == "desktop-application":
desktop_id = meta.get("desktop_id")
if desktop_id:
emit(
out, 2,
f'<launchable type="desktop-id">{escape(str(desktop_id))}</launchable>',
)
else:
warn(
f"{pkgname}: type=desktop-application without 'desktop_id' - "
"Discover will not be able to launch it after installation"
)
emit(out, 1, "</component>")
return True
def cmd_gen_appstream(origin, yaml_path, entries_dir):
defaults, by_name, id_prefix, yaml_origin = collect_metadata(yaml_path)
entries = read_entries(entries_dir)
if not entries:
warn(
f"no appstream-entries.tsv found under {entries_dir} - the catalog "
"will be empty and nothing will show up in Discover"
)
effective_origin = origin or yaml_origin or "rpm-mirror"
out = []
out.append('<?xml version="1.0" encoding="UTF-8"?>')
out.append(
f'<components version="{CATALOG_VERSION}" origin={quoteattr(effective_origin)}>'
)
written = 0
for pkgname in sorted(entries):
meta = by_name.get(pkgname, {})
if build_component(pkgname, meta, entries[pkgname], defaults,
id_prefix, out):
written += 1
out.append("</components>")
sys.stdout.write("\n".join(out) + "\n")
sys.stderr.write(
f"gen-appstream: wrote {written} component(s), origin={effective_origin}\n"
)
def main():
ap = argparse.ArgumentParser()
sub = ap.add_subparsers(dest="cmd", required=True)
p1 = sub.add_parser("parse-yaml")
p1.add_argument("path")
p2 = sub.add_parser("gen-appstream")
p2.add_argument("--origin", default="")
p2.add_argument("--yaml", required=True)
p2.add_argument("--entries-dir", required=True)
args = ap.parse_args()
if args.cmd == "parse-yaml":
cmd_parse_yaml(args.path)
elif args.cmd == "gen-appstream":
cmd_gen_appstream(args.origin, args.yaml, args.entries_dir)
if __name__ == "__main__":
main()