diff --git a/docker/syncer/sync.sh b/docker/syncer/sync.sh index b14efff..3c01bce 100644 --- a/docker/syncer/sync.sh +++ b/docker/syncer/sync.sh @@ -1,191 +1,523 @@ -#!/bin/bash -# Synct die Paket-Whitelist aus Git (oder aus einer lokal gemounteten Datei) -# und laedt die selektierten Pakete aus den jeweils angegebenen Fremd-Repos. -# Erzeugt anschliessend RPM- und - sofern der Generator vorhanden ist - -# AppStream-Metadaten. +#!/usr/bin/env bash +# +# sync.sh - Mirror selected packages from third-party RPM repos into ONE +# combined local repo consumable by AlmaLinux clients (dnf / +# PackageKit / GNOME Software / Discover) - WITHOUT pulling in or +# exposing packages that AlmaLinux itself already ships +# (AppStream, BaseOS, Extras, CRB/"Extras for Enterprise"). +# +# ALL sources defined in the YAML end up in a SINGLE repo +# (REPO_BASE_DIR/pkgs/) with a SINGLE client-facing .repo file +# (CLIENT_REPO_DIR/mirror.repo) - clients only ever add one repo, no matter +# how many sources you configure. +# +# What it does, per entry in the YAML source list: +# 1. Registers the source repo temporarily and resolves the full +# dependency closure of the requested packages, using the source repo +# AND the local AlmaLinux repos (needed so resolution succeeds, e.g. +# glibc-style deps that only exist in AlmaLinux). +# 2. Throws away every RPM in that closure that AlmaLinux itself provides. +# Only RPMs that genuinely only exist in the third-party repo are kept. +# 3. Verifies each kept RPM against the source's own gpgkey (if given in +# the YAML), then STRIPS the original signature and RE-SIGNS it with +# the mirror's own GPG key. Clients therefore only ever need to trust +# one key (this mirror's), never the individual upstream repos' keys. +# 4. Moves the signed RPMs into the single shared repo directory +# (REPO_BASE_DIR/pkgs/). +# Once all sources have been processed, the script rebuilds repo metadata +# for that ONE directory with createrepo_c and generates AppStream "generic +# component" metadata for ONLY the packages explicitly listed under +# "packages:" across all sources (never for pulled-in dependencies), +# injecting it once into that same repo. Only those show up as installable +# items in GNOME Software/Discover. +# +# Set "metadata_only: true" on a source to skip steps 2-4 entirely for that +# source: no RPMs are downloaded/mirrored, only curated AppStream visibility +# is contributed to the combined repo. Use this when baseurl already points +# at a repo your clients have enabled directly (e.g. AlmaLinux's own +# AppStream repo, which on AlmaLinux 10 ships dotnet-* itself) - you just +# want to narrow down what shows up in Discover without duplicating RPM +# content clients can already get natively. +# +# NOTE ON NAME COLLISIONS: if two different sources define an RPM with the +# exact same NEVRA (name-version-release.arch), the later one silently wins +# in the shared directory (harmless - it's the same package). If two +# sources define the SAME PACKAGE NAME with genuinely different content, +# that's not supported here - keep package names distinct across sources. +# +# IMPORTANT LIMITATION: +# GNOME Software's curated "Explore" front page only lists packages that +# ship a real desktop-application AppStream component (icon, screenshots, +# .desktop file). A "generic" component (what this script generates for +# CLI tools/SDKs/runtimes) makes the package installable from Discover and +# shows up in search/details, but it will not appear on that curated front +# page. This is GNOME Software/appstreamcli behaviour and cannot be fixed +# by repo metadata alone. +# +# SECURITY NOTE ON RE-SIGNING: +# The mirror's private signing key is generated once (unattended, without +# a passphrase - standard practice for automated repo signing) and kept +# under GPG_HOME. Only the exported PUBLIC key ends up in REPO_BASE_DIR +# where clients can fetch it. GPG_HOME itself must never be exposed via +# nginx/the webserver and should live on storage only the sync process can +# read. Anyone who can read GPG_HOME can sign packages as your mirror - +# treat it like any other private key material. +# +# A revocation certificate is generated alongside the key on first run +# (GPG_HOME/revocation-cert.asc) - since the key never expires, this +# certificate is the ONLY way to invalidate it later if it's ever +# compromised. Copy it to secure, offline storage right after first +# startup; see the log output / README for the exact command. +# +# Requires: dnf, dnf-plugins-core (for "dnf download"), createrepo_c, +# modifyrepo_c, rpm-sign (rpmsign/rpmkeys), gnupg2, python3, +# python3-pyyaml, git (if fetching the source list from git), curl. +# +# Run as root (needs to write a temporary .repo file to /etc/yum.repos.d). + set -euo pipefail -# createrepo_c uebernimmt die umask des Aufrufers. Ohne "umask 022" kann -# repodata/ mit 0700 landen; der nginx-Worker (User nginx) kommt dann nicht -# hinein und liefert 403 - bzw. 404, falls die nginx-Config try_files nutzt. -umask 022 +# --------------------------------------------------------------------------- +# Configuration (override via environment, e.g. in the systemd unit/cron job) +# --------------------------------------------------------------------------- -# --- Pfade ---------------------------------------------------------------- -# Muessen zu den Volumes in docker-compose.yml passen: -# repo-data -> /srv/repo (im nginx-Container: /usr/share/nginx/html/repo) -# sync-cache -> /var/cache/rpm-mirror-sync -# Daraus ergibt sich die oeffentliche URL: /repo/pkgs/repodata/repomd.xml +# Where the YAML source list lives. Either point SOURCES_GIT_REPO at a git +# repo (SOURCES_YAML_FILE is the path *inside* that repo), or leave +# SOURCES_GIT_REPO empty and point SOURCES_YAML_FILE at a local file. +SOURCES_GIT_REPO="${SOURCES_GIT_REPO:-}" +SOURCES_GIT_BRANCH="${SOURCES_GIT_BRANCH:-main}" +SOURCES_YAML_FILE="${SOURCES_YAML_FILE:-sources.yaml}" + +# Local working/output paths. WORK_DIR="${WORK_DIR:-/var/cache/rpm-mirror-sync}" -REPO_ROOT="${REPO_ROOT:-/srv/repo}" -DEST="$REPO_ROOT/pkgs" +REPO_BASE_DIR="${REPO_BASE_DIR:-/srv/repo}" +CLIENT_REPO_DIR="${CLIENT_REPO_DIR:-${REPO_BASE_DIR}/client-repos}" +# The ONE combined repo directory all sources' packages end up in. +PKG_DIR="${REPO_BASE_DIR}/pkgs" -CONFIG_DIR="$WORK_DIR/config" -LOCAL_PACKAGES_FILE=/config/packages.yaml -TMP_REPO_DIR=/etc/yum.repos.d -LOCKFILE="$WORK_DIR/sync.lock" -APPSTREAM_GEN="${APPSTREAM_GEN:-/usr/local/bin/appstream-gen.py}" +# Base URL under which REPO_BASE_DIR is actually served to clients (web +# server / reverse proxy in front of REPO_BASE_DIR). Used only to generate +# ready-to-use .repo files for clients. +CLIENT_BASE_URL="${CLIENT_BASE_URL:-http://mirror.example.local/repo}" -mkdir -p "$WORK_DIR" "$DEST" +# repoids of AlmaLinux's own repos on THIS machine, as shown by +# `dnf repolist`. Adjust to match your system if they differ. +# ("Extras for Enterprise" is the "extras" repo in AlmaLinux 10 naming; crb +# is CodeReady Builder / "extras-common" on some setups - check your +# `dnf repolist` output and adjust below.) +ALMA_REPO_IDS="${ALMA_REPO_IDS:-baseos appstream extras crb}" -exec 200>"$LOCKFILE" -if ! flock -n 200; then - echo "[sync] Ein anderer Sync-Lauf ist bereits aktiv, breche ab." - exit 0 -fi +# Origin of the generated AppStream catalog. Must be unique - if it collides +# with a distribution's own origin (e.g. "almalinux"), the icon/metadata +# caches on the clients overwrite each other. Can also be set via +# "appstream: origin:" in the YAML; this variable wins. +APPSTREAM_ORIGIN="${APPSTREAM_ORIGIN:-rpm-mirror}" -echo "[sync] $(date -Iseconds) Starte Sync-Lauf" +# The mirror's own signing identity. A key is generated once (on first run) +# under GPG_HOME and reused on every subsequent run - make sure GPG_HOME +# points at persistent storage (a volume), or you'll get a new key (and +# therefore a trust-breaking change for clients) on every restart. +GPG_HOME="${GPG_HOME:-${WORK_DIR}/gnupg}" +GPG_KEY_NAME="${GPG_KEY_NAME:-RPM Mirror}" +GPG_KEY_EMAIL="${GPG_KEY_EMAIL:-rpm-mirror@example.local}" +MIRROR_GPG_KEY_FILE="${REPO_BASE_DIR}/RPM-GPG-KEY-mirror" -# --- 1. Whitelist besorgen ------------------------------------------------ -# Eine gemountete /config/packages.yaml hat Vorrang. Nur wenn sie fehlt, -# wird SOURCES_GIT_REPO gebraucht. -if [ -f "$LOCAL_PACKAGES_FILE" ]; then - PACKAGES_FILE="$LOCAL_PACKAGES_FILE" - echo "[sync] Nutze lokal gemountete $PACKAGES_FILE" -else - : "${SOURCES_GIT_REPO:?weder SOURCES_GIT_REPO gesetzt noch /config/packages.yaml gemountet}" - CONFIG_REPO_BRANCH="${CONFIG_REPO_BRANCH:-main}" - PACKAGES_FILE="$CONFIG_DIR/packages.yaml" +LOG_TAG="rpm-mirror-sync" - if [ -n "${GIT_SSH_KEY_PATH:-}" ] && [ -f "$GIT_SSH_KEY_PATH" ]; then - export GIT_SSH_COMMAND="ssh -i ${GIT_SSH_KEY_PATH} -o StrictHostKeyChecking=accept-new" - fi +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- - if [ -d "$CONFIG_DIR/.git" ]; then - git -C "$CONFIG_DIR" fetch --depth=1 origin "$CONFIG_REPO_BRANCH" - git -C "$CONFIG_DIR" reset --hard "origin/$CONFIG_REPO_BRANCH" - else - # Reste eines abgebrochenen Clones wegraeumen, sonst scheitert git clone. - rm -rf "$CONFIG_DIR" - git clone --depth=1 --branch "$CONFIG_REPO_BRANCH" "$SOURCES_GIT_REPO" "$CONFIG_DIR" - fi -fi +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${WORK_DIR}/sync.log" >&2; } +die() { log "ERROR: $*"; exit 1; } -if [ ! -f "$PACKAGES_FILE" ]; then - echo "[sync] FEHLER: $PACKAGES_FILE nicht gefunden." - exit 1 -fi +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "required command '$1' not found (install it first)" +} -# --- 2. Pro Quelle: temporaeres Repo aktivieren, Pakete gezielt ziehen ----- -source_count=$(yq -o=json '.sources | length' "$PACKAGES_FILE") -download_failed=0 - -for i in $(seq 0 $((source_count - 1))); do - src=$(yq -o=json ".sources[$i]" "$PACKAGES_FILE") - name=$(echo "$src" | jq -r '.name') - baseurl=$(echo "$src" | jq -r '.baseurl') - gpgkey=$(echo "$src" | jq -r '.gpgkey // empty') - - # Standard: false. Nur bei echten Fremd-Repos aktivieren, deren - # Abhaengigkeiten die Clients nicht ueber ihre Basis-Repos bekommen. - resolve_deps=$(echo "$src" | jq -r '.resolve_deps // false') - - # Paketeintraege koennen Strings ODER Mappings mit Metadaten sein. - mapfile -t packages < <( - echo "$src" | jq -r '.packages[] | if type == "string" then . else .name end' - ) - - if [ "${#packages[@]}" -eq 0 ]; then - echo "[sync] Quelle '$name': keine Pakete gelistet, ueberspringe." - continue - fi - - echo "[sync] Quelle '$name': ${#packages[@]} Paket(e) -> $baseurl" - - repofile="$TMP_REPO_DIR/tmp-${name}.repo" - { - echo "[tmp-${name}]" - echo "name=${name}" - echo "baseurl=${baseurl}" - echo "enabled=0" - if [ -n "$gpgkey" ]; then - echo "gpgcheck=1" - echo "gpgkey=${gpgkey}" - else - echo "gpgcheck=0" +cleanup() { + # Remove any temp source .repo files we created, even on failure. + if [[ -n "${TEMP_REPO_FILES:-}" ]]; then + for f in "${TEMP_REPO_FILES[@]}"; do + [[ -n "$f" ]] && rm -f "$f" + done fi - } > "$repofile" +} - # Die Basis-Repos des Containers bleiben aktiviert, damit dnf - # Abhaengigkeiten wie glibc oder libstdc++ ueberhaupt aufloesen kann. - # Ohne sie scheitert bereits die Aufloesung, nicht erst der Download. - dl_args=(--destdir="$DEST" --enablerepo="tmp-${name}") +# Remove one entry from TEMP_REPO_FILES. +# NOTE: "${ARRAY[@]/pattern}" does SUBSTRING REPLACEMENT, so the element is +# left behind as an empty string rather than removed - which then makes +# cleanup() call `rm -f ""`. Rebuild the array instead. +forget_repo_file() { + local drop="$1" keep=() f + for f in "${TEMP_REPO_FILES[@]}"; do + [[ "$f" == "$drop" ]] || keep+=( "$f" ) + done + TEMP_REPO_FILES=( "${keep[@]}" ) +} +trap cleanup EXIT - if [ "$resolve_deps" = "true" ]; then - echo "[sync] Quelle '$name': Abhaengigkeiten werden mitgespiegelt" - # Bewusst OHNE --alldeps: bereits im Container installierte Pakete - # (glibc & Co.) haben die Clients ohnehin und muessen nicht ins Repo. - dl_args+=(--resolve) - fi +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +HELPER="${SCRIPT_DIR}/sync_helpers.py" - if ! dnf download "${dl_args[@]}" "${packages[@]}"; then - echo "[sync] WARNUNG: Download fuer Quelle '$name' fehlgeschlagen." - download_failed=1 - fi +# --------------------------------------------------------------------------- +# Preconditions +# --------------------------------------------------------------------------- - rm -f "$repofile" +[[ $EUID -eq 0 ]] || die "must run as root (writes to /etc/yum.repos.d and ${REPO_BASE_DIR})" + +for c in dnf createrepo_c modifyrepo_c rpm rpmsign rpmkeys gpg python3 curl gzip; do + require_cmd "$c" +done +[[ -f "$HELPER" ]] || die "sync_helpers.py not found next to sync.sh (expected: $HELPER)" +python3 -c "import yaml" 2>/dev/null || die "python3-pyyaml not installed (dnf install -y python3-pyyaml)" + +mkdir -p "$WORK_DIR" "$REPO_BASE_DIR" "$CLIENT_REPO_DIR" "$PKG_DIR" + +# --------------------------------------------------------------------------- +# Mirror signing key: generate once, reuse forever, export the public part +# --------------------------------------------------------------------------- + +MIRROR_GPG_KEY_ID="" +REVOCATION_CERT_FILE="${GPG_HOME}/revocation-cert.asc" + +init_mirror_gpg_key() { + mkdir -p "$GPG_HOME" + chmod 700 "$GPG_HOME" + + if [[ -z "$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons 2>/dev/null)" ]]; then + log "No mirror signing key found under ${GPG_HOME} - generating one (one-time)..." + gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \ + --quick-generate-key "${GPG_KEY_NAME} <${GPG_KEY_EMAIL}>" rsa4096 sign never \ + || die "failed to generate mirror GPG signing key" + + # GnuPG >= 2.1 automatically writes a revocation certificate under + # openpgp-revocs.d/.rev as part of key generation itself + # (visible in the gpg output above: "revocation certificate stored + # as ..."). We just need to copy it to a known, stable path - no + # need to (fragile-ly) drive gpg's interactive --gen-revoke menu + # ourselves, which behaves differently across gpg versions. + local auto_revoc + auto_revoc="$(find "${GPG_HOME}/openpgp-revocs.d" -maxdepth 1 -name '*.rev' 2>/dev/null | head -n1)" + [[ -n "$auto_revoc" && -s "$auto_revoc" ]] \ + || die "expected GnuPG to auto-generate a revocation certificate under ${GPG_HOME}/openpgp-revocs.d/ but none was found" + + cp "$auto_revoc" "$REVOCATION_CERT_FILE" + chmod 600 "$REVOCATION_CERT_FILE" + + log "!!! Revocation certificate written to: ${REVOCATION_CERT_FILE}" + log "!!! Copy it to secure, OFFLINE storage now, e.g.:" + log "!!! docker compose cp syncer:${REVOCATION_CERT_FILE} ./mirror-key-revocation-cert.asc" + log "!!! It is the only way to invalidate this key if it is ever compromised -" + log "!!! it never expires, so it needs the same protection as the private key." + fi + + MIRROR_GPG_KEY_ID="$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons \ + | awk -F: '/^sec:/ {print $5; exit}')" + [[ -n "$MIRROR_GPG_KEY_ID" ]] || die "could not determine mirror GPG key id" + + # rpmsign/rpm --addsign etc. read the signing identity from macros. + cat > "${HOME:-/root}/.rpmmacros" < "$MIRROR_GPG_KEY_FILE" + + log "Mirror signing key ready: ${MIRROR_GPG_KEY_ID} (public key: ${MIRROR_GPG_KEY_FILE})" +} + +init_mirror_gpg_key + + + +# --------------------------------------------------------------------------- +# Fetch the YAML source list +# --------------------------------------------------------------------------- + +resolve_yaml_path() { + if [[ -n "$SOURCES_GIT_REPO" ]]; then + if [[ "$SOURCES_YAML_FILE" == /* ]]; then + die "SOURCES_GIT_REPO is set, but SOURCES_YAML_FILE ('${SOURCES_YAML_FILE}') looks like an absolute path. When pulling the source list from git, SOURCES_YAML_FILE must be a path RELATIVE to that repo's root (e.g. 'sources.yaml' or 'config/sources.yaml') - not the '/config/...' path used for the local bind-mount option. Fix SOURCES_YAML_FILE in .env." + fi + local clone_dir="${WORK_DIR}/sources-repo" + if [[ -d "${clone_dir}/.git" ]]; then + log "Updating source list git repo..." + git -C "$clone_dir" fetch --depth 1 origin "$SOURCES_GIT_BRANCH" + git -C "$clone_dir" reset --hard "origin/${SOURCES_GIT_BRANCH}" + else + log "Cloning source list git repo..." + rm -rf "$clone_dir" + git clone --depth 1 --branch "$SOURCES_GIT_BRANCH" "$SOURCES_GIT_REPO" "$clone_dir" + fi + echo "${clone_dir}/${SOURCES_YAML_FILE}" + else + echo "$SOURCES_YAML_FILE" + fi +} + +YAML_PATH="$(resolve_yaml_path)" +[[ -f "$YAML_PATH" ]] || die "source YAML not found: $YAML_PATH" + +SOURCES_JSON="$(python3 "$HELPER" parse-yaml "$YAML_PATH")" || die "failed to parse $YAML_PATH" +SOURCE_COUNT="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$SOURCES_JSON")" +log "Loaded $SOURCE_COUNT source(s) from $YAML_PATH" + +# --------------------------------------------------------------------------- +# Per-source sync +# --------------------------------------------------------------------------- + +declare -a TEMP_REPO_FILES=() + +# Build --enablerepo flags for the AlmaLinux repos as a bash array once. +declare -a ALMA_ENABLE_FLAGS=() +for r in $ALMA_REPO_IDS; do + ALMA_ENABLE_FLAGS+=( "--enablerepo=${r}" ) done -# --- 3. Alte Paketversionen aufraeumen ------------------------------------ -# repomanage braucht bereits vorhandene Metadaten. Beim allerersten Lauf -# gibt es noch keine, deshalb der Existenztest. -if [ -f "$DEST/repodata/repomd.xml" ]; then - echo "[sync] Raeume veraltete Paketversionen auf" - if ! dnf repomanage --old "$DEST" | xargs -r rm -f; then - echo "[sync] WARNUNG: repomanage fehlgeschlagen, ueberspringe Aufraeumen." - fi -else - echo "[sync] Erster Lauf, kein Aufraeumen noetig." -fi +sync_one_source() { + local name="$1" baseurl="$2" gpgkey="$3" packages_json="$4" metadata_only="${5:-false}" -# --- 4. RPM-Metadaten neu erzeugen ---------------------------------------- -echo "[sync] Aktualisiere Repo-Metadaten" -createrepo_c --update "$DEST" + local repoid="mirror-src-${name}" + local repo_file="/etc/yum.repos.d/${repoid}.repo" + local tmp_dir="${WORK_DIR}/${name}/download" + local list_dir="${WORK_DIR}/${name}" + # This source's contribution to the combined AppStream metadata: one + # "namesummary" line per explicitly requested package. Collected + # here, merged across all sources, and turned into ONE appstream.xml + # after the whole source loop finishes. + local entries_tsv="${list_dir}/appstream-entries.tsv" -# --- 5. AppStream-Katalog erzeugen und einhaengen -------------------------- -# Ohne diesen Schritt sind die Pakete zwar per dnf installierbar, tauchen -# aber weder in Discover noch in GNOME Software auf. Der Generator ist -# optional: fehlt er, laeuft der Sync trotzdem sauber durch. -if [ -x "$APPSTREAM_GEN" ]; then - echo "[sync] Erzeuge AppStream-Katalog" + log "=== Syncing source '${name}' (metadata_only=${metadata_only}) ===" + mkdir -p "$tmp_dir" "$list_dir" + : > "$entries_tsv" - ASWORK=$(mktemp -d) - trap 'rm -rf "$ASWORK"' EXIT + # readarray of requested top-level package names for this source. + # parse-yaml normalises every entry to an object, so a package written + # as a plain string and one written as a mapping with AppStream metadata + # both arrive here as {"name": ...}. + mapfile -t packages < <(python3 -c ' +import json,sys +for p in json.load(sys.stdin): + print(p["name"]) +' <<<"$packages_json") + [[ ${#packages[@]} -gt 0 ]] || die "source '${name}' has no packages" - # Welche Pakete liegen tatsaechlich im Repo? Verhindert, dass Discover - # Eintraege anzeigt, deren RPM gar nicht da ist. - if compgen -G "$DEST"/*.rpm > /dev/null; then - rpm -qp --qf '%{NAME}\n' "$DEST"/*.rpm 2>/dev/null \ - | sort -u > "$ASWORK/vorhanden.txt" - else - echo "[sync] WARNUNG: Keine RPMs in $DEST gefunden." - : > "$ASWORK/vorhanden.txt" - fi + # --- 1. Register the source repo temporarily ------------------------- + { + echo "[${repoid}]" + echo "name=Mirror source: ${name}" + echo "baseurl=${baseurl}" + echo "enabled=1" + if [[ -n "$gpgkey" ]]; then + echo "gpgcheck=1" + echo "gpgkey=${gpgkey}" + else + echo "gpgcheck=0" + fi + } > "$repo_file" + TEMP_REPO_FILES+=( "$repo_file" ) + dnf clean expire-cache --disablerepo='*' --enablerepo="$repoid" >/dev/null 2>&1 || true - python3 "$APPSTREAM_GEN" "$PACKAGES_FILE" \ - --output "$ASWORK/appstream.xml.gz" \ - --available-packages "$ASWORK/vorhanden.txt" + # --- metadata_only sources: curate Discover/GNOME Software visibility + # WITHOUT mirroring any RPM content. Use this when baseurl already + # points at a repo your clients have enabled directly (e.g. AlmaLinux's + # own AppStream repo, which ships dotnet-* itself on AlmaLinux 10) - the + # actual package install is then resolved by dnf from that repo as + # normal; this source only contributes AppStream "generic component" + # data for the packages listed to the combined repo built after the + # source loop, so ONLY those show up as installable in Discover instead + # of AlmaLinux's entire (huge) AppStream catalog. + # PRECONDITION: clients must have the repo(s) that actually provide + # these packages enabled already (true for any stock AlmaLinux install - + # baseos/appstream/extras/crb are on by default). + if [[ "$metadata_only" == "true" ]]; then + for pkg in "${packages[@]}"; do + local summary + summary="$(dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \ + --qf '%{summary}' "$pkg" 2>/dev/null | head -n1)" + printf '%s\t%s\n' "$pkg" "$summary" >> "$entries_tsv" + done - # Optionale Validierung, falls appstreamcli im Image vorhanden ist. - if command -v appstreamcli > /dev/null 2>&1; then - if ! appstreamcli validate --no-net "$ASWORK/appstream.xml.gz"; then - echo "[sync] WARNUNG: AppStream-Katalog hat Validierungshinweise." + log "Source '${name}': metadata_only - queued AppStream visibility for ${#packages[@]} package(s), no RPMs mirrored" + + rm -f "$repo_file" + forget_repo_file "$repo_file" + return fi - fi - # modifyrepo_c MUSS nach createrepo_c laufen, da createrepo_c die - # repomd.xml komplett neu schreibt. - modifyrepo_c --mdtype=appstream "$ASWORK/appstream.xml.gz" "$DEST/repodata/" -else - echo "[sync] HINWEIS: $APPSTREAM_GEN nicht vorhanden - AppStream wird uebersprungen." + # --- 2. List everything the source repo itself provides -------------- + # (used later to decide "does this RPM genuinely come from the source, + # or did AlmaLinux already have it".) + local source_nevra_list="${list_dir}/source-nevra.txt" + dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \ + --qf '%{name}-%{version}-%{release}.%{arch}' -a \ + > "$source_nevra_list" \ + || die "failed to query source repo '${name}' (check baseurl/gpgkey)" + [[ -s "$source_nevra_list" ]] || die "source repo '${name}' returned no packages - check baseurl" + + # --- 3. Resolve full dependency closure for the requested packages --- + # Enable the source repo AND the local AlmaLinux repos so dependency + # resolution succeeds even for deps AlmaLinux normally provides - + # we filter those back out in step 4. + rm -rf "$tmp_dir"; mkdir -p "$tmp_dir" + log "Resolving + downloading dependency closure for: ${packages[*]}" + dnf download -y --resolve --alldeps \ + --destdir="$tmp_dir" \ + --disablerepo='*' \ + --enablerepo="$repoid" \ + "${ALMA_ENABLE_FLAGS[@]}" \ + "${packages[@]}" \ + || die "dnf download failed for source '${name}'" + + # --- 4. Drop everything AlmaLinux already provides, verify + re-sign -- + # the rest with the mirror's own key ----------------------------------- + local verify_dbpath="${list_dir}/verify-rpmdb" + if [[ -n "$gpgkey" ]]; then + local source_gpgkey_file="${list_dir}/source-gpgkey.asc" + curl -fsSL "$gpgkey" -o "$source_gpgkey_file" \ + || die "could not fetch gpgkey for source '${name}' from ${gpgkey}" + rm -rf "$verify_dbpath"; mkdir -p "$verify_dbpath" + rpm --dbpath "$verify_dbpath" --initdb + rpm --dbpath "$verify_dbpath" --import "$source_gpgkey_file" \ + || die "could not import gpgkey for source '${name}'" + else + log " WARNING: source '${name}' has no gpgkey configured - skipping upstream signature verification" + fi + + local kept=0 dropped=0 kept_requested=0 kept_dependency=0 + for f in "$tmp_dir"/*.rpm; do + [[ -e "$f" ]] || continue + local base; base="$(basename "$f" .rpm)" + + if ! grep -qxF "$base" "$source_nevra_list"; then + log " dropping (provided by AlmaLinux): $base" + rm -f "$f" + dropped=$((dropped + 1)) + continue + fi + + if [[ -n "$gpgkey" ]]; then + rpmkeys --dbpath "$verify_dbpath" --checksig "$f" >/dev/null 2>&1 \ + || die "signature verification FAILED for '${base}' from source '${name}' - refusing to mirror it (tampered download or wrong gpgkey configured)" + fi + + # Strip the upstream signature and sign with the mirror's own key, + # so clients only ever need to trust MIRROR_GPG_KEY_FILE. + rpmsign --resign "$f" >/dev/null \ + || die "failed to re-sign '${base}' with the mirror key" + + # Label each kept package as explicitly "requested" (listed under + # packages: in the YAML) or a pulled-in "dependency" - packages the + # requested ones need but AlmaLinux doesn't provide, so they have to + # be mirrored too or `dnf install ` breaks on clients. + # This is purely informational (log output); both kinds are kept. + local pkg_name tag + pkg_name="$(rpm -qp --qf '%{NAME}' "$f" 2>/dev/null)" + tag="dependency" + for req in "${packages[@]}"; do + if [[ "$pkg_name" == "$req" ]]; then + tag="requested" + break + fi + done + if [[ "$tag" == "requested" ]]; then + kept_requested=$((kept_requested + 1)) + local summary + summary="$(rpm -qp --qf '%{SUMMARY}' "$f" 2>/dev/null)" + printf '%s\t%s\n' "$pkg_name" "$summary" >> "$entries_tsv" + else + kept_dependency=$((kept_dependency + 1)) + fi + log " keeping (${tag}): $base" + + mv "$f" "${PKG_DIR}/" + kept=$((kept + 1)) + done + log "Source '${name}': kept ${kept} package(s) - ${kept_requested} explicitly requested + ${kept_dependency} pulled-in third-party dependencies (re-signed with mirror key), dropped ${dropped} AlmaLinux-provided package(s)" + [[ $kept -gt 0 ]] || die "nothing kept for source '${name}' - check baseurl/package names" + + # --- cleanup temp repo file for this source --------------------------- + rm -f "$repo_file" + forget_repo_file "$repo_file" +} + +# Iterate over sources (JSON array -> one JSON object per line via jq-less python) +while IFS= read -r src_json; do + name="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["name"])' "$src_json")" + baseurl="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["baseurl"])' "$src_json")" + gpgkey="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1]).get("gpgkey",""))' "$src_json")" + pkgs_json="$(python3 -c 'import json,sys;print(json.dumps(json.loads(sys.argv[1])["packages"]))' "$src_json")" + metadata_only="$(python3 -c 'import json,sys;print(str(bool(json.loads(sys.argv[1]).get("metadata_only", False))).lower())' "$src_json")" + sync_one_source "$name" "$baseurl" "$gpgkey" "$pkgs_json" "$metadata_only" +done < <(python3 -c ' +import json,sys +for s in json.load(sys.stdin): + print(json.dumps(s)) +' <<<"$SOURCES_JSON") + +# --------------------------------------------------------------------------- +# Build the ONE combined repo from everything all sources contributed +# --------------------------------------------------------------------------- + +createrepo_c --update "$PKG_DIR" >/dev/null + +# Build ONE AppStream catalog for the single shared repo. The helper reads +# every source's appstream-entries.tsv (written during its own processing +# above) to know WHICH packages made it into the mirror, and packages.yaml +# to know HOW to present them (display name, description, categories, +# keywords). YAML metadata wins; the upstream RPM summary from the TSV is +# the fallback. +APPSTREAM_XML="${WORK_DIR}/appstream.xml" +python3 "$HELPER" gen-appstream \ + --origin "$APPSTREAM_ORIGIN" \ + --yaml "$YAML_PATH" \ + --entries-dir "$WORK_DIR" \ + > "$APPSTREAM_XML" \ + || die "failed to generate AppStream catalog" + +# Purely informational: appstreamcli's checks target the style expectations +# for public app stores (content ratings, screenshots, ...). They do not +# affect whether Discover shows the packages. +if command -v appstreamcli >/dev/null 2>&1; then + appstreamcli validate --no-net "$APPSTREAM_XML" >/dev/null 2>&1 \ + || log "note: AppStream catalog has style hints (harmless) - run 'appstreamcli validate ${APPSTREAM_XML}' to see them" fi -# --- 6. Leserechte fuer den Webserver sicherstellen ------------------------ -# a+rX = Leserecht auf alles, Betretungsrecht nur auf Verzeichnisse. -chmod -R a+rX "$REPO_ROOT" +gzip -fk "$APPSTREAM_XML" +modifyrepo_c --mdtype=appstream "${APPSTREAM_XML}.gz" "${PKG_DIR}/repodata" >/dev/null -if [ "$download_failed" -ne 0 ]; then - echo "[sync] $(date -Iseconds) Sync-Lauf mit Fehlern abgeschlossen" - exit 1 -fi +# Detached-sign repomd.xml itself with the mirror key, so clients can verify +# the METADATA (which packages/versions exist, their checksums) hasn't been +# tampered with - not just each individual RPM. Must happen last, after +# createrepo_c/modifyrepo_c are done touching repomd.xml (they update its +# checksums), otherwise the signature would be over stale content. +gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \ + --detach-sign --armor \ + --output "${PKG_DIR}/repodata/repomd.xml.asc" \ + "${PKG_DIR}/repodata/repomd.xml" \ + || die "failed to sign repomd.xml with the mirror key" -echo "[sync] $(date -Iseconds) Sync-Lauf abgeschlossen" \ No newline at end of file +# --- The ONE client-facing .repo file - this is all clients ever need ----- +cat > "${CLIENT_REPO_DIR}/mirror.repo" <