From 21426e5f0f285870667db2cad27761cfbdf07068 Mon Sep 17 00:00:00 2001 From: "David M. Klein" <4+leander19961@noreply.localhost> Date: Wed, 26 Aug 2026 11:30:51 +0200 Subject: [PATCH] docker/sync.sh aktualisiert --- docker/sync.sh | 601 +++++++++---------------------------------------- 1 file changed, 104 insertions(+), 497 deletions(-) diff --git a/docker/sync.sh b/docker/sync.sh index a5c9041..10720d6 100644 --- a/docker/sync.sh +++ b/docker/sync.sh @@ -1,526 +1,133 @@ -#!/usr/bin/env bash -# -# sync.sh - Mirror selected packages from third-party RPM repos into ONE -# combined local repo consumable by AlmaLinux clients (dnf / -# PackageKit / GNOME Software / Discover) - WITHOUT pulling in or -# exposing packages that AlmaLinux itself already ships -# (AppStream, BaseOS, Extras, CRB/"Extras for Enterprise"). -# -# ALL sources defined in the YAML end up in a SINGLE repo -# (REPO_BASE_DIR/pkgs/) with a SINGLE client-facing .repo file -# (CLIENT_REPO_DIR/mirror.repo) - clients only ever add one repo, no matter -# how many sources you configure. -# -# What it does, per entry in the YAML source list: -# 1. Registers the source repo temporarily and resolves the full -# dependency closure of the requested packages, using the source repo -# AND the local AlmaLinux repos (needed so resolution succeeds, e.g. -# glibc-style deps that only exist in AlmaLinux). -# 2. Throws away every RPM in that closure that AlmaLinux itself provides. -# Only RPMs that genuinely only exist in the third-party repo are kept. -# 3. Verifies each kept RPM against the source's own gpgkey (if given in -# the YAML), then STRIPS the original signature and RE-SIGNS it with -# the mirror's own GPG key. Clients therefore only ever need to trust -# one key (this mirror's), never the individual upstream repos' keys. -# 4. Moves the signed RPMs into the single shared repo directory -# (REPO_BASE_DIR/pkgs/). -# Once all sources have been processed, the script rebuilds repo metadata -# for that ONE directory with createrepo_c and generates AppStream "generic -# component" metadata for ONLY the packages explicitly listed under -# "packages:" across all sources (never for pulled-in dependencies), -# injecting it once into that same repo. Only those show up as installable -# items in GNOME Software/Discover. -# -# Set "metadata_only: true" on a source to skip steps 2-4 entirely for that -# source: no RPMs are downloaded/mirrored, only curated AppStream visibility -# is contributed to the combined repo. Use this when baseurl already points -# at a repo your clients have enabled directly (e.g. AlmaLinux's own -# AppStream repo, which on AlmaLinux 10 ships dotnet-* itself) - you just -# want to narrow down what shows up in Discover without duplicating RPM -# content clients can already get natively. -# -# NOTE ON NAME COLLISIONS: if two different sources define an RPM with the -# exact same NEVRA (name-version-release.arch), the later one silently wins -# in the shared directory (harmless - it's the same package). If two -# sources define the SAME PACKAGE NAME with genuinely different content, -# that's not supported here - keep package names distinct across sources. -# -# IMPORTANT LIMITATION: -# GNOME Software's curated "Explore" front page only lists packages that -# ship a real desktop-application AppStream component (icon, screenshots, -# .desktop file). A "generic" component (what this script generates for -# CLI tools/SDKs/runtimes) makes the package installable from Discover and -# shows up in search/details, but it will not appear on that curated front -# page. This is GNOME Software/appstreamcli behaviour and cannot be fixed -# by repo metadata alone. -# -# SECURITY NOTE ON RE-SIGNING: -# The mirror's private signing key is generated once (unattended, without -# a passphrase - standard practice for automated repo signing) and kept -# under GPG_HOME. Only the exported PUBLIC key ends up in REPO_BASE_DIR -# where clients can fetch it. GPG_HOME itself must never be exposed via -# nginx/the webserver and should live on storage only the sync process can -# read. Anyone who can read GPG_HOME can sign packages as your mirror - -# treat it like any other private key material. -# -# A revocation certificate is generated alongside the key on first run -# (GPG_HOME/revocation-cert.asc) - since the key never expires, this -# certificate is the ONLY way to invalidate it later if it's ever -# compromised. Copy it to secure, offline storage right after first -# startup; see the log output / README for the exact command. -# -# Requires: dnf, dnf-plugins-core (for "dnf download"), createrepo_c, -# modifyrepo_c, rpm-sign (rpmsign/rpmkeys), gnupg2, python3, -# python3-pyyaml, git (if fetching the source list from git), curl. -# -# Run as root (needs to write a temporary .repo file to /etc/yum.repos.d). - +#!/bin/bash +# Synct die Paket-Whitelist aus Git und laedt die selektierten Pakete +# (inkl. Abhaengigkeiten) aus den jeweils angegebenen Fremd-Repos. +# Erzeugt anschliessend RPM- und AppStream-Metadaten. set -euo pipefail -# --------------------------------------------------------------------------- -# Configuration (override via environment, e.g. in the systemd unit/cron job) -# --------------------------------------------------------------------------- +CONFIG_DIR=/etc/mirror +CONFIG_REPO_URL="${CONFIG_REPO_URL:?CONFIG_REPO_URL ist nicht gesetzt}" +CONFIG_REPO_BRANCH="${CONFIG_REPO_BRANCH:-main}" +PACKAGES_FILE="$CONFIG_DIR/packages.yaml" +DEST=/data/repo/rpms +TMP_REPO_DIR=/etc/yum.repos.d +LOCKFILE=/tmp/sync.lock +APPSTREAM_GEN=/usr/local/bin/appstream-gen.py -# Where the YAML source list lives. Either point SOURCES_GIT_REPO at a git -# repo (SOURCES_YAML_FILE is the path *inside* that repo), or leave -# SOURCES_GIT_REPO empty and point SOURCES_YAML_FILE at a local file. -SOURCES_GIT_REPO="${SOURCES_GIT_REPO:-}" -SOURCES_GIT_BRANCH="${SOURCES_GIT_BRANCH:-main}" -SOURCES_YAML_FILE="${SOURCES_YAML_FILE:-sources.yaml}" +exec 200>"$LOCKFILE" +if ! flock -n 200; then + echo "[sync] Ein anderer Sync-Lauf ist bereits aktiv, breche ab." + exit 0 +fi -# Local working/output paths. -WORK_DIR="${WORK_DIR:-/var/cache/rpm-mirror-sync}" -REPO_BASE_DIR="${REPO_BASE_DIR:-/srv/repo}" -CLIENT_REPO_DIR="${CLIENT_REPO_DIR:-${REPO_BASE_DIR}/client-repos}" -# The ONE combined repo directory all sources' packages end up in. -PKG_DIR="${REPO_BASE_DIR}/pkgs" +echo "[sync] $(date -Iseconds) Starte Sync-Lauf" -# Base URL under which REPO_BASE_DIR is actually served to clients (web -# server / reverse proxy in front of REPO_BASE_DIR). Used only to generate -# ready-to-use .repo files for clients. -CLIENT_BASE_URL="${CLIENT_BASE_URL:-http://mirror.example.local/repo}" +# --- 1. Whitelist aus Git holen --------------------------------------- +mkdir -p "$CONFIG_DIR" -# repoids of AlmaLinux's own repos on THIS machine, as shown by -# `dnf repolist`. Adjust to match your system if they differ. -# ("Extras for Enterprise" is the "extras" repo in AlmaLinux 10 naming; crb -# is CodeReady Builder / "extras-common" on some setups - check your -# `dnf repolist` output and adjust below.) -ALMA_REPO_IDS="${ALMA_REPO_IDS:-baseos appstream extras crb}" +if [ -n "${GIT_SSH_KEY_PATH:-}" ] && [ -f "${GIT_SSH_KEY_PATH:-}" ]; then + export GIT_SSH_COMMAND="ssh -i ${GIT_SSH_KEY_PATH} -o StrictHostKeyChecking=accept-new" +fi -# The mirror's own signing identity. A key is generated once (on first run) -# under GPG_HOME and reused on every subsequent run - make sure GPG_HOME -# points at persistent storage (a volume), or you'll get a new key (and -# therefore a trust-breaking change for clients) on every restart. -GPG_HOME="${GPG_HOME:-${WORK_DIR}/gnupg}" -GPG_KEY_NAME="${GPG_KEY_NAME:-RPM Mirror}" -GPG_KEY_EMAIL="${GPG_KEY_EMAIL:-rpm-mirror@example.local}" -MIRROR_GPG_KEY_FILE="${REPO_BASE_DIR}/RPM-GPG-KEY-mirror" +if [ -d "$CONFIG_DIR/.git" ]; then + git -C "$CONFIG_DIR" fetch --depth=1 origin "$CONFIG_REPO_BRANCH" + git -C "$CONFIG_DIR" reset --hard "origin/$CONFIG_REPO_BRANCH" +else + git clone --depth=1 --branch "$CONFIG_REPO_BRANCH" "$CONFIG_REPO_URL" "$CONFIG_DIR" +fi -LOG_TAG="rpm-mirror-sync" +if [ ! -f "$PACKAGES_FILE" ]; then + echo "[sync] FEHLER: $PACKAGES_FILE nicht gefunden." + exit 1 +fi -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- +mkdir -p "$DEST" -log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${WORK_DIR}/sync.log" >&2; } -die() { log "ERROR: $*"; exit 1; } +# --- 2. Pro Quelle: temporaeres Repo aktivieren, Pakete gezielt ziehen -- +source_count=$(yq -o=json '.sources | length' "$PACKAGES_FILE") -require_cmd() { - command -v "$1" >/dev/null 2>&1 || die "required command '$1' not found (install it first)" -} +for i in $(seq 0 $((source_count - 1))); do + src=$(yq -o=json ".sources[$i]" "$PACKAGES_FILE") + name=$(echo "$src" | jq -r '.name') + baseurl=$(echo "$src" | jq -r '.baseurl') + gpgkey=$(echo "$src" | jq -r '.gpgkey // empty') -cleanup() { - # Remove any temp source .repo files we created, even on failure. - if [[ -n "${TEMP_REPO_FILES:-}" ]]; then - for f in "${TEMP_REPO_FILES[@]}"; do - rm -f "$f" - done - fi -} -trap cleanup EXIT + # Paketeintraege koennen Strings ODER Mappings mit Metadaten sein. + mapfile -t packages < <( + echo "$src" | jq -r '.packages[] | if type == "string" then . else .name end' + ) -SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -HELPER="${SCRIPT_DIR}/sync_helpers.py" + if [ "${#packages[@]}" -eq 0 ]; then + echo "[sync] Quelle '$name': keine Pakete gelistet, ueberspringe." + continue + fi -# --------------------------------------------------------------------------- -# Preconditions -# --------------------------------------------------------------------------- + echo "[sync] Quelle '$name': ${#packages[@]} Paket(e) -> $baseurl" -[[ $EUID -eq 0 ]] || die "must run as root (writes to /etc/yum.repos.d and ${REPO_BASE_DIR})" - -for c in dnf createrepo_c modifyrepo_c rpm rpmsign rpmkeys gpg python3 curl gzip; do - require_cmd "$c" -done -[[ -f "$HELPER" ]] || die "sync_helpers.py not found next to sync.sh (expected: $HELPER)" -python3 -c "import yaml" 2>/dev/null || die "python3-pyyaml not installed (dnf install -y python3-pyyaml)" - -mkdir -p "$WORK_DIR" "$REPO_BASE_DIR" "$CLIENT_REPO_DIR" "$PKG_DIR" - -# --------------------------------------------------------------------------- -# Mirror signing key: generate once, reuse forever, export the public part -# --------------------------------------------------------------------------- - -MIRROR_GPG_KEY_ID="" -REVOCATION_CERT_FILE="${GPG_HOME}/revocation-cert.asc" - -init_mirror_gpg_key() { - mkdir -p "$GPG_HOME" - chmod 700 "$GPG_HOME" - - if [[ -z "$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons 2>/dev/null)" ]]; then - log "No mirror signing key found under ${GPG_HOME} - generating one (one-time)..." - gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \ - --quick-generate-key "${GPG_KEY_NAME} <${GPG_KEY_EMAIL}>" rsa4096 sign never \ - || die "failed to generate mirror GPG signing key" - - # GnuPG >= 2.1 automatically writes a revocation certificate under - # openpgp-revocs.d/.rev as part of key generation itself - # (visible in the gpg output above: "revocation certificate stored - # as ..."). We just need to copy it to a known, stable path - no - # need to (fragile-ly) drive gpg's interactive --gen-revoke menu - # ourselves, which behaves differently across gpg versions. - local auto_revoc - auto_revoc="$(find "${GPG_HOME}/openpgp-revocs.d" -maxdepth 1 -name '*.rev' 2>/dev/null | head -n1)" - [[ -n "$auto_revoc" && -s "$auto_revoc" ]] \ - || die "expected GnuPG to auto-generate a revocation certificate under ${GPG_HOME}/openpgp-revocs.d/ but none was found" - - cp "$auto_revoc" "$REVOCATION_CERT_FILE" - chmod 600 "$REVOCATION_CERT_FILE" - - log "!!! Revocation certificate written to: ${REVOCATION_CERT_FILE}" - log "!!! Copy it to secure, OFFLINE storage now, e.g.:" - log "!!! docker compose cp syncer:${REVOCATION_CERT_FILE} ./mirror-key-revocation-cert.asc" - log "!!! It is the only way to invalidate this key if it is ever compromised -" - log "!!! it never expires, so it needs the same protection as the private key." - fi - - MIRROR_GPG_KEY_ID="$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons \ - | awk -F: '/^sec:/ {print $5; exit}')" - [[ -n "$MIRROR_GPG_KEY_ID" ]] || die "could not determine mirror GPG key id" - - # rpmsign/rpm --addsign etc. read the signing identity from macros. - cat > "${HOME:-/root}/.rpmmacros" < "$MIRROR_GPG_KEY_FILE" - - log "Mirror signing key ready: ${MIRROR_GPG_KEY_ID} (public key: ${MIRROR_GPG_KEY_FILE})" -} - -init_mirror_gpg_key - - - -# --------------------------------------------------------------------------- -# Fetch the YAML source list -# --------------------------------------------------------------------------- - -resolve_yaml_path() { - if [[ -n "$SOURCES_GIT_REPO" ]]; then - if [[ "$SOURCES_YAML_FILE" == /* ]]; then - die "SOURCES_GIT_REPO is set, but SOURCES_YAML_FILE ('${SOURCES_YAML_FILE}') looks like an absolute path. When pulling the source list from git, SOURCES_YAML_FILE must be a path RELATIVE to that repo's root (e.g. 'sources.yaml' or 'config/sources.yaml') - not the '/config/...' path used for the local bind-mount option. Fix SOURCES_YAML_FILE in .env." - fi - local clone_dir="${WORK_DIR}/sources-repo" - if [[ -d "${clone_dir}/.git" ]]; then - log "Updating source list git repo..." - # >&2 is critical here: resolve_yaml_path's stdout is captured - # by the caller via $(...) to get the final path. "git reset - # --hard" prints "HEAD is now at " to STDOUT by - # default - without this redirect that text gets appended to - # YAML_PATH, producing a garbage multi-line "path" that then - # fails the next -f check with a confusing error. - git -C "$clone_dir" fetch --depth 1 origin "$SOURCES_GIT_BRANCH" >&2 - git -C "$clone_dir" reset --hard "origin/${SOURCES_GIT_BRANCH}" >&2 - else - log "Cloning source list git repo..." - rm -rf "$clone_dir" - git clone --depth 1 --branch "$SOURCES_GIT_BRANCH" "$SOURCES_GIT_REPO" "$clone_dir" >&2 - fi - echo "${clone_dir}/${SOURCES_YAML_FILE}" + repofile="$TMP_REPO_DIR/tmp-${name}.repo" + { + echo "[tmp-${name}]" + echo "name=${name}" + echo "baseurl=${baseurl}" + echo "enabled=1" + if [ -n "$gpgkey" ]; then + echo "gpgcheck=1" + echo "gpgkey=${gpgkey}" else - echo "$SOURCES_YAML_FILE" + echo "gpgcheck=0" fi -} + } > "$repofile" -YAML_PATH="$(resolve_yaml_path)" -[[ -f "$YAML_PATH" ]] || die "source YAML not found: $YAML_PATH" + # Bei Fehlschlag (z.B. Paket nicht gefunden) diese Quelle nicht den + # gesamten Lauf abbrechen lassen, aber laut melden. + if ! dnf download --resolve --alldeps \ + --disablerepo='*' --enablerepo="tmp-${name}" \ + --destdir="$DEST" "${packages[@]}"; then + echo "[sync] WARNUNG: Download fuer Quelle '$name' teilweise fehlgeschlagen." + fi -SOURCES_JSON="$(python3 "$HELPER" parse-yaml "$YAML_PATH")" || die "failed to parse $YAML_PATH" -SOURCE_COUNT="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$SOURCES_JSON")" -log "Loaded $SOURCE_COUNT source(s) from $YAML_PATH" - -# --------------------------------------------------------------------------- -# Per-source sync -# --------------------------------------------------------------------------- - -declare -a TEMP_REPO_FILES=() - -# Build --enablerepo flags for the AlmaLinux repos as a bash array once. -declare -a ALMA_ENABLE_FLAGS=() -for r in $ALMA_REPO_IDS; do - ALMA_ENABLE_FLAGS+=( "--enablerepo=${r}" ) + rm -f "$repofile" done -sync_one_source() { - local name="$1" baseurl="$2" gpgkey="$3" packages_json="$4" metadata_only="${5:-false}" +# --- 3. Alte Paketversionen aufraeumen ----------------------------------- +echo "[sync] Raeume veraltete Paketversionen auf" +dnf repomanage --old "$DEST" | xargs -r rm -f || \ + echo "[sync] WARNUNG: repomanage fehlgeschlagen, ueberspringe Aufraeumen." - local repoid="mirror-src-${name}" - local repo_file="/etc/yum.repos.d/${repoid}.repo" - local tmp_dir="${WORK_DIR}/${name}/download" - local list_dir="${WORK_DIR}/${name}" - # This source's contribution to the combined AppStream metadata: one - # "namesummary" line per explicitly requested package. Collected - # here, merged across all sources, and turned into ONE appstream.xml - # after the whole source loop finishes. - local entries_tsv="${list_dir}/appstream-entries.tsv" +# --- 4. RPM-Metadaten neu erzeugen --------------------------------------- +echo "[sync] Aktualisiere Repo-Metadaten" +createrepo_c --update "$DEST" - log "=== Syncing source '${name}' (metadata_only=${metadata_only}) ===" - mkdir -p "$tmp_dir" "$list_dir" - : > "$entries_tsv" +# --- 5. AppStream-Katalog erzeugen und einhaengen ------------------------- +# Ohne diesen Schritt sind die Pakete zwar per dnf installierbar, tauchen +# aber weder in Discover noch in GNOME Software auf. +echo "[sync] Erzeuge AppStream-Katalog" - # readarray of requested top-level package names for this source. - mapfile -t packages < <(python3 -c ' -import json,sys -for p in json.load(sys.stdin): - print(p) -' <<<"$packages_json") - [[ ${#packages[@]} -gt 0 ]] || die "source '${name}' has no packages" +ASWORK=$(mktemp -d) +trap 'rm -rf "$ASWORK"' EXIT - # --- 1. Register the source repo temporarily ------------------------- - { - echo "[${repoid}]" - echo "name=Mirror source: ${name}" - echo "baseurl=${baseurl}" - echo "enabled=1" - if [[ -n "$gpgkey" ]]; then - echo "gpgcheck=1" - echo "gpgkey=${gpgkey}" - else - echo "gpgcheck=0" - fi - } > "$repo_file" - TEMP_REPO_FILES+=( "$repo_file" ) - dnf clean expire-cache --disablerepo='*' --enablerepo="$repoid" >/dev/null 2>&1 || true +# Welche Pakete liegen tatsaechlich im Repo? Verhindert, dass Discover +# Eintraege anzeigt, deren RPM gar nicht da ist. +if compgen -G "$DEST"/*.rpm > /dev/null; then + rpm -qp --qf '%{NAME}\n' "$DEST"/*.rpm 2>/dev/null \ + | sort -u > "$ASWORK/vorhanden.txt" +else + echo "[sync] WARNUNG: Keine RPMs in $DEST gefunden." + : > "$ASWORK/vorhanden.txt" +fi - # --- metadata_only sources: curate Discover/GNOME Software visibility - # WITHOUT mirroring any RPM content. Use this when baseurl already - # points at a repo your clients have enabled directly (e.g. AlmaLinux's - # own AppStream repo, which ships dotnet-* itself on AlmaLinux 10) - the - # actual package install is then resolved by dnf from that repo as - # normal; this source only contributes AppStream "generic component" - # data for the packages listed to the combined repo built after the - # source loop, so ONLY those show up as installable in Discover instead - # of AlmaLinux's entire (huge) AppStream catalog. - # PRECONDITION: clients must have the repo(s) that actually provide - # these packages enabled already (true for any stock AlmaLinux install - - # baseos/appstream/extras/crb are on by default). - if [[ "$metadata_only" == "true" ]]; then - for pkg in "${packages[@]}"; do - local summary - summary="$(dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \ - --qf '%{summary}' "$pkg" 2>/dev/null | head -n1)" - printf '%s\t%s\n' "$pkg" "$summary" >> "$entries_tsv" - done +python3 "$APPSTREAM_GEN" "$PACKAGES_FILE" \ + --output "$ASWORK/appstream.xml.gz" \ + --available-packages "$ASWORK/vorhanden.txt" - log "Source '${name}': metadata_only - queued AppStream visibility for ${#packages[@]} package(s), no RPMs mirrored" +# Optionale Validierung, falls appstreamcli im Image vorhanden ist. +if command -v appstreamcli > /dev/null 2>&1; then + if ! appstreamcli validate --no-net "$ASWORK/appstream.xml.gz"; then + echo "[sync] WARNUNG: AppStream-Katalog hat Validierungshinweise." + fi +fi - rm -f "$repo_file" - TEMP_REPO_FILES=( "${TEMP_REPO_FILES[@]/$repo_file}" ) - return - fi +# modifyrepo_c MUSS nach createrepo_c laufen, da createrepo_c die +# repomd.xml komplett neu schreibt. +modifyrepo_c --mdtype=appstream "$ASWORK/appstream.xml.gz" "$DEST/repodata/" - # --- 2. List everything the source repo itself provides -------------- - # (used later to decide "does this RPM genuinely come from the source, - # or did AlmaLinux already have it".) - local source_nevra_list="${list_dir}/source-nevra.txt" - dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \ - --qf '%{name}-%{version}-%{release}.%{arch}' -a \ - > "$source_nevra_list" \ - || die "failed to query source repo '${name}' (check baseurl/gpgkey)" - [[ -s "$source_nevra_list" ]] || die "source repo '${name}' returned no packages - check baseurl" - - # --- 3. Resolve full dependency closure for the requested packages --- - # Enable the source repo AND the local AlmaLinux repos so dependency - # resolution succeeds even for deps AlmaLinux normally provides - - # we filter those back out in step 4. - rm -rf "$tmp_dir"; mkdir -p "$tmp_dir" - log "Resolving + downloading dependency closure for: ${packages[*]}" - dnf download -y --resolve --alldeps \ - --destdir="$tmp_dir" \ - --disablerepo='*' \ - --enablerepo="$repoid" \ - "${ALMA_ENABLE_FLAGS[@]}" \ - "${packages[@]}" \ - || die "dnf download failed for source '${name}'" - - # --- 4. Drop everything AlmaLinux already provides, verify + re-sign -- - # the rest with the mirror's own key ----------------------------------- - local verify_dbpath="${list_dir}/verify-rpmdb" - if [[ -n "$gpgkey" ]]; then - local source_gpgkey_file="${list_dir}/source-gpgkey.asc" - curl -fsSL "$gpgkey" -o "$source_gpgkey_file" \ - || die "could not fetch gpgkey for source '${name}' from ${gpgkey}" - rm -rf "$verify_dbpath"; mkdir -p "$verify_dbpath" - rpm --dbpath "$verify_dbpath" --initdb - rpm --dbpath "$verify_dbpath" --import "$source_gpgkey_file" \ - || die "could not import gpgkey for source '${name}'" - else - log " WARNING: source '${name}' has no gpgkey configured - skipping upstream signature verification" - fi - - local kept=0 dropped=0 kept_requested=0 kept_dependency=0 - for f in "$tmp_dir"/*.rpm; do - [[ -e "$f" ]] || continue - local base; base="$(basename "$f" .rpm)" - - if ! grep -qxF "$base" "$source_nevra_list"; then - log " dropping (provided by AlmaLinux): $base" - rm -f "$f" - dropped=$((dropped + 1)) - continue - fi - - if [[ -n "$gpgkey" ]]; then - rpmkeys --dbpath "$verify_dbpath" --checksig "$f" >/dev/null 2>&1 \ - || die "signature verification FAILED for '${base}' from source '${name}' - refusing to mirror it (tampered download or wrong gpgkey configured)" - fi - - # Strip the upstream signature and sign with the mirror's own key, - # so clients only ever need to trust MIRROR_GPG_KEY_FILE. - rpmsign --resign "$f" >/dev/null \ - || die "failed to re-sign '${base}' with the mirror key" - - # Label each kept package as explicitly "requested" (listed under - # packages: in the YAML) or a pulled-in "dependency" - packages the - # requested ones need but AlmaLinux doesn't provide, so they have to - # be mirrored too or `dnf install ` breaks on clients. - # This is purely informational (log output); both kinds are kept. - local pkg_name tag - pkg_name="$(rpm -qp --qf '%{NAME}' "$f" 2>/dev/null)" - tag="dependency" - for req in "${packages[@]}"; do - if [[ "$pkg_name" == "$req" ]]; then - tag="requested" - break - fi - done - if [[ "$tag" == "requested" ]]; then - kept_requested=$((kept_requested + 1)) - local summary - summary="$(rpm -qp --qf '%{SUMMARY}' "$f" 2>/dev/null)" - printf '%s\t%s\n' "$pkg_name" "$summary" >> "$entries_tsv" - else - kept_dependency=$((kept_dependency + 1)) - fi - log " keeping (${tag}): $base" - - mv "$f" "${PKG_DIR}/" - kept=$((kept + 1)) - done - log "Source '${name}': kept ${kept} package(s) - ${kept_requested} explicitly requested + ${kept_dependency} pulled-in third-party dependencies (re-signed with mirror key), dropped ${dropped} AlmaLinux-provided package(s)" - [[ $kept -gt 0 ]] || die "nothing kept for source '${name}' - check baseurl/package names" - - # --- cleanup temp repo file for this source --------------------------- - rm -f "$repo_file" - TEMP_REPO_FILES=( "${TEMP_REPO_FILES[@]/$repo_file}" ) -} - -# Iterate over sources (JSON array -> one JSON object per line via jq-less python) -while IFS= read -r src_json; do - name="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["name"])' "$src_json")" - baseurl="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["baseurl"])' "$src_json")" - gpgkey="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1]).get("gpgkey",""))' "$src_json")" - pkgs_json="$(python3 -c 'import json,sys;print(json.dumps(json.loads(sys.argv[1])["packages"]))' "$src_json")" - metadata_only="$(python3 -c 'import json,sys;print(str(bool(json.loads(sys.argv[1]).get("metadata_only", False))).lower())' "$src_json")" - sync_one_source "$name" "$baseurl" "$gpgkey" "$pkgs_json" "$metadata_only" -done < <(python3 -c ' -import json,sys -for s in json.load(sys.stdin): - print(json.dumps(s)) -' <<<"$SOURCES_JSON") - -# --------------------------------------------------------------------------- -# Build the ONE combined repo from everything all sources contributed -# --------------------------------------------------------------------------- - -createrepo_c --update "$PKG_DIR" >/dev/null - -# Merge every source's appstream-entries.tsv (written during its own -# processing above) into one de-duplicated list, then build ONE AppStream -# metadata blob for the single shared repo. Positional arg (not stdin) for -# the same reason noted elsewhere in this script: "python3 -" already -# consumes stdin to read the script itself. -COMBINED_ENTRIES_JSON="$(python3 - "$WORK_DIR" <<'PYEOF' -import glob, json, os, sys - -work_dir = sys.argv[1] -seen = {} -for f in sorted(glob.glob(os.path.join(work_dir, "*", "appstream-entries.tsv"))): - try: - with open(f, encoding="utf-8") as fh: - for line in fh: - line = line.rstrip("\n") - if not line: - continue - parts = line.split("\t", 1) - name = parts[0] - summary = parts[1] if len(parts) > 1 else "" - if not name: - continue - entry = {"name": name, "summary": summary} - if name not in seen: - seen[name] = entry - elif seen[name] != entry: - print( - f"WARNING: package '{name}' listed by multiple sources " - "with differing AppStream data - keeping the first one seen", - file=sys.stderr, - ) - except OSError: - continue -print(json.dumps(list(seen.values()))) -PYEOF -)" -[[ -n "$COMBINED_ENTRIES_JSON" ]] || die "failed to compute combined AppStream entries" - -APPSTREAM_XML="${WORK_DIR}/appstream.xml" -python3 "$HELPER" gen-appstream "rpm-mirror" "$COMBINED_ENTRIES_JSON" > "$APPSTREAM_XML" -gzip -fk "$APPSTREAM_XML" -modifyrepo_c --mdtype=appstream "${APPSTREAM_XML}.gz" "${PKG_DIR}/repodata" >/dev/null - -# Detached-sign repomd.xml itself with the mirror key, so clients can verify -# the METADATA (which packages/versions exist, their checksums) hasn't been -# tampered with - not just each individual RPM. Must happen last, after -# createrepo_c/modifyrepo_c are done touching repomd.xml (they update its -# checksums), otherwise the signature would be over stale content. -gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \ - --detach-sign --armor \ - --output "${PKG_DIR}/repodata/repomd.xml.asc" \ - "${PKG_DIR}/repodata/repomd.xml" \ - || die "failed to sign repomd.xml with the mirror key" - -# --- The ONE client-facing .repo file - this is all clients ever need ----- -cat > "${CLIENT_REPO_DIR}/mirror.repo" <