#!/usr/bin/env bash # # sync.sh - Mirror selected packages from third-party RPM repos into ONE # combined local repo consumable by AlmaLinux clients (dnf / # PackageKit / GNOME Software / Discover) - WITHOUT pulling in or # exposing packages that AlmaLinux itself already ships # (AppStream, BaseOS, Extras, CRB/"Extras for Enterprise"). # # ALL sources defined in the YAML end up in a SINGLE repo # (REPO_BASE_DIR/pkgs/) with a SINGLE client-facing .repo file # (CLIENT_REPO_DIR/mirror.repo) - clients only ever add one repo, no matter # how many sources you configure. # # What it does, per entry in the YAML source list: # 1. Registers the source repo temporarily and resolves the full # dependency closure of the requested packages, using the source repo # AND the local AlmaLinux repos (needed so resolution succeeds, e.g. # glibc-style deps that only exist in AlmaLinux). # 2. Throws away every RPM in that closure that AlmaLinux itself provides. # Only RPMs that genuinely only exist in the third-party repo are kept. # 3. Verifies each kept RPM against the source's own gpgkey (if given in # the YAML), then STRIPS the original signature and RE-SIGNS it with # the mirror's own GPG key. Clients therefore only ever need to trust # one key (this mirror's), never the individual upstream repos' keys. # 4. Moves the signed RPMs into the single shared repo directory # (REPO_BASE_DIR/pkgs/). # Once all sources have been processed, the script rebuilds repo metadata # for that ONE directory with createrepo_c and generates AppStream "generic # component" metadata for ONLY the packages explicitly listed under # "packages:" across all sources (never for pulled-in dependencies), # injecting it once into that same repo. Only those show up as installable # items in GNOME Software/Discover. # # Set "metadata_only: true" on a source to skip steps 2-4 entirely for that # source: no RPMs are downloaded/mirrored, only curated AppStream visibility # is contributed to the combined repo. Use this when baseurl already points # at a repo your clients have enabled directly (e.g. AlmaLinux's own # AppStream repo, which on AlmaLinux 10 ships dotnet-* itself) - you just # want to narrow down what shows up in Discover without duplicating RPM # content clients can already get natively. # # NOTE ON NAME COLLISIONS: if two different sources define an RPM with the # exact same NEVRA (name-version-release.arch), the later one silently wins # in the shared directory (harmless - it's the same package). If two # sources define the SAME PACKAGE NAME with genuinely different content, # that's not supported here - keep package names distinct across sources. # # IMPORTANT LIMITATION: # Discover/GNOME Software hide plain "generic" AppStream components # entirely unless the user enables "technische Pakete anzeigen"/"Show # technical packages" - so every package in packages.yaml needs either # "binary: " (-> console-application, visible by default, # findable via search) or "type: desktop-application" + "desktop_id:" # (needs a real .desktop file, also shows up on the curated "Explore" # front page and in categories - console-application does not). The # AppStream generator (sync_helpers.py) warns loudly whenever a package # would fall back to "generic" so this doesn't go unnoticed. This is # Discover/GNOME Software/appstreamcli behaviour and cannot be fixed by # repo metadata alone. # # SECURITY NOTE ON RE-SIGNING: # The mirror's private signing key is generated once (unattended, without # a passphrase - standard practice for automated repo signing) and kept # under GPG_HOME. Only the exported PUBLIC key ends up in REPO_BASE_DIR # where clients can fetch it. GPG_HOME itself must never be exposed via # nginx/the webserver and should live on storage only the sync process can # read. Anyone who can read GPG_HOME can sign packages as your mirror - # treat it like any other private key material. # # A revocation certificate is generated alongside the key on first run # (GPG_HOME/revocation-cert.asc) - since the key never expires, this # certificate is the ONLY way to invalidate it later if it's ever # compromised. Copy it to secure, offline storage right after first # startup; see the log output / README for the exact command. # # Requires: dnf, dnf-plugins-core (for "dnf download"), createrepo_c, # modifyrepo_c, rpm-sign (rpmsign/rpmkeys), gnupg2, python3, # python3-pyyaml, git (if fetching the source list from git), curl. # # Run as root (needs to write a temporary .repo file to /etc/yum.repos.d). set -euo pipefail # --------------------------------------------------------------------------- # Configuration (override via environment, e.g. in the systemd unit/cron job) # --------------------------------------------------------------------------- # Where the YAML source list lives. Either point SOURCES_GIT_REPO at a git # repo (SOURCES_YAML_FILE is the path *inside* that repo), or leave # SOURCES_GIT_REPO empty and point SOURCES_YAML_FILE at a local file. SOURCES_GIT_REPO="${SOURCES_GIT_REPO:-}" SOURCES_GIT_BRANCH="${SOURCES_GIT_BRANCH:-main}" SOURCES_YAML_FILE="${SOURCES_YAML_FILE:-sources.yaml}" # Local working/output paths. WORK_DIR="${WORK_DIR:-/var/cache/rpm-mirror-sync}" REPO_BASE_DIR="${REPO_BASE_DIR:-/srv/repo}" CLIENT_REPO_DIR="${CLIENT_REPO_DIR:-${REPO_BASE_DIR}/client-repos}" # The ONE combined repo directory all sources' packages end up in. PKG_DIR="${REPO_BASE_DIR}/pkgs" # Base URL under which REPO_BASE_DIR is actually served to clients (web # server / reverse proxy in front of REPO_BASE_DIR). Used only to generate # ready-to-use .repo files for clients. CLIENT_BASE_URL="${CLIENT_BASE_URL:-http://mirror.example.local/repo}" # repoids of AlmaLinux's own repos on THIS machine, as shown by # `dnf repolist`. Adjust to match your system if they differ. # ("Extras for Enterprise" is the "extras" repo in AlmaLinux 10 naming; crb # is CodeReady Builder / "extras-common" on some setups - check your # `dnf repolist` output and adjust below.) ALMA_REPO_IDS="${ALMA_REPO_IDS:-baseos appstream extras crb}" # Origin of the generated AppStream catalog. Must be unique - if it collides # with a distribution's own origin (e.g. "almalinux"), the icon/metadata # caches on the clients overwrite each other. Can also be set via # "appstream: origin:" in the YAML; this variable wins. APPSTREAM_ORIGIN="${APPSTREAM_ORIGIN:-rpm-mirror}" # The mirror's own signing identity. A key is generated once (on first run) # under GPG_HOME and reused on every subsequent run - make sure GPG_HOME # points at persistent storage (a volume), or you'll get a new key (and # therefore a trust-breaking change for clients) on every restart. GPG_HOME="${GPG_HOME:-${WORK_DIR}/gnupg}" GPG_KEY_NAME="${GPG_KEY_NAME:-RPM Mirror}" GPG_KEY_EMAIL="${GPG_KEY_EMAIL:-rpm-mirror@example.local}" MIRROR_GPG_KEY_FILE="${REPO_BASE_DIR}/RPM-GPG-KEY-mirror" LOG_TAG="rpm-mirror-sync" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${WORK_DIR}/sync.log" >&2; } die() { log "ERROR: $*"; exit 1; } require_cmd() { command -v "$1" >/dev/null 2>&1 || die "required command '$1' not found (install it first)" } cleanup() { # Remove any temp source .repo files we created, even on failure. if [[ -n "${TEMP_REPO_FILES:-}" ]]; then for f in "${TEMP_REPO_FILES[@]}"; do [[ -n "$f" ]] && rm -f "$f" done fi } # Remove one entry from TEMP_REPO_FILES. # NOTE: "${ARRAY[@]/pattern}" does SUBSTRING REPLACEMENT, so the element is # left behind as an empty string rather than removed - which then makes # cleanup() call `rm -f ""`. Rebuild the array instead. forget_repo_file() { local drop="$1" keep=() f for f in "${TEMP_REPO_FILES[@]}"; do [[ "$f" == "$drop" ]] || keep+=( "$f" ) done TEMP_REPO_FILES=( "${keep[@]}" ) } trap cleanup EXIT SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" HELPER="${SCRIPT_DIR}/sync_helpers.py" # --------------------------------------------------------------------------- # Preconditions # --------------------------------------------------------------------------- [[ $EUID -eq 0 ]] || die "must run as root (writes to /etc/yum.repos.d and ${REPO_BASE_DIR})" for c in dnf createrepo_c modifyrepo_c rpm rpmsign rpmkeys gpg python3 curl gzip; do require_cmd "$c" done [[ -f "$HELPER" ]] || die "sync_helpers.py not found next to sync.sh (expected: $HELPER)" python3 -c "import yaml" 2>/dev/null || die "python3-pyyaml not installed (dnf install -y python3-pyyaml)" mkdir -p "$WORK_DIR" "$REPO_BASE_DIR" "$CLIENT_REPO_DIR" "$PKG_DIR" # --------------------------------------------------------------------------- # Mirror signing key: generate once, reuse forever, export the public part # --------------------------------------------------------------------------- MIRROR_GPG_KEY_ID="" REVOCATION_CERT_FILE="${GPG_HOME}/revocation-cert.asc" init_mirror_gpg_key() { mkdir -p "$GPG_HOME" chmod 700 "$GPG_HOME" if [[ -z "$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons 2>/dev/null)" ]]; then log "No mirror signing key found under ${GPG_HOME} - generating one (one-time)..." gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \ --quick-generate-key "${GPG_KEY_NAME} <${GPG_KEY_EMAIL}>" rsa4096 sign never \ || die "failed to generate mirror GPG signing key" # GnuPG >= 2.1 automatically writes a revocation certificate under # openpgp-revocs.d/.rev as part of key generation itself # (visible in the gpg output above: "revocation certificate stored # as ..."). We just need to copy it to a known, stable path - no # need to (fragile-ly) drive gpg's interactive --gen-revoke menu # ourselves, which behaves differently across gpg versions. local auto_revoc auto_revoc="$(find "${GPG_HOME}/openpgp-revocs.d" -maxdepth 1 -name '*.rev' 2>/dev/null | head -n1)" [[ -n "$auto_revoc" && -s "$auto_revoc" ]] \ || die "expected GnuPG to auto-generate a revocation certificate under ${GPG_HOME}/openpgp-revocs.d/ but none was found" cp "$auto_revoc" "$REVOCATION_CERT_FILE" chmod 600 "$REVOCATION_CERT_FILE" log "!!! Revocation certificate written to: ${REVOCATION_CERT_FILE}" log "!!! Copy it to secure, OFFLINE storage now, e.g.:" log "!!! docker compose cp syncer:${REVOCATION_CERT_FILE} ./mirror-key-revocation-cert.asc" log "!!! It is the only way to invalidate this key if it is ever compromised -" log "!!! it never expires, so it needs the same protection as the private key." fi MIRROR_GPG_KEY_ID="$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons \ | awk -F: '/^sec:/ {print $5; exit}')" [[ -n "$MIRROR_GPG_KEY_ID" ]] || die "could not determine mirror GPG key id" # rpmsign/rpm --addsign etc. read the signing identity from macros. cat > "${HOME:-/root}/.rpmmacros" < "$MIRROR_GPG_KEY_FILE" log "Mirror signing key ready: ${MIRROR_GPG_KEY_ID} (public key: ${MIRROR_GPG_KEY_FILE})" } init_mirror_gpg_key # --------------------------------------------------------------------------- # Fetch the YAML source list # --------------------------------------------------------------------------- resolve_yaml_path() { if [[ -n "$SOURCES_GIT_REPO" ]]; then if [[ "$SOURCES_YAML_FILE" == /* ]]; then die "SOURCES_GIT_REPO is set, but SOURCES_YAML_FILE ('${SOURCES_YAML_FILE}') looks like an absolute path. When pulling the source list from git, SOURCES_YAML_FILE must be a path RELATIVE to that repo's root (e.g. 'sources.yaml' or 'config/sources.yaml') - not the '/config/...' path used for the local bind-mount option. Fix SOURCES_YAML_FILE in .env." fi local clone_dir="${WORK_DIR}/sources-repo" if [[ -d "${clone_dir}/.git" ]]; then log "Updating source list git repo..." git -C "$clone_dir" fetch --depth 1 origin "$SOURCES_GIT_BRANCH" git -C "$clone_dir" reset --hard "origin/${SOURCES_GIT_BRANCH}" else log "Cloning source list git repo..." rm -rf "$clone_dir" git clone --depth 1 --branch "$SOURCES_GIT_BRANCH" "$SOURCES_GIT_REPO" "$clone_dir" fi echo "${clone_dir}/${SOURCES_YAML_FILE}" else echo "$SOURCES_YAML_FILE" fi } YAML_PATH="$(resolve_yaml_path)" [[ -f "$YAML_PATH" ]] || die "source YAML not found: $YAML_PATH" SOURCES_JSON="$(python3 "$HELPER" parse-yaml "$YAML_PATH")" || die "failed to parse $YAML_PATH" SOURCE_COUNT="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$SOURCES_JSON")" log "Loaded $SOURCE_COUNT source(s) from $YAML_PATH" # --------------------------------------------------------------------------- # Per-source sync # --------------------------------------------------------------------------- declare -a TEMP_REPO_FILES=() # Build --enablerepo flags for the AlmaLinux repos as a bash array once. declare -a ALMA_ENABLE_FLAGS=() for r in $ALMA_REPO_IDS; do ALMA_ENABLE_FLAGS+=( "--enablerepo=${r}" ) done sync_one_source() { local name="$1" baseurl="$2" gpgkey="$3" packages_json="$4" metadata_only="${5:-false}" local repoid="mirror-src-${name}" local repo_file="/etc/yum.repos.d/${repoid}.repo" local tmp_dir="${WORK_DIR}/${name}/download" local list_dir="${WORK_DIR}/${name}" # This source's contribution to the combined AppStream metadata: one # "namesummary" line per explicitly requested package. Collected # here, merged across all sources, and turned into ONE appstream.xml # after the whole source loop finishes. local entries_tsv="${list_dir}/appstream-entries.tsv" log "=== Syncing source '${name}' (metadata_only=${metadata_only}) ===" mkdir -p "$tmp_dir" "$list_dir" : > "$entries_tsv" # readarray of requested top-level package names for this source. # parse-yaml normalises every entry to an object, so a package written # as a plain string and one written as a mapping with AppStream metadata # both arrive here as {"name": ...}. mapfile -t packages < <(python3 -c ' import json,sys for p in json.load(sys.stdin): print(p["name"]) ' <<<"$packages_json") [[ ${#packages[@]} -gt 0 ]] || die "source '${name}' has no packages" # --- 1. Register the source repo temporarily ------------------------- { echo "[${repoid}]" echo "name=Mirror source: ${name}" echo "baseurl=${baseurl}" echo "enabled=1" if [[ -n "$gpgkey" ]]; then echo "gpgcheck=1" echo "gpgkey=${gpgkey}" else echo "gpgcheck=0" fi } > "$repo_file" TEMP_REPO_FILES+=( "$repo_file" ) dnf clean expire-cache --disablerepo='*' --enablerepo="$repoid" >/dev/null 2>&1 || true # --- metadata_only sources: curate Discover/GNOME Software visibility # WITHOUT mirroring any RPM content. Use this when baseurl already # points at a repo your clients have enabled directly (e.g. AlmaLinux's # own AppStream repo, which ships dotnet-* itself on AlmaLinux 10) - the # actual package install is then resolved by dnf from that repo as # normal; this source only contributes AppStream "generic component" # data for the packages listed to the combined repo built after the # source loop, so ONLY those show up as installable in Discover instead # of AlmaLinux's entire (huge) AppStream catalog. # PRECONDITION: clients must have the repo(s) that actually provide # these packages enabled already (true for any stock AlmaLinux install - # baseos/appstream/extras/crb are on by default). if [[ "$metadata_only" == "true" ]]; then for pkg in "${packages[@]}"; do local summary summary="$(dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \ --qf '%{summary}' "$pkg" 2>/dev/null | head -n1)" printf '%s\t%s\n' "$pkg" "$summary" >> "$entries_tsv" done log "Source '${name}': metadata_only - queued AppStream visibility for ${#packages[@]} package(s), no RPMs mirrored" rm -f "$repo_file" forget_repo_file "$repo_file" return fi # --- 2. List everything the source repo itself provides -------------- # (used later to decide "does this RPM genuinely come from the source, # or did AlmaLinux already have it".) local source_nevra_list="${list_dir}/source-nevra.txt" dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \ --qf '%{name}-%{version}-%{release}.%{arch}' -a \ > "$source_nevra_list" \ || die "failed to query source repo '${name}' (check baseurl/gpgkey)" [[ -s "$source_nevra_list" ]] || die "source repo '${name}' returned no packages - check baseurl" # --- 3. Resolve full dependency closure for the requested packages --- # Enable the source repo AND the local AlmaLinux repos so dependency # resolution succeeds even for deps AlmaLinux normally provides - # we filter those back out in step 4. rm -rf "$tmp_dir"; mkdir -p "$tmp_dir" log "Resolving + downloading dependency closure for: ${packages[*]}" dnf download -y --resolve --alldeps \ --destdir="$tmp_dir" \ --disablerepo='*' \ --enablerepo="$repoid" \ "${ALMA_ENABLE_FLAGS[@]}" \ "${packages[@]}" \ || die "dnf download failed for source '${name}'" # --- 4. Drop everything AlmaLinux already provides, verify + re-sign -- # the rest with the mirror's own key ----------------------------------- local verify_dbpath="${list_dir}/verify-rpmdb" if [[ -n "$gpgkey" ]]; then local source_gpgkey_file="${list_dir}/source-gpgkey.asc" curl -fsSL "$gpgkey" -o "$source_gpgkey_file" \ || die "could not fetch gpgkey for source '${name}' from ${gpgkey}" rm -rf "$verify_dbpath"; mkdir -p "$verify_dbpath" rpm --dbpath "$verify_dbpath" --initdb rpm --dbpath "$verify_dbpath" --import "$source_gpgkey_file" \ || die "could not import gpgkey for source '${name}'" else log " WARNING: source '${name}' has no gpgkey configured - skipping upstream signature verification" fi local kept=0 dropped=0 kept_requested=0 kept_dependency=0 for f in "$tmp_dir"/*.rpm; do [[ -e "$f" ]] || continue local base; base="$(basename "$f" .rpm)" if ! grep -qxF "$base" "$source_nevra_list"; then log " dropping (provided by AlmaLinux): $base" rm -f "$f" dropped=$((dropped + 1)) continue fi if [[ -n "$gpgkey" ]]; then rpmkeys --dbpath "$verify_dbpath" --checksig "$f" >/dev/null 2>&1 \ || die "signature verification FAILED for '${base}' from source '${name}' - refusing to mirror it (tampered download or wrong gpgkey configured)" fi # Strip the upstream signature and sign with the mirror's own key, # so clients only ever need to trust MIRROR_GPG_KEY_FILE. rpmsign --resign "$f" >/dev/null \ || die "failed to re-sign '${base}' with the mirror key" # Label each kept package as explicitly "requested" (listed under # packages: in the YAML) or a pulled-in "dependency" - packages the # requested ones need but AlmaLinux doesn't provide, so they have to # be mirrored too or `dnf install ` breaks on clients. # This is purely informational (log output); both kinds are kept. local pkg_name tag pkg_name="$(rpm -qp --qf '%{NAME}' "$f" 2>/dev/null)" tag="dependency" for req in "${packages[@]}"; do if [[ "$pkg_name" == "$req" ]]; then tag="requested" break fi done if [[ "$tag" == "requested" ]]; then kept_requested=$((kept_requested + 1)) local summary summary="$(rpm -qp --qf '%{SUMMARY}' "$f" 2>/dev/null)" printf '%s\t%s\n' "$pkg_name" "$summary" >> "$entries_tsv" else kept_dependency=$((kept_dependency + 1)) fi log " keeping (${tag}): $base" mv "$f" "${PKG_DIR}/" kept=$((kept + 1)) done log "Source '${name}': kept ${kept} package(s) - ${kept_requested} explicitly requested + ${kept_dependency} pulled-in third-party dependencies (re-signed with mirror key), dropped ${dropped} AlmaLinux-provided package(s)" [[ $kept -gt 0 ]] || die "nothing kept for source '${name}' - check baseurl/package names" # --- cleanup temp repo file for this source --------------------------- rm -f "$repo_file" forget_repo_file "$repo_file" } # Iterate over sources (JSON array -> one JSON object per line via jq-less python) while IFS= read -r src_json; do name="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["name"])' "$src_json")" baseurl="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["baseurl"])' "$src_json")" gpgkey="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1]).get("gpgkey",""))' "$src_json")" pkgs_json="$(python3 -c 'import json,sys;print(json.dumps(json.loads(sys.argv[1])["packages"]))' "$src_json")" metadata_only="$(python3 -c 'import json,sys;print(str(bool(json.loads(sys.argv[1]).get("metadata_only", False))).lower())' "$src_json")" sync_one_source "$name" "$baseurl" "$gpgkey" "$pkgs_json" "$metadata_only" done < <(python3 -c ' import json,sys for s in json.load(sys.stdin): print(json.dumps(s)) ' <<<"$SOURCES_JSON") # --------------------------------------------------------------------------- # Build the ONE combined repo from everything all sources contributed # --------------------------------------------------------------------------- createrepo_c --update "$PKG_DIR" >/dev/null # Build ONE AppStream catalog for the single shared repo. The helper reads # every source's appstream-entries.tsv (written during its own processing # above) to know WHICH packages made it into the mirror, and packages.yaml # to know HOW to present them (display name, description, categories, # keywords). YAML metadata wins; the upstream RPM summary from the TSV is # the fallback. APPSTREAM_XML="${WORK_DIR}/appstream.xml" python3 "$HELPER" gen-appstream \ --origin "$APPSTREAM_ORIGIN" \ --yaml "$YAML_PATH" \ --entries-dir "$WORK_DIR" \ > "$APPSTREAM_XML" \ || die "failed to generate AppStream catalog" # Purely informational: appstreamcli's checks target the style expectations # for public app stores (content ratings, screenshots, ...). They do not # affect whether Discover shows the packages. if command -v appstreamcli >/dev/null 2>&1; then appstreamcli validate --no-net "$APPSTREAM_XML" >/dev/null 2>&1 \ || log "note: AppStream catalog has style hints (harmless) - run 'appstreamcli validate ${APPSTREAM_XML}' to see them" fi gzip -fk "$APPSTREAM_XML" modifyrepo_c --mdtype=appstream "${APPSTREAM_XML}.gz" "${PKG_DIR}/repodata" >/dev/null # Detached-sign repomd.xml itself with the mirror key, so clients can verify # the METADATA (which packages/versions exist, their checksums) hasn't been # tampered with - not just each individual RPM. Must happen last, after # createrepo_c/modifyrepo_c are done touching repomd.xml (they update its # checksums), otherwise the signature would be over stale content. gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \ --detach-sign --armor \ --output "${PKG_DIR}/repodata/repomd.xml.asc" \ "${PKG_DIR}/repodata/repomd.xml" \ || die "failed to sign repomd.xml with the mirror key" # --- The ONE client-facing .repo file - this is all clients ever need ----- cat > "${CLIENT_REPO_DIR}/mirror.repo" <