docker/sync.sh aktualisiert
This commit is contained in:
+478
-114
@@ -1,159 +1,523 @@
|
|||||||
#!/bin/bash
|
#!/usr/bin/env bash
|
||||||
# Synct die Paket-Whitelist aus Git und laedt die selektierten Pakete
|
#
|
||||||
# aus den jeweils angegebenen Fremd-Repos.
|
# sync.sh - Mirror selected packages from third-party RPM repos into ONE
|
||||||
# Erzeugt anschliessend RPM- und AppStream-Metadaten.
|
# combined local repo consumable by AlmaLinux clients (dnf /
|
||||||
|
# PackageKit / GNOME Software / Discover) - WITHOUT pulling in or
|
||||||
|
# exposing packages that AlmaLinux itself already ships
|
||||||
|
# (AppStream, BaseOS, Extras, CRB/"Extras for Enterprise").
|
||||||
|
#
|
||||||
|
# ALL sources defined in the YAML end up in a SINGLE repo
|
||||||
|
# (REPO_BASE_DIR/pkgs/) with a SINGLE client-facing .repo file
|
||||||
|
# (CLIENT_REPO_DIR/mirror.repo) - clients only ever add one repo, no matter
|
||||||
|
# how many sources you configure.
|
||||||
|
#
|
||||||
|
# What it does, per entry in the YAML source list:
|
||||||
|
# 1. Registers the source repo temporarily and resolves the full
|
||||||
|
# dependency closure of the requested packages, using the source repo
|
||||||
|
# AND the local AlmaLinux repos (needed so resolution succeeds, e.g.
|
||||||
|
# glibc-style deps that only exist in AlmaLinux).
|
||||||
|
# 2. Throws away every RPM in that closure that AlmaLinux itself provides.
|
||||||
|
# Only RPMs that genuinely only exist in the third-party repo are kept.
|
||||||
|
# 3. Verifies each kept RPM against the source's own gpgkey (if given in
|
||||||
|
# the YAML), then STRIPS the original signature and RE-SIGNS it with
|
||||||
|
# the mirror's own GPG key. Clients therefore only ever need to trust
|
||||||
|
# one key (this mirror's), never the individual upstream repos' keys.
|
||||||
|
# 4. Moves the signed RPMs into the single shared repo directory
|
||||||
|
# (REPO_BASE_DIR/pkgs/).
|
||||||
|
# Once all sources have been processed, the script rebuilds repo metadata
|
||||||
|
# for that ONE directory with createrepo_c and generates AppStream "generic
|
||||||
|
# component" metadata for ONLY the packages explicitly listed under
|
||||||
|
# "packages:" across all sources (never for pulled-in dependencies),
|
||||||
|
# injecting it once into that same repo. Only those show up as installable
|
||||||
|
# items in GNOME Software/Discover.
|
||||||
|
#
|
||||||
|
# Set "metadata_only: true" on a source to skip steps 2-4 entirely for that
|
||||||
|
# source: no RPMs are downloaded/mirrored, only curated AppStream visibility
|
||||||
|
# is contributed to the combined repo. Use this when baseurl already points
|
||||||
|
# at a repo your clients have enabled directly (e.g. AlmaLinux's own
|
||||||
|
# AppStream repo, which on AlmaLinux 10 ships dotnet-* itself) - you just
|
||||||
|
# want to narrow down what shows up in Discover without duplicating RPM
|
||||||
|
# content clients can already get natively.
|
||||||
|
#
|
||||||
|
# NOTE ON NAME COLLISIONS: if two different sources define an RPM with the
|
||||||
|
# exact same NEVRA (name-version-release.arch), the later one silently wins
|
||||||
|
# in the shared directory (harmless - it's the same package). If two
|
||||||
|
# sources define the SAME PACKAGE NAME with genuinely different content,
|
||||||
|
# that's not supported here - keep package names distinct across sources.
|
||||||
|
#
|
||||||
|
# IMPORTANT LIMITATION:
|
||||||
|
# GNOME Software's curated "Explore" front page only lists packages that
|
||||||
|
# ship a real desktop-application AppStream component (icon, screenshots,
|
||||||
|
# .desktop file). A "generic" component (what this script generates for
|
||||||
|
# CLI tools/SDKs/runtimes) makes the package installable from Discover and
|
||||||
|
# shows up in search/details, but it will not appear on that curated front
|
||||||
|
# page. This is GNOME Software/appstreamcli behaviour and cannot be fixed
|
||||||
|
# by repo metadata alone.
|
||||||
|
#
|
||||||
|
# SECURITY NOTE ON RE-SIGNING:
|
||||||
|
# The mirror's private signing key is generated once (unattended, without
|
||||||
|
# a passphrase - standard practice for automated repo signing) and kept
|
||||||
|
# under GPG_HOME. Only the exported PUBLIC key ends up in REPO_BASE_DIR
|
||||||
|
# where clients can fetch it. GPG_HOME itself must never be exposed via
|
||||||
|
# nginx/the webserver and should live on storage only the sync process can
|
||||||
|
# read. Anyone who can read GPG_HOME can sign packages as your mirror -
|
||||||
|
# treat it like any other private key material.
|
||||||
|
#
|
||||||
|
# A revocation certificate is generated alongside the key on first run
|
||||||
|
# (GPG_HOME/revocation-cert.asc) - since the key never expires, this
|
||||||
|
# certificate is the ONLY way to invalidate it later if it's ever
|
||||||
|
# compromised. Copy it to secure, offline storage right after first
|
||||||
|
# startup; see the log output / README for the exact command.
|
||||||
|
#
|
||||||
|
# Requires: dnf, dnf-plugins-core (for "dnf download"), createrepo_c,
|
||||||
|
# modifyrepo_c, rpm-sign (rpmsign/rpmkeys), gnupg2, python3,
|
||||||
|
# python3-pyyaml, git (if fetching the source list from git), curl.
|
||||||
|
#
|
||||||
|
# Run as root (needs to write a temporary .repo file to /etc/yum.repos.d).
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
CONFIG_DIR=/etc/mirror
|
# ---------------------------------------------------------------------------
|
||||||
CONFIG_REPO_URL="${CONFIG_REPO_URL:?CONFIG_REPO_URL ist nicht gesetzt}"
|
# Configuration (override via environment, e.g. in the systemd unit/cron job)
|
||||||
CONFIG_REPO_BRANCH="${CONFIG_REPO_BRANCH:-main}"
|
# ---------------------------------------------------------------------------
|
||||||
PACKAGES_FILE="$CONFIG_DIR/packages.yaml"
|
|
||||||
DEST=/data/repo/rpms
|
|
||||||
TMP_REPO_DIR=/etc/yum.repos.d
|
|
||||||
LOCKFILE=/tmp/sync.lock
|
|
||||||
APPSTREAM_GEN=/usr/local/bin/appstream-gen.py
|
|
||||||
|
|
||||||
exec 200>"$LOCKFILE"
|
# Where the YAML source list lives. Either point SOURCES_GIT_REPO at a git
|
||||||
if ! flock -n 200; then
|
# repo (SOURCES_YAML_FILE is the path *inside* that repo), or leave
|
||||||
echo "[sync] Ein anderer Sync-Lauf ist bereits aktiv, breche ab."
|
# SOURCES_GIT_REPO empty and point SOURCES_YAML_FILE at a local file.
|
||||||
exit 0
|
SOURCES_GIT_REPO="${SOURCES_GIT_REPO:-}"
|
||||||
|
SOURCES_GIT_BRANCH="${SOURCES_GIT_BRANCH:-main}"
|
||||||
|
SOURCES_YAML_FILE="${SOURCES_YAML_FILE:-sources.yaml}"
|
||||||
|
|
||||||
|
# Local working/output paths.
|
||||||
|
WORK_DIR="${WORK_DIR:-/var/cache/rpm-mirror-sync}"
|
||||||
|
REPO_BASE_DIR="${REPO_BASE_DIR:-/srv/repo}"
|
||||||
|
CLIENT_REPO_DIR="${CLIENT_REPO_DIR:-${REPO_BASE_DIR}/client-repos}"
|
||||||
|
# The ONE combined repo directory all sources' packages end up in.
|
||||||
|
PKG_DIR="${REPO_BASE_DIR}/pkgs"
|
||||||
|
|
||||||
|
# Base URL under which REPO_BASE_DIR is actually served to clients (web
|
||||||
|
# server / reverse proxy in front of REPO_BASE_DIR). Used only to generate
|
||||||
|
# ready-to-use .repo files for clients.
|
||||||
|
CLIENT_BASE_URL="${CLIENT_BASE_URL:-http://mirror.example.local/repo}"
|
||||||
|
|
||||||
|
# repoids of AlmaLinux's own repos on THIS machine, as shown by
|
||||||
|
# `dnf repolist`. Adjust to match your system if they differ.
|
||||||
|
# ("Extras for Enterprise" is the "extras" repo in AlmaLinux 10 naming; crb
|
||||||
|
# is CodeReady Builder / "extras-common" on some setups - check your
|
||||||
|
# `dnf repolist` output and adjust below.)
|
||||||
|
ALMA_REPO_IDS="${ALMA_REPO_IDS:-baseos appstream extras crb}"
|
||||||
|
|
||||||
|
# Origin of the generated AppStream catalog. Must be unique - if it collides
|
||||||
|
# with a distribution's own origin (e.g. "almalinux"), the icon/metadata
|
||||||
|
# caches on the clients overwrite each other. Can also be set via
|
||||||
|
# "appstream: origin:" in the YAML; this variable wins.
|
||||||
|
APPSTREAM_ORIGIN="${APPSTREAM_ORIGIN:-rpm-mirror}"
|
||||||
|
|
||||||
|
# The mirror's own signing identity. A key is generated once (on first run)
|
||||||
|
# under GPG_HOME and reused on every subsequent run - make sure GPG_HOME
|
||||||
|
# points at persistent storage (a volume), or you'll get a new key (and
|
||||||
|
# therefore a trust-breaking change for clients) on every restart.
|
||||||
|
GPG_HOME="${GPG_HOME:-${WORK_DIR}/gnupg}"
|
||||||
|
GPG_KEY_NAME="${GPG_KEY_NAME:-RPM Mirror}"
|
||||||
|
GPG_KEY_EMAIL="${GPG_KEY_EMAIL:-rpm-mirror@example.local}"
|
||||||
|
MIRROR_GPG_KEY_FILE="${REPO_BASE_DIR}/RPM-GPG-KEY-mirror"
|
||||||
|
|
||||||
|
LOG_TAG="rpm-mirror-sync"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${WORK_DIR}/sync.log" >&2; }
|
||||||
|
die() { log "ERROR: $*"; exit 1; }
|
||||||
|
|
||||||
|
require_cmd() {
|
||||||
|
command -v "$1" >/dev/null 2>&1 || die "required command '$1' not found (install it first)"
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup() {
|
||||||
|
# Remove any temp source .repo files we created, even on failure.
|
||||||
|
if [[ -n "${TEMP_REPO_FILES:-}" ]]; then
|
||||||
|
for f in "${TEMP_REPO_FILES[@]}"; do
|
||||||
|
[[ -n "$f" ]] && rm -f "$f"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Remove one entry from TEMP_REPO_FILES.
|
||||||
|
# NOTE: "${ARRAY[@]/pattern}" does SUBSTRING REPLACEMENT, so the element is
|
||||||
|
# left behind as an empty string rather than removed - which then makes
|
||||||
|
# cleanup() call `rm -f ""`. Rebuild the array instead.
|
||||||
|
forget_repo_file() {
|
||||||
|
local drop="$1" keep=() f
|
||||||
|
for f in "${TEMP_REPO_FILES[@]}"; do
|
||||||
|
[[ "$f" == "$drop" ]] || keep+=( "$f" )
|
||||||
|
done
|
||||||
|
TEMP_REPO_FILES=( "${keep[@]}" )
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
|
||||||
|
HELPER="${SCRIPT_DIR}/sync_helpers.py"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Preconditions
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
[[ $EUID -eq 0 ]] || die "must run as root (writes to /etc/yum.repos.d and ${REPO_BASE_DIR})"
|
||||||
|
|
||||||
|
for c in dnf createrepo_c modifyrepo_c rpm rpmsign rpmkeys gpg python3 curl gzip; do
|
||||||
|
require_cmd "$c"
|
||||||
|
done
|
||||||
|
[[ -f "$HELPER" ]] || die "sync_helpers.py not found next to sync.sh (expected: $HELPER)"
|
||||||
|
python3 -c "import yaml" 2>/dev/null || die "python3-pyyaml not installed (dnf install -y python3-pyyaml)"
|
||||||
|
|
||||||
|
mkdir -p "$WORK_DIR" "$REPO_BASE_DIR" "$CLIENT_REPO_DIR" "$PKG_DIR"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Mirror signing key: generate once, reuse forever, export the public part
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
MIRROR_GPG_KEY_ID=""
|
||||||
|
REVOCATION_CERT_FILE="${GPG_HOME}/revocation-cert.asc"
|
||||||
|
|
||||||
|
init_mirror_gpg_key() {
|
||||||
|
mkdir -p "$GPG_HOME"
|
||||||
|
chmod 700 "$GPG_HOME"
|
||||||
|
|
||||||
|
if [[ -z "$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons 2>/dev/null)" ]]; then
|
||||||
|
log "No mirror signing key found under ${GPG_HOME} - generating one (one-time)..."
|
||||||
|
gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \
|
||||||
|
--quick-generate-key "${GPG_KEY_NAME} <${GPG_KEY_EMAIL}>" rsa4096 sign never \
|
||||||
|
|| die "failed to generate mirror GPG signing key"
|
||||||
|
|
||||||
|
# GnuPG >= 2.1 automatically writes a revocation certificate under
|
||||||
|
# openpgp-revocs.d/<fingerprint>.rev as part of key generation itself
|
||||||
|
# (visible in the gpg output above: "revocation certificate stored
|
||||||
|
# as ..."). We just need to copy it to a known, stable path - no
|
||||||
|
# need to (fragile-ly) drive gpg's interactive --gen-revoke menu
|
||||||
|
# ourselves, which behaves differently across gpg versions.
|
||||||
|
local auto_revoc
|
||||||
|
auto_revoc="$(find "${GPG_HOME}/openpgp-revocs.d" -maxdepth 1 -name '*.rev' 2>/dev/null | head -n1)"
|
||||||
|
[[ -n "$auto_revoc" && -s "$auto_revoc" ]] \
|
||||||
|
|| die "expected GnuPG to auto-generate a revocation certificate under ${GPG_HOME}/openpgp-revocs.d/ but none was found"
|
||||||
|
|
||||||
|
cp "$auto_revoc" "$REVOCATION_CERT_FILE"
|
||||||
|
chmod 600 "$REVOCATION_CERT_FILE"
|
||||||
|
|
||||||
|
log "!!! Revocation certificate written to: ${REVOCATION_CERT_FILE}"
|
||||||
|
log "!!! Copy it to secure, OFFLINE storage now, e.g.:"
|
||||||
|
log "!!! docker compose cp syncer:${REVOCATION_CERT_FILE} ./mirror-key-revocation-cert.asc"
|
||||||
|
log "!!! It is the only way to invalidate this key if it is ever compromised -"
|
||||||
|
log "!!! it never expires, so it needs the same protection as the private key."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "[sync] $(date -Iseconds) Starte Sync-Lauf"
|
MIRROR_GPG_KEY_ID="$(gpg --homedir "$GPG_HOME" --list-secret-keys --with-colons \
|
||||||
|
| awk -F: '/^sec:/ {print $5; exit}')"
|
||||||
|
[[ -n "$MIRROR_GPG_KEY_ID" ]] || die "could not determine mirror GPG key id"
|
||||||
|
|
||||||
# --- 1. Whitelist aus Git holen ---------------------------------------
|
# rpmsign/rpm --addsign etc. read the signing identity from macros.
|
||||||
mkdir -p "$CONFIG_DIR"
|
cat > "${HOME:-/root}/.rpmmacros" <<EOF
|
||||||
|
%_gpg_name ${MIRROR_GPG_KEY_ID}
|
||||||
|
%_gpg_path ${GPG_HOME}
|
||||||
|
%_signature gpg
|
||||||
|
EOF
|
||||||
|
|
||||||
if [ -n "${GIT_SSH_KEY_PATH:-}" ] && [ -f "${GIT_SSH_KEY_PATH:-}" ]; then
|
# Publish the PUBLIC key once at the repo root. This is the only key
|
||||||
export GIT_SSH_COMMAND="ssh -i ${GIT_SSH_KEY_PATH} -o StrictHostKeyChecking=accept-new"
|
# clients will ever need, no matter how many upstream sources exist.
|
||||||
|
gpg --homedir "$GPG_HOME" --armor --export "$MIRROR_GPG_KEY_ID" > "$MIRROR_GPG_KEY_FILE"
|
||||||
|
|
||||||
|
log "Mirror signing key ready: ${MIRROR_GPG_KEY_ID} (public key: ${MIRROR_GPG_KEY_FILE})"
|
||||||
|
}
|
||||||
|
|
||||||
|
init_mirror_gpg_key
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fetch the YAML source list
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
resolve_yaml_path() {
|
||||||
|
if [[ -n "$SOURCES_GIT_REPO" ]]; then
|
||||||
|
if [[ "$SOURCES_YAML_FILE" == /* ]]; then
|
||||||
|
die "SOURCES_GIT_REPO is set, but SOURCES_YAML_FILE ('${SOURCES_YAML_FILE}') looks like an absolute path. When pulling the source list from git, SOURCES_YAML_FILE must be a path RELATIVE to that repo's root (e.g. 'sources.yaml' or 'config/sources.yaml') - not the '/config/...' path used for the local bind-mount option. Fix SOURCES_YAML_FILE in .env."
|
||||||
fi
|
fi
|
||||||
|
local clone_dir="${WORK_DIR}/sources-repo"
|
||||||
if [ -d "$CONFIG_DIR/.git" ]; then
|
if [[ -d "${clone_dir}/.git" ]]; then
|
||||||
git -C "$CONFIG_DIR" fetch --depth=1 origin "$CONFIG_REPO_BRANCH"
|
log "Updating source list git repo..."
|
||||||
git -C "$CONFIG_DIR" reset --hard "origin/$CONFIG_REPO_BRANCH"
|
git -C "$clone_dir" fetch --depth 1 origin "$SOURCES_GIT_BRANCH"
|
||||||
|
git -C "$clone_dir" reset --hard "origin/${SOURCES_GIT_BRANCH}"
|
||||||
else
|
else
|
||||||
git clone --depth=1 --branch "$CONFIG_REPO_BRANCH" "$CONFIG_REPO_URL" "$CONFIG_DIR"
|
log "Cloning source list git repo..."
|
||||||
|
rm -rf "$clone_dir"
|
||||||
|
git clone --depth 1 --branch "$SOURCES_GIT_BRANCH" "$SOURCES_GIT_REPO" "$clone_dir"
|
||||||
fi
|
fi
|
||||||
|
echo "${clone_dir}/${SOURCES_YAML_FILE}"
|
||||||
if [ ! -f "$PACKAGES_FILE" ]; then
|
else
|
||||||
echo "[sync] FEHLER: $PACKAGES_FILE nicht gefunden."
|
echo "$SOURCES_YAML_FILE"
|
||||||
exit 1
|
|
||||||
fi
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
mkdir -p "$DEST"
|
YAML_PATH="$(resolve_yaml_path)"
|
||||||
|
[[ -f "$YAML_PATH" ]] || die "source YAML not found: $YAML_PATH"
|
||||||
|
|
||||||
# --- 2. Pro Quelle: temporaeres Repo aktivieren, Pakete gezielt ziehen --
|
SOURCES_JSON="$(python3 "$HELPER" parse-yaml "$YAML_PATH")" || die "failed to parse $YAML_PATH"
|
||||||
source_count=$(yq -o=json '.sources | length' "$PACKAGES_FILE")
|
SOURCE_COUNT="$(python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' <<<"$SOURCES_JSON")"
|
||||||
download_failed=0
|
log "Loaded $SOURCE_COUNT source(s) from $YAML_PATH"
|
||||||
|
|
||||||
for i in $(seq 0 $((source_count - 1))); do
|
# ---------------------------------------------------------------------------
|
||||||
src=$(yq -o=json ".sources[$i]" "$PACKAGES_FILE")
|
# Per-source sync
|
||||||
name=$(echo "$src" | jq -r '.name')
|
# ---------------------------------------------------------------------------
|
||||||
baseurl=$(echo "$src" | jq -r '.baseurl')
|
|
||||||
gpgkey=$(echo "$src" | jq -r '.gpgkey // empty')
|
|
||||||
|
|
||||||
# Standard: false. Nur bei echten Fremd-Repos aktivieren, deren
|
declare -a TEMP_REPO_FILES=()
|
||||||
# Abhaengigkeiten die Clients nicht ueber ihre Basis-Repos bekommen.
|
|
||||||
resolve_deps=$(echo "$src" | jq -r '.resolve_deps // false')
|
|
||||||
|
|
||||||
# Paketeintraege koennen Strings ODER Mappings mit Metadaten sein.
|
# Build --enablerepo flags for the AlmaLinux repos as a bash array once.
|
||||||
mapfile -t packages < <(
|
declare -a ALMA_ENABLE_FLAGS=()
|
||||||
echo "$src" | jq -r '.packages[] | if type == "string" then . else .name end'
|
for r in $ALMA_REPO_IDS; do
|
||||||
)
|
ALMA_ENABLE_FLAGS+=( "--enablerepo=${r}" )
|
||||||
|
done
|
||||||
|
|
||||||
if [ "${#packages[@]}" -eq 0 ]; then
|
sync_one_source() {
|
||||||
echo "[sync] Quelle '$name': keine Pakete gelistet, ueberspringe."
|
local name="$1" baseurl="$2" gpgkey="$3" packages_json="$4" metadata_only="${5:-false}"
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "[sync] Quelle '$name': ${#packages[@]} Paket(e) -> $baseurl"
|
local repoid="mirror-src-${name}"
|
||||||
|
local repo_file="/etc/yum.repos.d/${repoid}.repo"
|
||||||
|
local tmp_dir="${WORK_DIR}/${name}/download"
|
||||||
|
local list_dir="${WORK_DIR}/${name}"
|
||||||
|
# This source's contribution to the combined AppStream metadata: one
|
||||||
|
# "name<TAB>summary" line per explicitly requested package. Collected
|
||||||
|
# here, merged across all sources, and turned into ONE appstream.xml
|
||||||
|
# after the whole source loop finishes.
|
||||||
|
local entries_tsv="${list_dir}/appstream-entries.tsv"
|
||||||
|
|
||||||
repofile="$TMP_REPO_DIR/tmp-${name}.repo"
|
log "=== Syncing source '${name}' (metadata_only=${metadata_only}) ==="
|
||||||
|
mkdir -p "$tmp_dir" "$list_dir"
|
||||||
|
: > "$entries_tsv"
|
||||||
|
|
||||||
|
# readarray of requested top-level package names for this source.
|
||||||
|
# parse-yaml normalises every entry to an object, so a package written
|
||||||
|
# as a plain string and one written as a mapping with AppStream metadata
|
||||||
|
# both arrive here as {"name": ...}.
|
||||||
|
mapfile -t packages < <(python3 -c '
|
||||||
|
import json,sys
|
||||||
|
for p in json.load(sys.stdin):
|
||||||
|
print(p["name"])
|
||||||
|
' <<<"$packages_json")
|
||||||
|
[[ ${#packages[@]} -gt 0 ]] || die "source '${name}' has no packages"
|
||||||
|
|
||||||
|
# --- 1. Register the source repo temporarily -------------------------
|
||||||
{
|
{
|
||||||
echo "[tmp-${name}]"
|
echo "[${repoid}]"
|
||||||
echo "name=${name}"
|
echo "name=Mirror source: ${name}"
|
||||||
echo "baseurl=${baseurl}"
|
echo "baseurl=${baseurl}"
|
||||||
echo "enabled=0"
|
echo "enabled=1"
|
||||||
if [ -n "$gpgkey" ]; then
|
if [[ -n "$gpgkey" ]]; then
|
||||||
echo "gpgcheck=1"
|
echo "gpgcheck=1"
|
||||||
echo "gpgkey=${gpgkey}"
|
echo "gpgkey=${gpgkey}"
|
||||||
else
|
else
|
||||||
echo "gpgcheck=0"
|
echo "gpgcheck=0"
|
||||||
fi
|
fi
|
||||||
} > "$repofile"
|
} > "$repo_file"
|
||||||
|
TEMP_REPO_FILES+=( "$repo_file" )
|
||||||
|
dnf clean expire-cache --disablerepo='*' --enablerepo="$repoid" >/dev/null 2>&1 || true
|
||||||
|
|
||||||
# Die Basis-Repos des Containers bleiben aktiviert, damit dnf
|
# --- metadata_only sources: curate Discover/GNOME Software visibility
|
||||||
# Abhaengigkeiten wie glibc oder libstdc++ ueberhaupt aufloesen kann.
|
# WITHOUT mirroring any RPM content. Use this when baseurl already
|
||||||
# Ohne sie scheitert bereits die Aufloesung, nicht erst der Download.
|
# points at a repo your clients have enabled directly (e.g. AlmaLinux's
|
||||||
dl_args=(--destdir="$DEST" --enablerepo="tmp-${name}")
|
# own AppStream repo, which ships dotnet-* itself on AlmaLinux 10) - the
|
||||||
|
# actual package install is then resolved by dnf from that repo as
|
||||||
if [ "$resolve_deps" = "true" ]; then
|
# normal; this source only contributes AppStream "generic component"
|
||||||
echo "[sync] Quelle '$name': Abhaengigkeiten werden mitgespiegelt"
|
# data for the packages listed to the combined repo built after the
|
||||||
# Bewusst OHNE --alldeps: bereits im Container installierte Pakete
|
# source loop, so ONLY those show up as installable in Discover instead
|
||||||
# (glibc & Co.) haben die Clients ohnehin und muessen nicht ins Repo.
|
# of AlmaLinux's entire (huge) AppStream catalog.
|
||||||
dl_args+=(--resolve)
|
# PRECONDITION: clients must have the repo(s) that actually provide
|
||||||
fi
|
# these packages enabled already (true for any stock AlmaLinux install -
|
||||||
|
# baseos/appstream/extras/crb are on by default).
|
||||||
if ! dnf download "${dl_args[@]}" "${packages[@]}"; then
|
if [[ "$metadata_only" == "true" ]]; then
|
||||||
echo "[sync] WARNUNG: Download fuer Quelle '$name' fehlgeschlagen."
|
for pkg in "${packages[@]}"; do
|
||||||
download_failed=1
|
local summary
|
||||||
fi
|
summary="$(dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \
|
||||||
|
--qf '%{summary}' "$pkg" 2>/dev/null | head -n1)"
|
||||||
rm -f "$repofile"
|
printf '%s\t%s\n' "$pkg" "$summary" >> "$entries_tsv"
|
||||||
done
|
done
|
||||||
|
|
||||||
# --- 3. Alte Paketversionen aufraeumen -----------------------------------
|
log "Source '${name}': metadata_only - queued AppStream visibility for ${#packages[@]} package(s), no RPMs mirrored"
|
||||||
# repomanage braucht bereits vorhandene Metadaten. Beim allerersten Lauf
|
|
||||||
# gibt es noch keine, deshalb der Existenztest.
|
rm -f "$repo_file"
|
||||||
if [ -f "$DEST/repodata/repomd.xml" ]; then
|
forget_repo_file "$repo_file"
|
||||||
echo "[sync] Raeume veraltete Paketversionen auf"
|
return
|
||||||
if ! dnf repomanage --old "$DEST" | xargs -r rm -f; then
|
|
||||||
echo "[sync] WARNUNG: repomanage fehlgeschlagen, ueberspringe Aufraeumen."
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# --- 2. List everything the source repo itself provides --------------
|
||||||
|
# (used later to decide "does this RPM genuinely come from the source,
|
||||||
|
# or did AlmaLinux already have it".)
|
||||||
|
local source_nevra_list="${list_dir}/source-nevra.txt"
|
||||||
|
dnf repoquery -y --disablerepo='*' --enablerepo="$repoid" \
|
||||||
|
--qf '%{name}-%{version}-%{release}.%{arch}' -a \
|
||||||
|
> "$source_nevra_list" \
|
||||||
|
|| die "failed to query source repo '${name}' (check baseurl/gpgkey)"
|
||||||
|
[[ -s "$source_nevra_list" ]] || die "source repo '${name}' returned no packages - check baseurl"
|
||||||
|
|
||||||
|
# --- 3. Resolve full dependency closure for the requested packages ---
|
||||||
|
# Enable the source repo AND the local AlmaLinux repos so dependency
|
||||||
|
# resolution succeeds even for deps AlmaLinux normally provides -
|
||||||
|
# we filter those back out in step 4.
|
||||||
|
rm -rf "$tmp_dir"; mkdir -p "$tmp_dir"
|
||||||
|
log "Resolving + downloading dependency closure for: ${packages[*]}"
|
||||||
|
dnf download -y --resolve --alldeps \
|
||||||
|
--destdir="$tmp_dir" \
|
||||||
|
--disablerepo='*' \
|
||||||
|
--enablerepo="$repoid" \
|
||||||
|
"${ALMA_ENABLE_FLAGS[@]}" \
|
||||||
|
"${packages[@]}" \
|
||||||
|
|| die "dnf download failed for source '${name}'"
|
||||||
|
|
||||||
|
# --- 4. Drop everything AlmaLinux already provides, verify + re-sign --
|
||||||
|
# the rest with the mirror's own key -----------------------------------
|
||||||
|
local verify_dbpath="${list_dir}/verify-rpmdb"
|
||||||
|
if [[ -n "$gpgkey" ]]; then
|
||||||
|
local source_gpgkey_file="${list_dir}/source-gpgkey.asc"
|
||||||
|
curl -fsSL "$gpgkey" -o "$source_gpgkey_file" \
|
||||||
|
|| die "could not fetch gpgkey for source '${name}' from ${gpgkey}"
|
||||||
|
rm -rf "$verify_dbpath"; mkdir -p "$verify_dbpath"
|
||||||
|
rpm --dbpath "$verify_dbpath" --initdb
|
||||||
|
rpm --dbpath "$verify_dbpath" --import "$source_gpgkey_file" \
|
||||||
|
|| die "could not import gpgkey for source '${name}'"
|
||||||
else
|
else
|
||||||
echo "[sync] Erster Lauf, kein Aufraeumen noetig."
|
log " WARNING: source '${name}' has no gpgkey configured - skipping upstream signature verification"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# --- 4. RPM-Metadaten neu erzeugen ---------------------------------------
|
local kept=0 dropped=0 kept_requested=0 kept_dependency=0
|
||||||
echo "[sync] Aktualisiere Repo-Metadaten"
|
for f in "$tmp_dir"/*.rpm; do
|
||||||
createrepo_c --update "$DEST"
|
[[ -e "$f" ]] || continue
|
||||||
|
local base; base="$(basename "$f" .rpm)"
|
||||||
|
|
||||||
# --- 5. AppStream-Katalog erzeugen und einhaengen -------------------------
|
if ! grep -qxF "$base" "$source_nevra_list"; then
|
||||||
# Ohne diesen Schritt sind die Pakete zwar per dnf installierbar, tauchen
|
log " dropping (provided by AlmaLinux): $base"
|
||||||
# aber weder in Discover noch in GNOME Software auf.
|
rm -f "$f"
|
||||||
echo "[sync] Erzeuge AppStream-Katalog"
|
dropped=$((dropped + 1))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
ASWORK=$(mktemp -d)
|
if [[ -n "$gpgkey" ]]; then
|
||||||
trap 'rm -rf "$ASWORK"' EXIT
|
rpmkeys --dbpath "$verify_dbpath" --checksig "$f" >/dev/null 2>&1 \
|
||||||
|
|| die "signature verification FAILED for '${base}' from source '${name}' - refusing to mirror it (tampered download or wrong gpgkey configured)"
|
||||||
|
fi
|
||||||
|
|
||||||
# Welche Pakete liegen tatsaechlich im Repo? Verhindert, dass Discover
|
# Strip the upstream signature and sign with the mirror's own key,
|
||||||
# Eintraege anzeigt, deren RPM gar nicht da ist.
|
# so clients only ever need to trust MIRROR_GPG_KEY_FILE.
|
||||||
if compgen -G "$DEST"/*.rpm > /dev/null; then
|
rpmsign --resign "$f" >/dev/null \
|
||||||
rpm -qp --qf '%{NAME}\n' "$DEST"/*.rpm 2>/dev/null \
|
|| die "failed to re-sign '${base}' with the mirror key"
|
||||||
| sort -u > "$ASWORK/vorhanden.txt"
|
|
||||||
|
# Label each kept package as explicitly "requested" (listed under
|
||||||
|
# packages: in the YAML) or a pulled-in "dependency" - packages the
|
||||||
|
# requested ones need but AlmaLinux doesn't provide, so they have to
|
||||||
|
# be mirrored too or `dnf install <requested>` breaks on clients.
|
||||||
|
# This is purely informational (log output); both kinds are kept.
|
||||||
|
local pkg_name tag
|
||||||
|
pkg_name="$(rpm -qp --qf '%{NAME}' "$f" 2>/dev/null)"
|
||||||
|
tag="dependency"
|
||||||
|
for req in "${packages[@]}"; do
|
||||||
|
if [[ "$pkg_name" == "$req" ]]; then
|
||||||
|
tag="requested"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if [[ "$tag" == "requested" ]]; then
|
||||||
|
kept_requested=$((kept_requested + 1))
|
||||||
|
local summary
|
||||||
|
summary="$(rpm -qp --qf '%{SUMMARY}' "$f" 2>/dev/null)"
|
||||||
|
printf '%s\t%s\n' "$pkg_name" "$summary" >> "$entries_tsv"
|
||||||
else
|
else
|
||||||
echo "[sync] WARNUNG: Keine RPMs in $DEST gefunden."
|
kept_dependency=$((kept_dependency + 1))
|
||||||
: > "$ASWORK/vorhanden.txt"
|
|
||||||
fi
|
fi
|
||||||
|
log " keeping (${tag}): $base"
|
||||||
|
|
||||||
python3 "$APPSTREAM_GEN" "$PACKAGES_FILE" \
|
mv "$f" "${PKG_DIR}/"
|
||||||
--output "$ASWORK/appstream.xml.gz" \
|
kept=$((kept + 1))
|
||||||
--available-packages "$ASWORK/vorhanden.txt"
|
done
|
||||||
|
log "Source '${name}': kept ${kept} package(s) - ${kept_requested} explicitly requested + ${kept_dependency} pulled-in third-party dependencies (re-signed with mirror key), dropped ${dropped} AlmaLinux-provided package(s)"
|
||||||
|
[[ $kept -gt 0 ]] || die "nothing kept for source '${name}' - check baseurl/package names"
|
||||||
|
|
||||||
# Optionale Validierung, falls appstreamcli im Image vorhanden ist.
|
# --- cleanup temp repo file for this source ---------------------------
|
||||||
|
rm -f "$repo_file"
|
||||||
|
forget_repo_file "$repo_file"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Iterate over sources (JSON array -> one JSON object per line via jq-less python)
|
||||||
|
while IFS= read -r src_json; do
|
||||||
|
name="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["name"])' "$src_json")"
|
||||||
|
baseurl="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1])["baseurl"])' "$src_json")"
|
||||||
|
gpgkey="$(python3 -c 'import json,sys;print(json.loads(sys.argv[1]).get("gpgkey",""))' "$src_json")"
|
||||||
|
pkgs_json="$(python3 -c 'import json,sys;print(json.dumps(json.loads(sys.argv[1])["packages"]))' "$src_json")"
|
||||||
|
metadata_only="$(python3 -c 'import json,sys;print(str(bool(json.loads(sys.argv[1]).get("metadata_only", False))).lower())' "$src_json")"
|
||||||
|
sync_one_source "$name" "$baseurl" "$gpgkey" "$pkgs_json" "$metadata_only"
|
||||||
|
done < <(python3 -c '
|
||||||
|
import json,sys
|
||||||
|
for s in json.load(sys.stdin):
|
||||||
|
print(json.dumps(s))
|
||||||
|
' <<<"$SOURCES_JSON")
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Build the ONE combined repo from everything all sources contributed
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
createrepo_c --update "$PKG_DIR" >/dev/null
|
||||||
|
|
||||||
|
# Build ONE AppStream catalog for the single shared repo. The helper reads
|
||||||
|
# every source's appstream-entries.tsv (written during its own processing
|
||||||
|
# above) to know WHICH packages made it into the mirror, and packages.yaml
|
||||||
|
# to know HOW to present them (display name, description, categories,
|
||||||
|
# keywords). YAML metadata wins; the upstream RPM summary from the TSV is
|
||||||
|
# the fallback.
|
||||||
|
APPSTREAM_XML="${WORK_DIR}/appstream.xml"
|
||||||
|
python3 "$HELPER" gen-appstream \
|
||||||
|
--origin "$APPSTREAM_ORIGIN" \
|
||||||
|
--yaml "$YAML_PATH" \
|
||||||
|
--entries-dir "$WORK_DIR" \
|
||||||
|
> "$APPSTREAM_XML" \
|
||||||
|
|| die "failed to generate AppStream catalog"
|
||||||
|
|
||||||
|
# Purely informational: appstreamcli's checks target the style expectations
|
||||||
|
# for public app stores (content ratings, screenshots, ...). They do not
|
||||||
|
# affect whether Discover shows the packages.
|
||||||
if command -v appstreamcli >/dev/null 2>&1; then
|
if command -v appstreamcli >/dev/null 2>&1; then
|
||||||
if ! appstreamcli validate --no-net "$ASWORK/appstream.xml.gz"; then
|
appstreamcli validate --no-net "$APPSTREAM_XML" >/dev/null 2>&1 \
|
||||||
echo "[sync] WARNUNG: AppStream-Katalog hat Validierungshinweise."
|
|| log "note: AppStream catalog has style hints (harmless) - run 'appstreamcli validate ${APPSTREAM_XML}' to see them"
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# modifyrepo_c MUSS nach createrepo_c laufen, da createrepo_c die
|
gzip -fk "$APPSTREAM_XML"
|
||||||
# repomd.xml komplett neu schreibt.
|
modifyrepo_c --mdtype=appstream "${APPSTREAM_XML}.gz" "${PKG_DIR}/repodata" >/dev/null
|
||||||
modifyrepo_c --mdtype=appstream "$ASWORK/appstream.xml.gz" "$DEST/repodata/"
|
|
||||||
|
|
||||||
if [ "$download_failed" -ne 0 ]; then
|
# Detached-sign repomd.xml itself with the mirror key, so clients can verify
|
||||||
echo "[sync] $(date -Iseconds) Sync-Lauf mit Fehlern abgeschlossen"
|
# the METADATA (which packages/versions exist, their checksums) hasn't been
|
||||||
exit 1
|
# tampered with - not just each individual RPM. Must happen last, after
|
||||||
fi
|
# createrepo_c/modifyrepo_c are done touching repomd.xml (they update its
|
||||||
|
# checksums), otherwise the signature would be over stale content.
|
||||||
|
gpg --homedir "$GPG_HOME" --batch --pinentry-mode loopback --passphrase '' \
|
||||||
|
--detach-sign --armor \
|
||||||
|
--output "${PKG_DIR}/repodata/repomd.xml.asc" \
|
||||||
|
"${PKG_DIR}/repodata/repomd.xml" \
|
||||||
|
|| die "failed to sign repomd.xml with the mirror key"
|
||||||
|
|
||||||
echo "[sync] $(date -Iseconds) Sync-Lauf abgeschlossen"
|
# --- The ONE client-facing .repo file - this is all clients ever need -----
|
||||||
|
cat > "${CLIENT_REPO_DIR}/mirror.repo" <<EOF
|
||||||
|
[rpm-mirror]
|
||||||
|
name=RPM mirror
|
||||||
|
baseurl=${CLIENT_BASE_URL}/pkgs/
|
||||||
|
enabled=1
|
||||||
|
gpgcheck=1
|
||||||
|
gpgkey=${CLIENT_BASE_URL}/RPM-GPG-KEY-mirror
|
||||||
|
# gpgcheck=1 verifies each RPM's own signature (re-signed with the mirror
|
||||||
|
# key). repo_gpgcheck=1 additionally verifies repomd.xml itself against
|
||||||
|
# repomd.xml.asc (generated above) - together these confirm both the
|
||||||
|
# package contents AND the package listing/checksums came from this
|
||||||
|
# mirror, unmodified.
|
||||||
|
repo_gpgcheck=1
|
||||||
|
EOF
|
||||||
|
|
||||||
|
log "All sources synced into a single repo at ${PKG_DIR}/"
|
||||||
|
log "All mirrored packages are signed with mirror key ${MIRROR_GPG_KEY_ID} - clients only need ${MIRROR_GPG_KEY_FILE}."
|
||||||
|
log "Client setup (one command, one repo, done):"
|
||||||
|
log " curl -fsSL ${CLIENT_BASE_URL}/client-repos/mirror.repo -o /etc/yum.repos.d/rpm-mirror.repo && dnf makecache"
|
||||||
Reference in New Issue
Block a user