Dateien nach "docker/syncer" hochladen
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
# Build-Kontext ist das Repo-Root (siehe docker-compose.yml: context: ..),
|
||||
# deshalb sind alle COPY-Pfade relativ zum Repo-Root angegeben.
|
||||
FROM almalinux:latest
|
||||
|
||||
# dnf-plugins-core -> "dnf download" und "dnf repomanage"
|
||||
# createrepo_c -> createrepo_c und modifyrepo_c
|
||||
# appstream -> appstreamcli fuer die optionale Katalog-Validierung
|
||||
# python3 -> Interpreter fuer appstream-gen.py
|
||||
RUN dnf install -y \
|
||||
dnf-plugins-core \
|
||||
createrepo_c \
|
||||
appstream \
|
||||
python3 \
|
||||
git \
|
||||
jq \
|
||||
openssh-clients \
|
||||
wget \
|
||||
&& dnf clean all
|
||||
|
||||
RUN wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 \
|
||||
-O /usr/local/bin/yq \
|
||||
&& chmod +x /usr/local/bin/yq
|
||||
|
||||
COPY docker/syncer/sync.sh /usr/local/bin/sync.sh
|
||||
COPY docker/syncer/sync_helpers.py /usr/local/bin/sync_helpers.py
|
||||
COPY docker/syncer/appstream-gen.py /usr/local/bin/appstream-gen.py
|
||||
COPY docker/syncer/entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/sync.sh /entrypoint.sh
|
||||
|
||||
# Bewusst kein VOLUME und kein EXPOSE: die Volumes definiert docker-compose.yml,
|
||||
# und ausgeliefert wird ueber den separaten nginx-Container.
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Erzeugt einen AppStream-Katalog (Collection-XML) aus packages.yaml.
|
||||
|
||||
Der Katalog referenziert die gespiegelten RPMs direkt ueber <pkgname>.
|
||||
Es werden keine Wrapper-Pakete gebaut. Das Ergebnis wird per
|
||||
`modifyrepo_c --mdtype=appstream` in die Repo-Metadaten eingehaengt,
|
||||
damit Discover / GNOME Software die Pakete findet.
|
||||
|
||||
Aufruf:
|
||||
appstream-gen.py packages.yaml -o appstream.xml.gz \
|
||||
[--available-packages vorhanden.txt] [--origin firmen-repo]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import re
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from xml.dom import minidom
|
||||
|
||||
import yaml
|
||||
|
||||
# AppStream-Spezifikationsversion des Katalogformats.
|
||||
CATALOG_VERSION = "0.14"
|
||||
|
||||
DEFAULT_TYPE = "console-application"
|
||||
DEFAULT_ICON = "application-x-executable"
|
||||
DEFAULT_NOTE = (
|
||||
"Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. "
|
||||
"Fragen und Paketwünsche bitte an den IT-Service-Desk."
|
||||
)
|
||||
|
||||
|
||||
def die(msg):
|
||||
sys.stdout.flush()
|
||||
print(f"[appstream-gen] FEHLER: {msg}", file=sys.stderr, flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def warn(msg):
|
||||
sys.stdout.flush()
|
||||
print(f"[appstream-gen] WARNUNG: {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def info(msg):
|
||||
print(f"[appstream-gen] {msg}", flush=True)
|
||||
|
||||
|
||||
def sanitize_id_part(value):
|
||||
"""Macht aus einem Paketnamen ein gueltiges AppStream-ID-Segment."""
|
||||
part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-")
|
||||
if not part:
|
||||
die(f"Paketname ergibt kein gueltiges ID-Segment: {value!r}")
|
||||
if part[0].isdigit():
|
||||
part = "pkg-" + part
|
||||
return part
|
||||
|
||||
|
||||
def prettify_name(pkgname):
|
||||
"""Fallback-Anzeigename, wenn in der YAML keiner gesetzt ist.
|
||||
|
||||
dotnet-sdk-9.0 -> Dotnet Sdk 9.0
|
||||
"""
|
||||
words = re.split(r"[-_]", pkgname)
|
||||
return " ".join(w[:1].upper() + w[1:] if w else w for w in words)
|
||||
|
||||
|
||||
def as_list(value):
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple)):
|
||||
return list(value)
|
||||
return [value]
|
||||
|
||||
|
||||
def normalize_entry(entry, source_name):
|
||||
"""Akzeptiert sowohl 'paketname' als auch ein Mapping mit Metadaten."""
|
||||
if isinstance(entry, str):
|
||||
return {"name": entry, "_source": source_name}
|
||||
if isinstance(entry, dict):
|
||||
if "name" not in entry:
|
||||
die(f"Paketeintrag in Quelle {source_name!r} ohne 'name': {entry!r}")
|
||||
data = dict(entry)
|
||||
data["_source"] = source_name
|
||||
return data
|
||||
die(f"Unerwarteter Paketeintrag in Quelle {source_name!r}: {entry!r}")
|
||||
|
||||
|
||||
def pick(entry, defaults, key, fallback=None):
|
||||
"""Wert aus dem Paketeintrag, sonst aus defaults, sonst fallback."""
|
||||
if key in entry and entry[key] is not None:
|
||||
return entry[key]
|
||||
if key in defaults and defaults[key] is not None:
|
||||
return defaults[key]
|
||||
return fallback
|
||||
|
||||
|
||||
def build_component(entry, defaults, id_prefix):
|
||||
pkgname = entry["name"]
|
||||
|
||||
comp_type = pick(entry, defaults, "type", DEFAULT_TYPE)
|
||||
component = ET.Element("component", {"type": comp_type})
|
||||
|
||||
comp_id = entry.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}"
|
||||
ET.SubElement(component, "id").text = comp_id
|
||||
ET.SubElement(component, "pkgname").text = pkgname
|
||||
|
||||
display_name = entry.get("display_name") or entry.get("name_display")
|
||||
if not display_name:
|
||||
display_name = prettify_name(pkgname)
|
||||
ET.SubElement(component, "name").text = display_name
|
||||
|
||||
note = pick(entry, defaults, "note", DEFAULT_NOTE)
|
||||
summary = entry.get("summary") or note
|
||||
ET.SubElement(component, "summary").text = summary
|
||||
|
||||
# Beschreibung: eigener Text plus Hinweis auf die Herkunft.
|
||||
description = ET.SubElement(component, "description")
|
||||
body = entry.get("description")
|
||||
if body:
|
||||
for paragraph in [p.strip() for p in str(body).split("\n\n") if p.strip()]:
|
||||
ET.SubElement(description, "p").text = paragraph
|
||||
if note and note != summary:
|
||||
ET.SubElement(description, "p").text = note
|
||||
elif not body and note == summary:
|
||||
ET.SubElement(description, "p").text = note
|
||||
|
||||
icon = pick(entry, defaults, "icon", DEFAULT_ICON)
|
||||
if icon:
|
||||
ET.SubElement(component, "icon", {"type": "stock"}).text = icon
|
||||
|
||||
categories = as_list(pick(entry, defaults, "categories"))
|
||||
if categories:
|
||||
cat_el = ET.SubElement(component, "categories")
|
||||
for category in categories:
|
||||
ET.SubElement(cat_el, "category").text = str(category)
|
||||
|
||||
keywords = as_list(entry.get("keywords"))
|
||||
if keywords:
|
||||
kw_el = ET.SubElement(component, "keywords")
|
||||
for keyword in keywords:
|
||||
ET.SubElement(kw_el, "keyword").text = str(keyword)
|
||||
|
||||
binaries = as_list(entry.get("binary") or entry.get("binaries"))
|
||||
if binaries:
|
||||
provides = ET.SubElement(component, "provides")
|
||||
for binary in binaries:
|
||||
ET.SubElement(provides, "binary").text = str(binary)
|
||||
|
||||
homepage = entry.get("homepage") or defaults.get("homepage")
|
||||
if homepage:
|
||||
ET.SubElement(component, "url", {"type": "homepage"}).text = str(homepage)
|
||||
|
||||
help_url = entry.get("help_url") or defaults.get("help_url")
|
||||
if help_url:
|
||||
ET.SubElement(component, "url", {"type": "help"}).text = str(help_url)
|
||||
|
||||
developer = entry.get("developer") or defaults.get("developer")
|
||||
if developer:
|
||||
# Das id-Attribut ist seit AppStream 1.0 erwartet; ohne es meldet
|
||||
# appstreamcli 'developer-id-missing'.
|
||||
dev_id = entry.get("developer_id") or defaults.get("developer_id")
|
||||
attrs = {"id": str(dev_id)} if dev_id else {}
|
||||
dev_el = ET.SubElement(component, "developer", attrs)
|
||||
ET.SubElement(dev_el, "name").text = str(developer)
|
||||
|
||||
project_license = entry.get("project_license") or defaults.get("project_license")
|
||||
if project_license:
|
||||
ET.SubElement(component, "project_license").text = str(project_license)
|
||||
|
||||
# desktop-application braucht einen Launchable-Eintrag, sonst ignoriert
|
||||
# Discover die Komponente beim Starten der installierten App.
|
||||
if comp_type == "desktop-application":
|
||||
desktop_id = entry.get("desktop_id")
|
||||
if desktop_id:
|
||||
ET.SubElement(
|
||||
component, "launchable", {"type": "desktop-id"}
|
||||
).text = str(desktop_id)
|
||||
else:
|
||||
warn(
|
||||
f"{pkgname}: type=desktop-application ohne 'desktop_id'. "
|
||||
"Discover kann die Anwendung dann nicht starten."
|
||||
)
|
||||
|
||||
return comp_id, component
|
||||
|
||||
|
||||
def load_available(path):
|
||||
if not path:
|
||||
return None
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
names = {line.strip() for line in handle if line.strip()}
|
||||
except OSError as exc:
|
||||
die(f"Konnte Paketliste {path!r} nicht lesen: {exc}")
|
||||
info(f"{len(names)} tatsaechlich vorhandene Pakete eingelesen")
|
||||
return names
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Erzeugt einen AppStream-Katalog aus packages.yaml"
|
||||
)
|
||||
parser.add_argument("packages_yaml", help="Pfad zur packages.yaml")
|
||||
parser.add_argument(
|
||||
"-o", "--output", default="appstream.xml.gz",
|
||||
help="Zieldatei (.xml.gz oder .xml), Standard: appstream.xml.gz",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--origin",
|
||||
help="Origin des Katalogs. Ueberschreibt 'appstream.origin' aus der YAML.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--available-packages",
|
||||
help="Datei mit den tatsaechlich gespiegelten Paketnamen (einer pro Zeile). "
|
||||
"Nicht vorhandene Pakete werden dann uebersprungen.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
help="Nur Pakete dieser einen Quelle beruecksichtigen. Fuer Setups "
|
||||
"mit einem Repo-Verzeichnis pro Quelle.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict", action="store_true",
|
||||
help="Mit Fehler abbrechen, wenn ein gelistetes Paket nicht vorhanden ist.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
with open(args.packages_yaml, "r", encoding="utf-8") as handle:
|
||||
config = yaml.safe_load(handle) or {}
|
||||
except OSError as exc:
|
||||
die(f"Konnte {args.packages_yaml!r} nicht lesen: {exc}")
|
||||
except yaml.YAMLError as exc:
|
||||
die(f"YAML-Fehler in {args.packages_yaml!r}: {exc}")
|
||||
|
||||
appstream_cfg = config.get("appstream") or {}
|
||||
defaults = appstream_cfg.get("defaults") or {}
|
||||
origin = args.origin or appstream_cfg.get("origin") or "firmen-repo"
|
||||
id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo"
|
||||
|
||||
# Falls kein developer_id gesetzt ist, die ersten beiden Segmente des
|
||||
# id_prefix verwenden: de.firma.repo -> de.firma
|
||||
if "developer_id" not in defaults or not defaults.get("developer_id"):
|
||||
segments = id_prefix.split(".")
|
||||
if len(segments) >= 2:
|
||||
defaults = dict(defaults)
|
||||
defaults["developer_id"] = ".".join(segments[:2])
|
||||
|
||||
available = load_available(args.available_packages)
|
||||
|
||||
sources = config.get("sources") or []
|
||||
if not sources:
|
||||
die("Keine 'sources' in der packages.yaml gefunden.")
|
||||
|
||||
if args.source:
|
||||
sources = [s for s in sources if s.get("name") == args.source]
|
||||
if not sources:
|
||||
die(f"Keine Quelle mit dem Namen {args.source!r} in der YAML gefunden.")
|
||||
info(f"Beschraenke auf Quelle {args.source!r}")
|
||||
|
||||
root = ET.Element("components", {"version": CATALOG_VERSION, "origin": origin})
|
||||
|
||||
seen_ids = {}
|
||||
seen_pkgs = set()
|
||||
skipped = []
|
||||
count = 0
|
||||
|
||||
for source in sources:
|
||||
source_name = source.get("name", "<unbenannt>")
|
||||
for raw_entry in source.get("packages") or []:
|
||||
entry = normalize_entry(raw_entry, source_name)
|
||||
pkgname = entry["name"]
|
||||
|
||||
if entry.get("appstream") is False or entry.get("hidden") is True:
|
||||
info(f"{pkgname}: per Konfiguration von AppStream ausgenommen")
|
||||
continue
|
||||
|
||||
if available is not None and pkgname not in available:
|
||||
skipped.append(pkgname)
|
||||
continue
|
||||
|
||||
if pkgname in seen_pkgs:
|
||||
warn(f"{pkgname}: mehrfach gelistet, spaetere Eintraege ignoriert")
|
||||
continue
|
||||
seen_pkgs.add(pkgname)
|
||||
|
||||
comp_id, component = build_component(entry, defaults, id_prefix)
|
||||
if comp_id in seen_ids:
|
||||
die(
|
||||
f"Doppelte AppStream-ID {comp_id!r} "
|
||||
f"({seen_ids[comp_id]} und {pkgname}). "
|
||||
"Setze 'id:' fuer eines der beiden Pakete."
|
||||
)
|
||||
seen_ids[comp_id] = pkgname
|
||||
root.append(component)
|
||||
count += 1
|
||||
|
||||
if skipped:
|
||||
message = (
|
||||
f"{len(skipped)} gelistete Pakete sind nicht im Repo vorhanden "
|
||||
f"und wurden uebersprungen: {', '.join(sorted(skipped))}"
|
||||
)
|
||||
if args.strict:
|
||||
die(message)
|
||||
warn(message)
|
||||
|
||||
raw = ET.tostring(root, encoding="utf-8")
|
||||
pretty = minidom.parseString(raw).toprettyxml(indent=" ", encoding="UTF-8")
|
||||
|
||||
try:
|
||||
if args.output.endswith(".gz"):
|
||||
with gzip.open(args.output, "wb") as handle:
|
||||
handle.write(pretty)
|
||||
else:
|
||||
with open(args.output, "wb") as handle:
|
||||
handle.write(pretty)
|
||||
except OSError as exc:
|
||||
die(f"Konnte {args.output!r} nicht schreiben: {exc}")
|
||||
|
||||
info(f"{count} Komponente(n) nach {args.output} geschrieben (origin={origin})")
|
||||
if count == 0:
|
||||
warn("Katalog ist leer. Discover wird nichts anzeigen.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
# Container entrypoint: run sync.sh once, or in a loop every
|
||||
# SYNC_INTERVAL_SECONDS. There's no systemd/cron inside the container, so a
|
||||
# simple sleep-loop is the pragmatic choice.
|
||||
set -euo pipefail
|
||||
|
||||
: "${SYNC_INTERVAL_SECONDS:=3600}"
|
||||
: "${RUN_ONCE:=false}"
|
||||
|
||||
log() { echo "[entrypoint] $(date '+%Y-%m-%d %H:%M:%S') $*"; }
|
||||
|
||||
run_once() {
|
||||
/usr/local/bin/sync.sh
|
||||
}
|
||||
|
||||
if [[ "$RUN_ONCE" == "true" ]]; then
|
||||
log "RUN_ONCE=true - running sync.sh once and exiting"
|
||||
run_once
|
||||
exit $?
|
||||
fi
|
||||
|
||||
log "Starting sync loop (interval: ${SYNC_INTERVAL_SECONDS}s)"
|
||||
|
||||
# Graceful shutdown on docker stop / compose down
|
||||
term_handler() {
|
||||
log "Received SIGTERM, shutting down"
|
||||
exit 0
|
||||
}
|
||||
trap term_handler SIGTERM SIGINT
|
||||
|
||||
while true; do
|
||||
# NOTE: don't inline `run_once` into `if ! run_once; then ...` - inside
|
||||
# that then-branch, $? would reflect the (always-0) negated test itself,
|
||||
# not run_once's real exit code. Capture it explicitly instead.
|
||||
rc=0
|
||||
run_once || rc=$?
|
||||
if [[ $rc -ne 0 ]]; then
|
||||
log "sync.sh failed (exit ${rc}) - will retry after the next interval"
|
||||
fi
|
||||
log "Sleeping ${SYNC_INTERVAL_SECONDS}s until next sync..."
|
||||
sleep "${SYNC_INTERVAL_SECONDS}" &
|
||||
wait $!
|
||||
done
|
||||
@@ -0,0 +1,191 @@
|
||||
#!/bin/bash
|
||||
# Synct die Paket-Whitelist aus Git (oder aus einer lokal gemounteten Datei)
|
||||
# und laedt die selektierten Pakete aus den jeweils angegebenen Fremd-Repos.
|
||||
# Erzeugt anschliessend RPM- und - sofern der Generator vorhanden ist -
|
||||
# AppStream-Metadaten.
|
||||
set -euo pipefail
|
||||
|
||||
# createrepo_c uebernimmt die umask des Aufrufers. Ohne "umask 022" kann
|
||||
# repodata/ mit 0700 landen; der nginx-Worker (User nginx) kommt dann nicht
|
||||
# hinein und liefert 403 - bzw. 404, falls die nginx-Config try_files nutzt.
|
||||
umask 022
|
||||
|
||||
# --- Pfade ----------------------------------------------------------------
|
||||
# Muessen zu den Volumes in docker-compose.yml passen:
|
||||
# repo-data -> /srv/repo (im nginx-Container: /usr/share/nginx/html/repo)
|
||||
# sync-cache -> /var/cache/rpm-mirror-sync
|
||||
# Daraus ergibt sich die oeffentliche URL: <host>/repo/pkgs/repodata/repomd.xml
|
||||
WORK_DIR="${WORK_DIR:-/var/cache/rpm-mirror-sync}"
|
||||
REPO_ROOT="${REPO_ROOT:-/srv/repo}"
|
||||
DEST="$REPO_ROOT/pkgs"
|
||||
|
||||
CONFIG_DIR="$WORK_DIR/config"
|
||||
LOCAL_PACKAGES_FILE=/config/packages.yaml
|
||||
TMP_REPO_DIR=/etc/yum.repos.d
|
||||
LOCKFILE="$WORK_DIR/sync.lock"
|
||||
APPSTREAM_GEN="${APPSTREAM_GEN:-/usr/local/bin/appstream-gen.py}"
|
||||
|
||||
mkdir -p "$WORK_DIR" "$DEST"
|
||||
|
||||
exec 200>"$LOCKFILE"
|
||||
if ! flock -n 200; then
|
||||
echo "[sync] Ein anderer Sync-Lauf ist bereits aktiv, breche ab."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[sync] $(date -Iseconds) Starte Sync-Lauf"
|
||||
|
||||
# --- 1. Whitelist besorgen ------------------------------------------------
|
||||
# Eine gemountete /config/packages.yaml hat Vorrang. Nur wenn sie fehlt,
|
||||
# wird SOURCES_GIT_REPO gebraucht.
|
||||
if [ -f "$LOCAL_PACKAGES_FILE" ]; then
|
||||
PACKAGES_FILE="$LOCAL_PACKAGES_FILE"
|
||||
echo "[sync] Nutze lokal gemountete $PACKAGES_FILE"
|
||||
else
|
||||
: "${SOURCES_GIT_REPO:?weder SOURCES_GIT_REPO gesetzt noch /config/packages.yaml gemountet}"
|
||||
CONFIG_REPO_BRANCH="${CONFIG_REPO_BRANCH:-main}"
|
||||
PACKAGES_FILE="$CONFIG_DIR/packages.yaml"
|
||||
|
||||
if [ -n "${GIT_SSH_KEY_PATH:-}" ] && [ -f "$GIT_SSH_KEY_PATH" ]; then
|
||||
export GIT_SSH_COMMAND="ssh -i ${GIT_SSH_KEY_PATH} -o StrictHostKeyChecking=accept-new"
|
||||
fi
|
||||
|
||||
if [ -d "$CONFIG_DIR/.git" ]; then
|
||||
git -C "$CONFIG_DIR" fetch --depth=1 origin "$CONFIG_REPO_BRANCH"
|
||||
git -C "$CONFIG_DIR" reset --hard "origin/$CONFIG_REPO_BRANCH"
|
||||
else
|
||||
# Reste eines abgebrochenen Clones wegraeumen, sonst scheitert git clone.
|
||||
rm -rf "$CONFIG_DIR"
|
||||
git clone --depth=1 --branch "$CONFIG_REPO_BRANCH" "$SOURCES_GIT_REPO" "$CONFIG_DIR"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f "$PACKAGES_FILE" ]; then
|
||||
echo "[sync] FEHLER: $PACKAGES_FILE nicht gefunden."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- 2. Pro Quelle: temporaeres Repo aktivieren, Pakete gezielt ziehen -----
|
||||
source_count=$(yq -o=json '.sources | length' "$PACKAGES_FILE")
|
||||
download_failed=0
|
||||
|
||||
for i in $(seq 0 $((source_count - 1))); do
|
||||
src=$(yq -o=json ".sources[$i]" "$PACKAGES_FILE")
|
||||
name=$(echo "$src" | jq -r '.name')
|
||||
baseurl=$(echo "$src" | jq -r '.baseurl')
|
||||
gpgkey=$(echo "$src" | jq -r '.gpgkey // empty')
|
||||
|
||||
# Standard: false. Nur bei echten Fremd-Repos aktivieren, deren
|
||||
# Abhaengigkeiten die Clients nicht ueber ihre Basis-Repos bekommen.
|
||||
resolve_deps=$(echo "$src" | jq -r '.resolve_deps // false')
|
||||
|
||||
# Paketeintraege koennen Strings ODER Mappings mit Metadaten sein.
|
||||
mapfile -t packages < <(
|
||||
echo "$src" | jq -r '.packages[] | if type == "string" then . else .name end'
|
||||
)
|
||||
|
||||
if [ "${#packages[@]}" -eq 0 ]; then
|
||||
echo "[sync] Quelle '$name': keine Pakete gelistet, ueberspringe."
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[sync] Quelle '$name': ${#packages[@]} Paket(e) -> $baseurl"
|
||||
|
||||
repofile="$TMP_REPO_DIR/tmp-${name}.repo"
|
||||
{
|
||||
echo "[tmp-${name}]"
|
||||
echo "name=${name}"
|
||||
echo "baseurl=${baseurl}"
|
||||
echo "enabled=0"
|
||||
if [ -n "$gpgkey" ]; then
|
||||
echo "gpgcheck=1"
|
||||
echo "gpgkey=${gpgkey}"
|
||||
else
|
||||
echo "gpgcheck=0"
|
||||
fi
|
||||
} > "$repofile"
|
||||
|
||||
# Die Basis-Repos des Containers bleiben aktiviert, damit dnf
|
||||
# Abhaengigkeiten wie glibc oder libstdc++ ueberhaupt aufloesen kann.
|
||||
# Ohne sie scheitert bereits die Aufloesung, nicht erst der Download.
|
||||
dl_args=(--destdir="$DEST" --enablerepo="tmp-${name}")
|
||||
|
||||
if [ "$resolve_deps" = "true" ]; then
|
||||
echo "[sync] Quelle '$name': Abhaengigkeiten werden mitgespiegelt"
|
||||
# Bewusst OHNE --alldeps: bereits im Container installierte Pakete
|
||||
# (glibc & Co.) haben die Clients ohnehin und muessen nicht ins Repo.
|
||||
dl_args+=(--resolve)
|
||||
fi
|
||||
|
||||
if ! dnf download "${dl_args[@]}" "${packages[@]}"; then
|
||||
echo "[sync] WARNUNG: Download fuer Quelle '$name' fehlgeschlagen."
|
||||
download_failed=1
|
||||
fi
|
||||
|
||||
rm -f "$repofile"
|
||||
done
|
||||
|
||||
# --- 3. Alte Paketversionen aufraeumen ------------------------------------
|
||||
# repomanage braucht bereits vorhandene Metadaten. Beim allerersten Lauf
|
||||
# gibt es noch keine, deshalb der Existenztest.
|
||||
if [ -f "$DEST/repodata/repomd.xml" ]; then
|
||||
echo "[sync] Raeume veraltete Paketversionen auf"
|
||||
if ! dnf repomanage --old "$DEST" | xargs -r rm -f; then
|
||||
echo "[sync] WARNUNG: repomanage fehlgeschlagen, ueberspringe Aufraeumen."
|
||||
fi
|
||||
else
|
||||
echo "[sync] Erster Lauf, kein Aufraeumen noetig."
|
||||
fi
|
||||
|
||||
# --- 4. RPM-Metadaten neu erzeugen ----------------------------------------
|
||||
echo "[sync] Aktualisiere Repo-Metadaten"
|
||||
createrepo_c --update "$DEST"
|
||||
|
||||
# --- 5. AppStream-Katalog erzeugen und einhaengen --------------------------
|
||||
# Ohne diesen Schritt sind die Pakete zwar per dnf installierbar, tauchen
|
||||
# aber weder in Discover noch in GNOME Software auf. Der Generator ist
|
||||
# optional: fehlt er, laeuft der Sync trotzdem sauber durch.
|
||||
if [ -x "$APPSTREAM_GEN" ]; then
|
||||
echo "[sync] Erzeuge AppStream-Katalog"
|
||||
|
||||
ASWORK=$(mktemp -d)
|
||||
trap 'rm -rf "$ASWORK"' EXIT
|
||||
|
||||
# Welche Pakete liegen tatsaechlich im Repo? Verhindert, dass Discover
|
||||
# Eintraege anzeigt, deren RPM gar nicht da ist.
|
||||
if compgen -G "$DEST"/*.rpm > /dev/null; then
|
||||
rpm -qp --qf '%{NAME}\n' "$DEST"/*.rpm 2>/dev/null \
|
||||
| sort -u > "$ASWORK/vorhanden.txt"
|
||||
else
|
||||
echo "[sync] WARNUNG: Keine RPMs in $DEST gefunden."
|
||||
: > "$ASWORK/vorhanden.txt"
|
||||
fi
|
||||
|
||||
python3 "$APPSTREAM_GEN" "$PACKAGES_FILE" \
|
||||
--output "$ASWORK/appstream.xml.gz" \
|
||||
--available-packages "$ASWORK/vorhanden.txt"
|
||||
|
||||
# Optionale Validierung, falls appstreamcli im Image vorhanden ist.
|
||||
if command -v appstreamcli > /dev/null 2>&1; then
|
||||
if ! appstreamcli validate --no-net "$ASWORK/appstream.xml.gz"; then
|
||||
echo "[sync] WARNUNG: AppStream-Katalog hat Validierungshinweise."
|
||||
fi
|
||||
fi
|
||||
|
||||
# modifyrepo_c MUSS nach createrepo_c laufen, da createrepo_c die
|
||||
# repomd.xml komplett neu schreibt.
|
||||
modifyrepo_c --mdtype=appstream "$ASWORK/appstream.xml.gz" "$DEST/repodata/"
|
||||
else
|
||||
echo "[sync] HINWEIS: $APPSTREAM_GEN nicht vorhanden - AppStream wird uebersprungen."
|
||||
fi
|
||||
|
||||
# --- 6. Leserechte fuer den Webserver sicherstellen ------------------------
|
||||
# a+rX = Leserecht auf alles, Betretungsrecht nur auf Verzeichnisse.
|
||||
chmod -R a+rX "$REPO_ROOT"
|
||||
|
||||
if [ "$download_failed" -ne 0 ]; then
|
||||
echo "[sync] $(date -Iseconds) Sync-Lauf mit Fehlern abgeschlossen"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[sync] $(date -Iseconds) Sync-Lauf abgeschlossen"
|
||||
@@ -0,0 +1,367 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
sync_helpers.py - small helper used by sync.sh
|
||||
|
||||
Reliable YAML parsing and XML escaping in pure bash/sed is a losing game, so
|
||||
these two jobs live here:
|
||||
|
||||
parse-yaml <file>
|
||||
Reads the "sources:" list from the YAML file, NORMALISES it and prints
|
||||
it as JSON on stdout, so sync.sh can loop over it safely.
|
||||
|
||||
Normalisation: every entry under "packages:" becomes an object with at
|
||||
least a "name" key, whether it was written as a plain string or as a
|
||||
mapping with extra AppStream metadata. sync.sh can therefore always do
|
||||
p["name"] without caring which form the YAML used.
|
||||
|
||||
gen-appstream --origin O --yaml F --entries-dir D
|
||||
Builds the AppStream catalog for the combined repo.
|
||||
|
||||
Metadata precedence, highest first:
|
||||
1. what packages.yaml says about the package
|
||||
2. the upstream RPM summary collected by sync.sh into the
|
||||
appstream-entries.tsv files under --entries-dir
|
||||
3. a generated fallback
|
||||
|
||||
Only packages that actually made it into the mirror (i.e. that appear
|
||||
in one of the TSV files) get a component - never pulled-in
|
||||
dependencies, and never packages whose download failed.
|
||||
|
||||
NOTE ON DISCOVER/GNOME SOFTWARE:
|
||||
Components without an icon/screenshots/.desktop file make a package
|
||||
installable and findable via SEARCH, but they will not appear on the
|
||||
curated "Explore" front page or when browsing categories. That is a
|
||||
frontend behaviour for non-desktop-app components and cannot be fixed
|
||||
at the repo-metadata level. Only a real desktop-application component
|
||||
(type: desktop-application + desktop_id: in packages.yaml, which
|
||||
requires the package to ship a .desktop file) shows up there.
|
||||
"""
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from xml.sax.saxutils import escape, quoteattr
|
||||
|
||||
CATALOG_VERSION = "0.14"
|
||||
DEFAULT_NOTE = (
|
||||
"Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. "
|
||||
"Fragen und Paketwünsche bitte an den IT-Service-Desk."
|
||||
)
|
||||
|
||||
|
||||
def die(msg):
|
||||
sys.stdout.flush()
|
||||
sys.stderr.write(f"ERROR: {msg}\n")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def warn(msg):
|
||||
sys.stdout.flush()
|
||||
sys.stderr.write(f"WARNING: {msg}\n")
|
||||
|
||||
|
||||
def load_yaml(path):
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
die(
|
||||
"python3-pyyaml (PyYAML) is not installed.\n"
|
||||
"Install it with: dnf install -y python3-pyyaml"
|
||||
)
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
return yaml.safe_load(fh) or {}
|
||||
except OSError as exc:
|
||||
die(f"could not read {path!r}: {exc}")
|
||||
except Exception as exc: # yaml.YAMLError and friends
|
||||
die(f"could not parse {path!r}: {exc}")
|
||||
|
||||
|
||||
def normalise_package(entry, source_name, index):
|
||||
"""Accept both 'pkgname' and {name: pkgname, display_name: ...}."""
|
||||
if isinstance(entry, str):
|
||||
return {"name": entry}
|
||||
if isinstance(entry, dict):
|
||||
if not entry.get("name"):
|
||||
die(
|
||||
f"package #{index} in source {source_name!r} has no 'name' key: "
|
||||
f"{entry!r}"
|
||||
)
|
||||
return dict(entry)
|
||||
die(
|
||||
f"package #{index} in source {source_name!r} must be a string or a "
|
||||
f"mapping, got {type(entry).__name__}"
|
||||
)
|
||||
|
||||
|
||||
def normalise_sources(data):
|
||||
sources = data.get("sources") or []
|
||||
if not isinstance(sources, list):
|
||||
die("'sources' in YAML is not a list")
|
||||
|
||||
for i, src in enumerate(sources):
|
||||
if not isinstance(src, dict):
|
||||
die(f"source #{i} is not a mapping")
|
||||
for field in ("name", "baseurl", "packages"):
|
||||
if field not in src:
|
||||
die(f"source #{i} is missing required field '{field}'")
|
||||
if not isinstance(src["packages"], list) or not src["packages"]:
|
||||
die(
|
||||
f"source '{src.get('name')}' has an empty/invalid packages list"
|
||||
)
|
||||
src["packages"] = [
|
||||
normalise_package(p, src.get("name"), j)
|
||||
for j, p in enumerate(src["packages"])
|
||||
]
|
||||
return sources
|
||||
|
||||
|
||||
def cmd_parse_yaml(path):
|
||||
data = load_yaml(path)
|
||||
json.dump(normalise_sources(data), sys.stdout)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AppStream catalog generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def sanitize_id_part(value):
|
||||
part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-")
|
||||
if not part:
|
||||
die(f"package name yields no usable AppStream id segment: {value!r}")
|
||||
if part[0].isdigit():
|
||||
part = "pkg-" + part
|
||||
return part
|
||||
|
||||
|
||||
def as_list(value):
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple)):
|
||||
return list(value)
|
||||
return [value]
|
||||
|
||||
|
||||
def pick(meta, defaults, key, fallback=None):
|
||||
if meta.get(key) is not None:
|
||||
return meta[key]
|
||||
if defaults.get(key) is not None:
|
||||
return defaults[key]
|
||||
return fallback
|
||||
|
||||
|
||||
def read_entries(entries_dir):
|
||||
"""Merge every source's appstream-entries.tsv into {name: summary}."""
|
||||
entries = {}
|
||||
pattern = os.path.join(entries_dir, "*", "appstream-entries.tsv")
|
||||
for path in sorted(glob.glob(pattern)):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.rstrip("\n")
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split("\t", 1)
|
||||
name = parts[0].strip()
|
||||
summary = parts[1].strip() if len(parts) > 1 else ""
|
||||
if not name:
|
||||
continue
|
||||
if name in entries and entries[name] and summary \
|
||||
and entries[name] != summary:
|
||||
warn(
|
||||
f"package '{name}' was contributed by more than one "
|
||||
"source with differing summaries - keeping the first"
|
||||
)
|
||||
continue
|
||||
entries.setdefault(name, summary)
|
||||
except OSError:
|
||||
continue
|
||||
return entries
|
||||
|
||||
|
||||
def collect_metadata(yaml_path):
|
||||
"""Return (defaults, {pkgname: metadata}, id_prefix, yaml_origin)."""
|
||||
data = load_yaml(yaml_path)
|
||||
appstream_cfg = data.get("appstream") or {}
|
||||
defaults = dict(appstream_cfg.get("defaults") or {})
|
||||
id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo"
|
||||
yaml_origin = appstream_cfg.get("origin")
|
||||
|
||||
if not defaults.get("developer_id"):
|
||||
segments = id_prefix.split(".")
|
||||
if len(segments) >= 2:
|
||||
defaults["developer_id"] = ".".join(segments[:2])
|
||||
|
||||
by_name = {}
|
||||
for src in normalise_sources(data):
|
||||
for pkg in src["packages"]:
|
||||
name = pkg["name"]
|
||||
if name in by_name:
|
||||
warn(f"package '{name}' listed more than once - keeping the first")
|
||||
continue
|
||||
by_name[name] = pkg
|
||||
return defaults, by_name, id_prefix, yaml_origin
|
||||
|
||||
|
||||
def emit(out, depth, text):
|
||||
out.append(" " * depth + text)
|
||||
|
||||
|
||||
def build_component(pkgname, meta, upstream_summary, defaults, id_prefix, out):
|
||||
if meta.get("appstream") is False or meta.get("hidden") is True:
|
||||
return False
|
||||
|
||||
binaries = as_list(meta.get("binary") or meta.get("binaries"))
|
||||
|
||||
# A console-application component without <provides><binary> trips
|
||||
# appstreamcli's 'console-app-no-binary' check, so only claim that type
|
||||
# when a binary is actually declared. Runtimes and libraries stay
|
||||
# 'generic', which Discover still finds via search.
|
||||
comp_type = pick(meta, defaults, "type")
|
||||
if not comp_type:
|
||||
comp_type = "console-application" if binaries else "generic"
|
||||
|
||||
comp_id = meta.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}"
|
||||
|
||||
display_name = meta.get("display_name") or pkgname
|
||||
|
||||
note = pick(meta, defaults, "note", DEFAULT_NOTE)
|
||||
summary = meta.get("summary") or upstream_summary or note
|
||||
|
||||
emit(out, 1, f"<component type={quoteattr(comp_type)}>")
|
||||
emit(out, 2, f"<id>{escape(comp_id)}</id>")
|
||||
emit(out, 2, f"<pkgname>{escape(pkgname)}</pkgname>")
|
||||
emit(out, 2, f"<name>{escape(display_name)}</name>")
|
||||
emit(out, 2, f"<summary>{escape(summary)}</summary>")
|
||||
|
||||
emit(out, 2, "<description>")
|
||||
body = meta.get("description")
|
||||
wrote_paragraph = False
|
||||
if body:
|
||||
for para in [p.strip() for p in str(body).split("\n\n") if p.strip()]:
|
||||
emit(out, 3, f"<p>{escape(para)}</p>")
|
||||
wrote_paragraph = True
|
||||
if note and (not wrote_paragraph or note != summary):
|
||||
emit(out, 3, f"<p>{escape(note)}</p>")
|
||||
elif not wrote_paragraph:
|
||||
emit(out, 3, f"<p>{escape(summary)}</p>")
|
||||
emit(out, 2, "</description>")
|
||||
|
||||
icon = pick(meta, defaults, "icon", "application-x-executable")
|
||||
if icon:
|
||||
emit(out, 2, f'<icon type="stock">{escape(str(icon))}</icon>')
|
||||
|
||||
categories = as_list(pick(meta, defaults, "categories"))
|
||||
if categories:
|
||||
emit(out, 2, "<categories>")
|
||||
for cat in categories:
|
||||
emit(out, 3, f"<category>{escape(str(cat))}</category>")
|
||||
emit(out, 2, "</categories>")
|
||||
|
||||
keywords = as_list(meta.get("keywords"))
|
||||
if keywords:
|
||||
emit(out, 2, "<keywords>")
|
||||
for kw in keywords:
|
||||
emit(out, 3, f"<keyword>{escape(str(kw))}</keyword>")
|
||||
emit(out, 2, "</keywords>")
|
||||
|
||||
if binaries:
|
||||
emit(out, 2, "<provides>")
|
||||
for binary in binaries:
|
||||
emit(out, 3, f"<binary>{escape(str(binary))}</binary>")
|
||||
emit(out, 2, "</provides>")
|
||||
|
||||
homepage = meta.get("homepage") or defaults.get("homepage")
|
||||
if homepage:
|
||||
emit(out, 2, f'<url type="homepage">{escape(str(homepage))}</url>')
|
||||
help_url = meta.get("help_url") or defaults.get("help_url")
|
||||
if help_url:
|
||||
emit(out, 2, f'<url type="help">{escape(str(help_url))}</url>')
|
||||
|
||||
developer = meta.get("developer") or defaults.get("developer")
|
||||
if developer:
|
||||
dev_id = meta.get("developer_id") or defaults.get("developer_id")
|
||||
attr = f" id={quoteattr(str(dev_id))}" if dev_id else ""
|
||||
emit(out, 2, f"<developer{attr}>")
|
||||
emit(out, 3, f"<name>{escape(str(developer))}</name>")
|
||||
emit(out, 2, "</developer>")
|
||||
|
||||
project_license = meta.get("project_license") or defaults.get("project_license")
|
||||
if project_license:
|
||||
emit(out, 2, f"<project_license>{escape(str(project_license))}</project_license>")
|
||||
|
||||
# A desktop-application component that Discover cannot launch is worse
|
||||
# than a generic one, so insist on the desktop file id.
|
||||
if comp_type == "desktop-application":
|
||||
desktop_id = meta.get("desktop_id")
|
||||
if desktop_id:
|
||||
emit(
|
||||
out, 2,
|
||||
f'<launchable type="desktop-id">{escape(str(desktop_id))}</launchable>',
|
||||
)
|
||||
else:
|
||||
warn(
|
||||
f"{pkgname}: type=desktop-application without 'desktop_id' - "
|
||||
"Discover will not be able to launch it after installation"
|
||||
)
|
||||
|
||||
emit(out, 1, "</component>")
|
||||
return True
|
||||
|
||||
|
||||
def cmd_gen_appstream(origin, yaml_path, entries_dir):
|
||||
defaults, by_name, id_prefix, yaml_origin = collect_metadata(yaml_path)
|
||||
entries = read_entries(entries_dir)
|
||||
|
||||
if not entries:
|
||||
warn(
|
||||
f"no appstream-entries.tsv found under {entries_dir} - the catalog "
|
||||
"will be empty and nothing will show up in Discover"
|
||||
)
|
||||
|
||||
effective_origin = origin or yaml_origin or "rpm-mirror"
|
||||
|
||||
out = []
|
||||
out.append('<?xml version="1.0" encoding="UTF-8"?>')
|
||||
out.append(
|
||||
f'<components version="{CATALOG_VERSION}" origin={quoteattr(effective_origin)}>'
|
||||
)
|
||||
|
||||
written = 0
|
||||
for pkgname in sorted(entries):
|
||||
meta = by_name.get(pkgname, {})
|
||||
if build_component(pkgname, meta, entries[pkgname], defaults,
|
||||
id_prefix, out):
|
||||
written += 1
|
||||
|
||||
out.append("</components>")
|
||||
sys.stdout.write("\n".join(out) + "\n")
|
||||
sys.stderr.write(
|
||||
f"gen-appstream: wrote {written} component(s), origin={effective_origin}\n"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
p1 = sub.add_parser("parse-yaml")
|
||||
p1.add_argument("path")
|
||||
|
||||
p2 = sub.add_parser("gen-appstream")
|
||||
p2.add_argument("--origin", default="")
|
||||
p2.add_argument("--yaml", required=True)
|
||||
p2.add_argument("--entries-dir", required=True)
|
||||
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.cmd == "parse-yaml":
|
||||
cmd_parse_yaml(args.path)
|
||||
elif args.cmd == "gen-appstream":
|
||||
cmd_gen_appstream(args.origin, args.yaml, args.entries_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user