leander19961/dnf-repo-test
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docker/sync_helpers.py aktualisiert

Browse Source
This commit is contained in:
leander19961
2026-08-26 14:32:37 +02:00
parent 9c9ced5b02
commit e386c53648
1 changed files with 321 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/sync_helpers.py
+320 -58
View File
@@ -2,87 +2,348 @@
"""
sync_helpers.py - small helper used by sync.sh
Two jobs, on purpose kept out of bash because reliable YAML parsing and XML
escaping in pure bash/sed is a losing game:
Reliable YAML parsing and XML escaping in pure bash/sed is a losing game, so
these two jobs live here:
parse-yaml <file>
Reads the "sources:" list from the YAML file and prints it as JSON
on stdout, so sync.sh can loop over it safely (no word-splitting
issues, handles quoting etc.).
Reads the "sources:" list from the YAML file, NORMALISES it and prints
it as JSON on stdout, so sync.sh can loop over it safely.
gen-appstream <origin> <entries_json>
entries_json is a JSON array of {"name":..., "summary":...} objects
(only the *explicitly requested* top-level packages, never pulled-in
dependencies). Prints a minimal AppStream "generic component" catalog
on stdout, which sync.sh gzips and injects into the repo metadata via
modifyrepo_c.
Normalisation: every entry under "packages:" becomes an object with at
least a "name" key, whether it was written as a plain string or as a
mapping with extra AppStream metadata. sync.sh can therefore always do
p["name"] without caring which form the YAML used.
NOTE: "generic" components (no icon/screenshots/.desktop file) make a
package installable from GNOME Software / Discover and show up in
search, but will NOT appear on the curated "Explore" front page -
that's a GNOME Software behaviour for non-desktop-app components, not
something fixable at the repo-metadata level.
gen-appstream --origin O --yaml F --entries-dir D
Builds the AppStream catalog for the combined repo.
Metadata precedence, highest first:
1. what packages.yaml says about the package
2. the upstream RPM summary collected by sync.sh into the
appstream-entries.tsv files under --entries-dir
3. a generated fallback
Only packages that actually made it into the mirror (i.e. that appear
in one of the TSV files) get a component - never pulled-in
dependencies, and never packages whose download failed.
NOTE ON DISCOVER/GNOME SOFTWARE:
Components without an icon/screenshots/.desktop file make a package
installable and findable via SEARCH, but they will not appear on the
curated "Explore" front page or when browsing categories. That is a
frontend behaviour for non-desktop-app components and cannot be fixed
at the repo-metadata level. Only a real desktop-application component
(type: desktop-application + desktop_id: in packages.yaml, which
requires the package to ship a .desktop file) shows up there.
"""
import sys
import json
import argparse
from xml.sax.saxutils import escape
import glob
import json
import os
import re
import sys
from xml.sax.saxutils import escape, quoteattr
CATALOG_VERSION = "0.14"
DEFAULT_NOTE = (
"Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. "
"Fragen und Paketwünsche bitte an den IT-Service-Desk."
)
def cmd_parse_yaml(path: str) -> None:
def die(msg):
sys.stdout.flush()
sys.stderr.write(f"ERROR: {msg}\n")
sys.exit(1)
def warn(msg):
sys.stdout.flush()
sys.stderr.write(f"WARNING: {msg}\n")
def load_yaml(path):
try:
import yaml
except ImportError:
sys.stderr.write(
"ERROR: python3-pyyaml (PyYAML) is not installed.\n"
"Install it with: dnf install -y python3-pyyaml\n"
die(
"python3-pyyaml (PyYAML) is not installed.\n"
"Install it with: dnf install -y python3-pyyaml"
)
sys.exit(1)
try:
with open(path, "r", encoding="utf-8") as fh:
return yaml.safe_load(fh) or {}
except OSError as exc:
die(f"could not read {path!r}: {exc}")
except Exception as exc: # yaml.YAMLError and friends
die(f"could not parse {path!r}: {exc}")
with open(path, "r", encoding="utf-8") as fh:
data = yaml.safe_load(fh) or {}
def normalise_package(entry, source_name, index):
"""Accept both 'pkgname' and {name: pkgname, display_name: ...}."""
if isinstance(entry, str):
return {"name": entry}
if isinstance(entry, dict):
if not entry.get("name"):
die(
f"package #{index} in source {source_name!r} has no 'name' key: "
f"{entry!r}"
)
return dict(entry)
die(
f"package #{index} in source {source_name!r} must be a string or a "
f"mapping, got {type(entry).__name__}"
)
def normalise_sources(data):
sources = data.get("sources") or []
if not isinstance(sources, list):
sys.stderr.write("ERROR: 'sources' in YAML is not a list\n")
sys.exit(1)
die("'sources' in YAML is not a list")
for i, src in enumerate(sources):
if not isinstance(src, dict):
die(f"source #{i} is not a mapping")
for field in ("name", "baseurl", "packages"):
if field not in src:
sys.stderr.write(
f"ERROR: source #{i} is missing required field '{field}'\n"
)
sys.exit(1)
die(f"source #{i} is missing required field '{field}'")
if not isinstance(src["packages"], list) or not src["packages"]:
sys.stderr.write(
f"ERROR: source '{src.get('name')}' has an empty/invalid packages list\n"
die(
f"source '{src.get('name')}' has an empty/invalid packages list"
)
sys.exit(1)
json.dump(sources, sys.stdout)
src["packages"] = [
normalise_package(p, src.get("name"), j)
for j, p in enumerate(src["packages"])
]
return sources
def cmd_gen_appstream(origin: str, entries_json: str) -> None:
entries = json.loads(entries_json)
print('<?xml version="1.0" encoding="UTF-8"?>')
print(f'<components version="0.14" origin="{escape(origin)}">')
for e in entries:
name = e.get("name", "")
summary = e.get("summary") or f"{name} (mirrored package)"
cid = f"{name}.generic"
print(' <component type="generic">')
print(f" <id>{escape(cid)}</id>")
print(f" <pkgname>{escape(name)}</pkgname>")
print(f" <name>{escape(name)}</name>")
print(f" <summary>{escape(summary)}</summary>")
print(" <metadata_license>CC0-1.0</metadata_license>")
print(" <project_license>unknown</project_license>")
print(" </component>")
print("</components>")
def cmd_parse_yaml(path):
data = load_yaml(path)
json.dump(normalise_sources(data), sys.stdout)
def main() -> None:
# ---------------------------------------------------------------------------
# AppStream catalog generation
# ---------------------------------------------------------------------------
def sanitize_id_part(value):
part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-")
if not part:
die(f"package name yields no usable AppStream id segment: {value!r}")
if part[0].isdigit():
part = "pkg-" + part
return part
def as_list(value):
if value is None:
return []
if isinstance(value, (list, tuple)):
return list(value)
return [value]
def pick(meta, defaults, key, fallback=None):
if meta.get(key) is not None:
return meta[key]
if defaults.get(key) is not None:
return defaults[key]
return fallback
def read_entries(entries_dir):
"""Merge every source's appstream-entries.tsv into {name: summary}."""
entries = {}
pattern = os.path.join(entries_dir, "*", "appstream-entries.tsv")
for path in sorted(glob.glob(pattern)):
try:
with open(path, encoding="utf-8") as fh:
for line in fh:
line = line.rstrip("\n")
if not line:
continue
parts = line.split("\t", 1)
name = parts[0].strip()
summary = parts[1].strip() if len(parts) > 1 else ""
if not name:
continue
if name in entries and entries[name] and summary \
and entries[name] != summary:
warn(
f"package '{name}' was contributed by more than one "
"source with differing summaries - keeping the first"
)
continue
entries.setdefault(name, summary)
except OSError:
continue
return entries
def collect_metadata(yaml_path):
"""Return (defaults, {pkgname: metadata}, id_prefix, yaml_origin)."""
data = load_yaml(yaml_path)
appstream_cfg = data.get("appstream") or {}
defaults = dict(appstream_cfg.get("defaults") or {})
id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo"
yaml_origin = appstream_cfg.get("origin")
if not defaults.get("developer_id"):
segments = id_prefix.split(".")
if len(segments) >= 2:
defaults["developer_id"] = ".".join(segments[:2])
by_name = {}
for src in normalise_sources(data):
for pkg in src["packages"]:
name = pkg["name"]
if name in by_name:
warn(f"package '{name}' listed more than once - keeping the first")
continue
by_name[name] = pkg
return defaults, by_name, id_prefix, yaml_origin
def emit(out, depth, text):
out.append(" " * depth + text)
def build_component(pkgname, meta, upstream_summary, defaults, id_prefix, out):
if meta.get("appstream") is False or meta.get("hidden") is True:
return False
binaries = as_list(meta.get("binary") or meta.get("binaries"))
# A console-application component without <provides><binary> trips
# appstreamcli's 'console-app-no-binary' check, so only claim that type
# when a binary is actually declared. Runtimes and libraries stay
# 'generic', which Discover still finds via search.
comp_type = pick(meta, defaults, "type")
if not comp_type:
comp_type = "console-application" if binaries else "generic"
comp_id = meta.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}"
display_name = meta.get("display_name") or pkgname
note = pick(meta, defaults, "note", DEFAULT_NOTE)
summary = meta.get("summary") or upstream_summary or note
emit(out, 1, f"<component type={quoteattr(comp_type)}>")
emit(out, 2, f"<id>{escape(comp_id)}</id>")
emit(out, 2, f"<pkgname>{escape(pkgname)}</pkgname>")
emit(out, 2, f"<name>{escape(display_name)}</name>")
emit(out, 2, f"<summary>{escape(summary)}</summary>")
emit(out, 2, "<description>")
body = meta.get("description")
wrote_paragraph = False
if body:
for para in [p.strip() for p in str(body).split("\n\n") if p.strip()]:
emit(out, 3, f"<p>{escape(para)}</p>")
wrote_paragraph = True
if note and (not wrote_paragraph or note != summary):
emit(out, 3, f"<p>{escape(note)}</p>")
elif not wrote_paragraph:
emit(out, 3, f"<p>{escape(summary)}</p>")
emit(out, 2, "</description>")
icon = pick(meta, defaults, "icon", "application-x-executable")
if icon:
emit(out, 2, f'<icon type="stock">{escape(str(icon))}</icon>')
categories = as_list(pick(meta, defaults, "categories"))
if categories:
emit(out, 2, "<categories>")
for cat in categories:
emit(out, 3, f"<category>{escape(str(cat))}</category>")
emit(out, 2, "</categories>")
keywords = as_list(meta.get("keywords"))
if keywords:
emit(out, 2, "<keywords>")
for kw in keywords:
emit(out, 3, f"<keyword>{escape(str(kw))}</keyword>")
emit(out, 2, "</keywords>")
if binaries:
emit(out, 2, "<provides>")
for binary in binaries:
emit(out, 3, f"<binary>{escape(str(binary))}</binary>")
emit(out, 2, "</provides>")
homepage = meta.get("homepage") or defaults.get("homepage")
if homepage:
emit(out, 2, f'<url type="homepage">{escape(str(homepage))}</url>')
help_url = meta.get("help_url") or defaults.get("help_url")
if help_url:
emit(out, 2, f'<url type="help">{escape(str(help_url))}</url>')
developer = meta.get("developer") or defaults.get("developer")
if developer:
dev_id = meta.get("developer_id") or defaults.get("developer_id")
attr = f" id={quoteattr(str(dev_id))}" if dev_id else ""
emit(out, 2, f"<developer{attr}>")
emit(out, 3, f"<name>{escape(str(developer))}</name>")
emit(out, 2, "</developer>")
project_license = meta.get("project_license") or defaults.get("project_license")
if project_license:
emit(out, 2, f"<project_license>{escape(str(project_license))}</project_license>")
# A desktop-application component that Discover cannot launch is worse
# than a generic one, so insist on the desktop file id.
if comp_type == "desktop-application":
desktop_id = meta.get("desktop_id")
if desktop_id:
emit(
out, 2,
f'<launchable type="desktop-id">{escape(str(desktop_id))}</launchable>',
)
else:
warn(
f"{pkgname}: type=desktop-application without 'desktop_id' - "
"Discover will not be able to launch it after installation"
)
emit(out, 1, "</component>")
return True
def cmd_gen_appstream(origin, yaml_path, entries_dir):
defaults, by_name, id_prefix, yaml_origin = collect_metadata(yaml_path)
entries = read_entries(entries_dir)
if not entries:
warn(
f"no appstream-entries.tsv found under {entries_dir} - the catalog "
"will be empty and nothing will show up in Discover"
)
effective_origin = origin or yaml_origin or "rpm-mirror"
out = []
out.append('<?xml version="1.0" encoding="UTF-8"?>')
out.append(
f'<components version="{CATALOG_VERSION}" origin={quoteattr(effective_origin)}>'
)
written = 0
for pkgname in sorted(entries):
meta = by_name.get(pkgname, {})
if build_component(pkgname, meta, entries[pkgname], defaults,
id_prefix, out):
written += 1
out.append("</components>")
sys.stdout.write("\n".join(out) + "\n")
sys.stderr.write(
f"gen-appstream: wrote {written} component(s), origin={effective_origin}\n"
)
def main():
ap = argparse.ArgumentParser()
sub = ap.add_subparsers(dest="cmd", required=True)
@@ -90,15 +351,16 @@ def main() -> None:
p1.add_argument("path")
p2 = sub.add_parser("gen-appstream")
p2.add_argument("origin")
p2.add_argument("entries_json")
p2.add_argument("--origin", default="")
p2.add_argument("--yaml", required=True)
p2.add_argument("--entries-dir", required=True)
args = ap.parse_args()
if args.cmd == "parse-yaml":
cmd_parse_yaml(args.path)
elif args.cmd == "gen-appstream":
cmd_gen_appstream(args.origin, args.entries_json)
cmd_gen_appstream(args.origin, args.yaml, args.entries_dir)
if __name__ == "__main__":