docker/sync_helpers.py aktualisiert
This commit is contained in:
+321
-59
@@ -2,87 +2,348 @@
|
|||||||
"""
|
"""
|
||||||
sync_helpers.py - small helper used by sync.sh
|
sync_helpers.py - small helper used by sync.sh
|
||||||
|
|
||||||
Two jobs, on purpose kept out of bash because reliable YAML parsing and XML
|
Reliable YAML parsing and XML escaping in pure bash/sed is a losing game, so
|
||||||
escaping in pure bash/sed is a losing game:
|
these two jobs live here:
|
||||||
|
|
||||||
parse-yaml <file>
|
parse-yaml <file>
|
||||||
Reads the "sources:" list from the YAML file and prints it as JSON
|
Reads the "sources:" list from the YAML file, NORMALISES it and prints
|
||||||
on stdout, so sync.sh can loop over it safely (no word-splitting
|
it as JSON on stdout, so sync.sh can loop over it safely.
|
||||||
issues, handles quoting etc.).
|
|
||||||
|
|
||||||
gen-appstream <origin> <entries_json>
|
Normalisation: every entry under "packages:" becomes an object with at
|
||||||
entries_json is a JSON array of {"name":..., "summary":...} objects
|
least a "name" key, whether it was written as a plain string or as a
|
||||||
(only the *explicitly requested* top-level packages, never pulled-in
|
mapping with extra AppStream metadata. sync.sh can therefore always do
|
||||||
dependencies). Prints a minimal AppStream "generic component" catalog
|
p["name"] without caring which form the YAML used.
|
||||||
on stdout, which sync.sh gzips and injects into the repo metadata via
|
|
||||||
modifyrepo_c.
|
|
||||||
|
|
||||||
NOTE: "generic" components (no icon/screenshots/.desktop file) make a
|
gen-appstream --origin O --yaml F --entries-dir D
|
||||||
package installable from GNOME Software / Discover and show up in
|
Builds the AppStream catalog for the combined repo.
|
||||||
search, but will NOT appear on the curated "Explore" front page -
|
|
||||||
that's a GNOME Software behaviour for non-desktop-app components, not
|
Metadata precedence, highest first:
|
||||||
something fixable at the repo-metadata level.
|
1. what packages.yaml says about the package
|
||||||
|
2. the upstream RPM summary collected by sync.sh into the
|
||||||
|
appstream-entries.tsv files under --entries-dir
|
||||||
|
3. a generated fallback
|
||||||
|
|
||||||
|
Only packages that actually made it into the mirror (i.e. that appear
|
||||||
|
in one of the TSV files) get a component - never pulled-in
|
||||||
|
dependencies, and never packages whose download failed.
|
||||||
|
|
||||||
|
NOTE ON DISCOVER/GNOME SOFTWARE:
|
||||||
|
Components without an icon/screenshots/.desktop file make a package
|
||||||
|
installable and findable via SEARCH, but they will not appear on the
|
||||||
|
curated "Explore" front page or when browsing categories. That is a
|
||||||
|
frontend behaviour for non-desktop-app components and cannot be fixed
|
||||||
|
at the repo-metadata level. Only a real desktop-application component
|
||||||
|
(type: desktop-application + desktop_id: in packages.yaml, which
|
||||||
|
requires the package to ship a .desktop file) shows up there.
|
||||||
"""
|
"""
|
||||||
import sys
|
|
||||||
import json
|
|
||||||
import argparse
|
import argparse
|
||||||
from xml.sax.saxutils import escape
|
import glob
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from xml.sax.saxutils import escape, quoteattr
|
||||||
|
|
||||||
|
CATALOG_VERSION = "0.14"
|
||||||
|
DEFAULT_NOTE = (
|
||||||
|
"Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. "
|
||||||
|
"Fragen und Paketwünsche bitte an den IT-Service-Desk."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def cmd_parse_yaml(path: str) -> None:
|
def die(msg):
|
||||||
|
sys.stdout.flush()
|
||||||
|
sys.stderr.write(f"ERROR: {msg}\n")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def warn(msg):
|
||||||
|
sys.stdout.flush()
|
||||||
|
sys.stderr.write(f"WARNING: {msg}\n")
|
||||||
|
|
||||||
|
|
||||||
|
def load_yaml(path):
|
||||||
try:
|
try:
|
||||||
import yaml
|
import yaml
|
||||||
except ImportError:
|
except ImportError:
|
||||||
sys.stderr.write(
|
die(
|
||||||
"ERROR: python3-pyyaml (PyYAML) is not installed.\n"
|
"python3-pyyaml (PyYAML) is not installed.\n"
|
||||||
"Install it with: dnf install -y python3-pyyaml\n"
|
"Install it with: dnf install -y python3-pyyaml"
|
||||||
)
|
)
|
||||||
sys.exit(1)
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as fh:
|
||||||
|
return yaml.safe_load(fh) or {}
|
||||||
|
except OSError as exc:
|
||||||
|
die(f"could not read {path!r}: {exc}")
|
||||||
|
except Exception as exc: # yaml.YAMLError and friends
|
||||||
|
die(f"could not parse {path!r}: {exc}")
|
||||||
|
|
||||||
with open(path, "r", encoding="utf-8") as fh:
|
|
||||||
data = yaml.safe_load(fh) or {}
|
|
||||||
|
|
||||||
|
def normalise_package(entry, source_name, index):
|
||||||
|
"""Accept both 'pkgname' and {name: pkgname, display_name: ...}."""
|
||||||
|
if isinstance(entry, str):
|
||||||
|
return {"name": entry}
|
||||||
|
if isinstance(entry, dict):
|
||||||
|
if not entry.get("name"):
|
||||||
|
die(
|
||||||
|
f"package #{index} in source {source_name!r} has no 'name' key: "
|
||||||
|
f"{entry!r}"
|
||||||
|
)
|
||||||
|
return dict(entry)
|
||||||
|
die(
|
||||||
|
f"package #{index} in source {source_name!r} must be a string or a "
|
||||||
|
f"mapping, got {type(entry).__name__}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def normalise_sources(data):
|
||||||
sources = data.get("sources") or []
|
sources = data.get("sources") or []
|
||||||
if not isinstance(sources, list):
|
if not isinstance(sources, list):
|
||||||
sys.stderr.write("ERROR: 'sources' in YAML is not a list\n")
|
die("'sources' in YAML is not a list")
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
for i, src in enumerate(sources):
|
for i, src in enumerate(sources):
|
||||||
|
if not isinstance(src, dict):
|
||||||
|
die(f"source #{i} is not a mapping")
|
||||||
for field in ("name", "baseurl", "packages"):
|
for field in ("name", "baseurl", "packages"):
|
||||||
if field not in src:
|
if field not in src:
|
||||||
sys.stderr.write(
|
die(f"source #{i} is missing required field '{field}'")
|
||||||
f"ERROR: source #{i} is missing required field '{field}'\n"
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
if not isinstance(src["packages"], list) or not src["packages"]:
|
if not isinstance(src["packages"], list) or not src["packages"]:
|
||||||
sys.stderr.write(
|
die(
|
||||||
f"ERROR: source '{src.get('name')}' has an empty/invalid packages list\n"
|
f"source '{src.get('name')}' has an empty/invalid packages list"
|
||||||
)
|
)
|
||||||
sys.exit(1)
|
src["packages"] = [
|
||||||
|
normalise_package(p, src.get("name"), j)
|
||||||
json.dump(sources, sys.stdout)
|
for j, p in enumerate(src["packages"])
|
||||||
|
]
|
||||||
|
return sources
|
||||||
|
|
||||||
|
|
||||||
def cmd_gen_appstream(origin: str, entries_json: str) -> None:
|
def cmd_parse_yaml(path):
|
||||||
entries = json.loads(entries_json)
|
data = load_yaml(path)
|
||||||
print('<?xml version="1.0" encoding="UTF-8"?>')
|
json.dump(normalise_sources(data), sys.stdout)
|
||||||
print(f'<components version="0.14" origin="{escape(origin)}">')
|
|
||||||
for e in entries:
|
|
||||||
name = e.get("name", "")
|
|
||||||
summary = e.get("summary") or f"{name} (mirrored package)"
|
|
||||||
cid = f"{name}.generic"
|
|
||||||
print(' <component type="generic">')
|
|
||||||
print(f" <id>{escape(cid)}</id>")
|
|
||||||
print(f" <pkgname>{escape(name)}</pkgname>")
|
|
||||||
print(f" <name>{escape(name)}</name>")
|
|
||||||
print(f" <summary>{escape(summary)}</summary>")
|
|
||||||
print(" <metadata_license>CC0-1.0</metadata_license>")
|
|
||||||
print(" <project_license>unknown</project_license>")
|
|
||||||
print(" </component>")
|
|
||||||
print("</components>")
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
# ---------------------------------------------------------------------------
|
||||||
|
# AppStream catalog generation
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def sanitize_id_part(value):
|
||||||
|
part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-")
|
||||||
|
if not part:
|
||||||
|
die(f"package name yields no usable AppStream id segment: {value!r}")
|
||||||
|
if part[0].isdigit():
|
||||||
|
part = "pkg-" + part
|
||||||
|
return part
|
||||||
|
|
||||||
|
|
||||||
|
def as_list(value):
|
||||||
|
if value is None:
|
||||||
|
return []
|
||||||
|
if isinstance(value, (list, tuple)):
|
||||||
|
return list(value)
|
||||||
|
return [value]
|
||||||
|
|
||||||
|
|
||||||
|
def pick(meta, defaults, key, fallback=None):
|
||||||
|
if meta.get(key) is not None:
|
||||||
|
return meta[key]
|
||||||
|
if defaults.get(key) is not None:
|
||||||
|
return defaults[key]
|
||||||
|
return fallback
|
||||||
|
|
||||||
|
|
||||||
|
def read_entries(entries_dir):
|
||||||
|
"""Merge every source's appstream-entries.tsv into {name: summary}."""
|
||||||
|
entries = {}
|
||||||
|
pattern = os.path.join(entries_dir, "*", "appstream-entries.tsv")
|
||||||
|
for path in sorted(glob.glob(pattern)):
|
||||||
|
try:
|
||||||
|
with open(path, encoding="utf-8") as fh:
|
||||||
|
for line in fh:
|
||||||
|
line = line.rstrip("\n")
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
parts = line.split("\t", 1)
|
||||||
|
name = parts[0].strip()
|
||||||
|
summary = parts[1].strip() if len(parts) > 1 else ""
|
||||||
|
if not name:
|
||||||
|
continue
|
||||||
|
if name in entries and entries[name] and summary \
|
||||||
|
and entries[name] != summary:
|
||||||
|
warn(
|
||||||
|
f"package '{name}' was contributed by more than one "
|
||||||
|
"source with differing summaries - keeping the first"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
entries.setdefault(name, summary)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def collect_metadata(yaml_path):
|
||||||
|
"""Return (defaults, {pkgname: metadata}, id_prefix, yaml_origin)."""
|
||||||
|
data = load_yaml(yaml_path)
|
||||||
|
appstream_cfg = data.get("appstream") or {}
|
||||||
|
defaults = dict(appstream_cfg.get("defaults") or {})
|
||||||
|
id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo"
|
||||||
|
yaml_origin = appstream_cfg.get("origin")
|
||||||
|
|
||||||
|
if not defaults.get("developer_id"):
|
||||||
|
segments = id_prefix.split(".")
|
||||||
|
if len(segments) >= 2:
|
||||||
|
defaults["developer_id"] = ".".join(segments[:2])
|
||||||
|
|
||||||
|
by_name = {}
|
||||||
|
for src in normalise_sources(data):
|
||||||
|
for pkg in src["packages"]:
|
||||||
|
name = pkg["name"]
|
||||||
|
if name in by_name:
|
||||||
|
warn(f"package '{name}' listed more than once - keeping the first")
|
||||||
|
continue
|
||||||
|
by_name[name] = pkg
|
||||||
|
return defaults, by_name, id_prefix, yaml_origin
|
||||||
|
|
||||||
|
|
||||||
|
def emit(out, depth, text):
|
||||||
|
out.append(" " * depth + text)
|
||||||
|
|
||||||
|
|
||||||
|
def build_component(pkgname, meta, upstream_summary, defaults, id_prefix, out):
|
||||||
|
if meta.get("appstream") is False or meta.get("hidden") is True:
|
||||||
|
return False
|
||||||
|
|
||||||
|
binaries = as_list(meta.get("binary") or meta.get("binaries"))
|
||||||
|
|
||||||
|
# A console-application component without <provides><binary> trips
|
||||||
|
# appstreamcli's 'console-app-no-binary' check, so only claim that type
|
||||||
|
# when a binary is actually declared. Runtimes and libraries stay
|
||||||
|
# 'generic', which Discover still finds via search.
|
||||||
|
comp_type = pick(meta, defaults, "type")
|
||||||
|
if not comp_type:
|
||||||
|
comp_type = "console-application" if binaries else "generic"
|
||||||
|
|
||||||
|
comp_id = meta.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}"
|
||||||
|
|
||||||
|
display_name = meta.get("display_name") or pkgname
|
||||||
|
|
||||||
|
note = pick(meta, defaults, "note", DEFAULT_NOTE)
|
||||||
|
summary = meta.get("summary") or upstream_summary or note
|
||||||
|
|
||||||
|
emit(out, 1, f"<component type={quoteattr(comp_type)}>")
|
||||||
|
emit(out, 2, f"<id>{escape(comp_id)}</id>")
|
||||||
|
emit(out, 2, f"<pkgname>{escape(pkgname)}</pkgname>")
|
||||||
|
emit(out, 2, f"<name>{escape(display_name)}</name>")
|
||||||
|
emit(out, 2, f"<summary>{escape(summary)}</summary>")
|
||||||
|
|
||||||
|
emit(out, 2, "<description>")
|
||||||
|
body = meta.get("description")
|
||||||
|
wrote_paragraph = False
|
||||||
|
if body:
|
||||||
|
for para in [p.strip() for p in str(body).split("\n\n") if p.strip()]:
|
||||||
|
emit(out, 3, f"<p>{escape(para)}</p>")
|
||||||
|
wrote_paragraph = True
|
||||||
|
if note and (not wrote_paragraph or note != summary):
|
||||||
|
emit(out, 3, f"<p>{escape(note)}</p>")
|
||||||
|
elif not wrote_paragraph:
|
||||||
|
emit(out, 3, f"<p>{escape(summary)}</p>")
|
||||||
|
emit(out, 2, "</description>")
|
||||||
|
|
||||||
|
icon = pick(meta, defaults, "icon", "application-x-executable")
|
||||||
|
if icon:
|
||||||
|
emit(out, 2, f'<icon type="stock">{escape(str(icon))}</icon>')
|
||||||
|
|
||||||
|
categories = as_list(pick(meta, defaults, "categories"))
|
||||||
|
if categories:
|
||||||
|
emit(out, 2, "<categories>")
|
||||||
|
for cat in categories:
|
||||||
|
emit(out, 3, f"<category>{escape(str(cat))}</category>")
|
||||||
|
emit(out, 2, "</categories>")
|
||||||
|
|
||||||
|
keywords = as_list(meta.get("keywords"))
|
||||||
|
if keywords:
|
||||||
|
emit(out, 2, "<keywords>")
|
||||||
|
for kw in keywords:
|
||||||
|
emit(out, 3, f"<keyword>{escape(str(kw))}</keyword>")
|
||||||
|
emit(out, 2, "</keywords>")
|
||||||
|
|
||||||
|
if binaries:
|
||||||
|
emit(out, 2, "<provides>")
|
||||||
|
for binary in binaries:
|
||||||
|
emit(out, 3, f"<binary>{escape(str(binary))}</binary>")
|
||||||
|
emit(out, 2, "</provides>")
|
||||||
|
|
||||||
|
homepage = meta.get("homepage") or defaults.get("homepage")
|
||||||
|
if homepage:
|
||||||
|
emit(out, 2, f'<url type="homepage">{escape(str(homepage))}</url>')
|
||||||
|
help_url = meta.get("help_url") or defaults.get("help_url")
|
||||||
|
if help_url:
|
||||||
|
emit(out, 2, f'<url type="help">{escape(str(help_url))}</url>')
|
||||||
|
|
||||||
|
developer = meta.get("developer") or defaults.get("developer")
|
||||||
|
if developer:
|
||||||
|
dev_id = meta.get("developer_id") or defaults.get("developer_id")
|
||||||
|
attr = f" id={quoteattr(str(dev_id))}" if dev_id else ""
|
||||||
|
emit(out, 2, f"<developer{attr}>")
|
||||||
|
emit(out, 3, f"<name>{escape(str(developer))}</name>")
|
||||||
|
emit(out, 2, "</developer>")
|
||||||
|
|
||||||
|
project_license = meta.get("project_license") or defaults.get("project_license")
|
||||||
|
if project_license:
|
||||||
|
emit(out, 2, f"<project_license>{escape(str(project_license))}</project_license>")
|
||||||
|
|
||||||
|
# A desktop-application component that Discover cannot launch is worse
|
||||||
|
# than a generic one, so insist on the desktop file id.
|
||||||
|
if comp_type == "desktop-application":
|
||||||
|
desktop_id = meta.get("desktop_id")
|
||||||
|
if desktop_id:
|
||||||
|
emit(
|
||||||
|
out, 2,
|
||||||
|
f'<launchable type="desktop-id">{escape(str(desktop_id))}</launchable>',
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
warn(
|
||||||
|
f"{pkgname}: type=desktop-application without 'desktop_id' - "
|
||||||
|
"Discover will not be able to launch it after installation"
|
||||||
|
)
|
||||||
|
|
||||||
|
emit(out, 1, "</component>")
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_gen_appstream(origin, yaml_path, entries_dir):
|
||||||
|
defaults, by_name, id_prefix, yaml_origin = collect_metadata(yaml_path)
|
||||||
|
entries = read_entries(entries_dir)
|
||||||
|
|
||||||
|
if not entries:
|
||||||
|
warn(
|
||||||
|
f"no appstream-entries.tsv found under {entries_dir} - the catalog "
|
||||||
|
"will be empty and nothing will show up in Discover"
|
||||||
|
)
|
||||||
|
|
||||||
|
effective_origin = origin or yaml_origin or "rpm-mirror"
|
||||||
|
|
||||||
|
out = []
|
||||||
|
out.append('<?xml version="1.0" encoding="UTF-8"?>')
|
||||||
|
out.append(
|
||||||
|
f'<components version="{CATALOG_VERSION}" origin={quoteattr(effective_origin)}>'
|
||||||
|
)
|
||||||
|
|
||||||
|
written = 0
|
||||||
|
for pkgname in sorted(entries):
|
||||||
|
meta = by_name.get(pkgname, {})
|
||||||
|
if build_component(pkgname, meta, entries[pkgname], defaults,
|
||||||
|
id_prefix, out):
|
||||||
|
written += 1
|
||||||
|
|
||||||
|
out.append("</components>")
|
||||||
|
sys.stdout.write("\n".join(out) + "\n")
|
||||||
|
sys.stderr.write(
|
||||||
|
f"gen-appstream: wrote {written} component(s), origin={effective_origin}\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||||
|
|
||||||
@@ -90,16 +351,17 @@ def main() -> None:
|
|||||||
p1.add_argument("path")
|
p1.add_argument("path")
|
||||||
|
|
||||||
p2 = sub.add_parser("gen-appstream")
|
p2 = sub.add_parser("gen-appstream")
|
||||||
p2.add_argument("origin")
|
p2.add_argument("--origin", default="")
|
||||||
p2.add_argument("entries_json")
|
p2.add_argument("--yaml", required=True)
|
||||||
|
p2.add_argument("--entries-dir", required=True)
|
||||||
|
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
if args.cmd == "parse-yaml":
|
if args.cmd == "parse-yaml":
|
||||||
cmd_parse_yaml(args.path)
|
cmd_parse_yaml(args.path)
|
||||||
elif args.cmd == "gen-appstream":
|
elif args.cmd == "gen-appstream":
|
||||||
cmd_gen_appstream(args.origin, args.entries_json)
|
cmd_gen_appstream(args.origin, args.yaml, args.entries_dir)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
Reference in New Issue
Block a user