diff --git a/docker/sync_helpers.py b/docker/sync_helpers.py index 338e44e..0cd96f8 100644 --- a/docker/sync_helpers.py +++ b/docker/sync_helpers.py @@ -2,87 +2,348 @@ """ sync_helpers.py - small helper used by sync.sh -Two jobs, on purpose kept out of bash because reliable YAML parsing and XML -escaping in pure bash/sed is a losing game: +Reliable YAML parsing and XML escaping in pure bash/sed is a losing game, so +these two jobs live here: parse-yaml - Reads the "sources:" list from the YAML file and prints it as JSON - on stdout, so sync.sh can loop over it safely (no word-splitting - issues, handles quoting etc.). + Reads the "sources:" list from the YAML file, NORMALISES it and prints + it as JSON on stdout, so sync.sh can loop over it safely. - gen-appstream - entries_json is a JSON array of {"name":..., "summary":...} objects - (only the *explicitly requested* top-level packages, never pulled-in - dependencies). Prints a minimal AppStream "generic component" catalog - on stdout, which sync.sh gzips and injects into the repo metadata via - modifyrepo_c. + Normalisation: every entry under "packages:" becomes an object with at + least a "name" key, whether it was written as a plain string or as a + mapping with extra AppStream metadata. sync.sh can therefore always do + p["name"] without caring which form the YAML used. - NOTE: "generic" components (no icon/screenshots/.desktop file) make a - package installable from GNOME Software / Discover and show up in - search, but will NOT appear on the curated "Explore" front page - - that's a GNOME Software behaviour for non-desktop-app components, not - something fixable at the repo-metadata level. + gen-appstream --origin O --yaml F --entries-dir D + Builds the AppStream catalog for the combined repo. + + Metadata precedence, highest first: + 1. what packages.yaml says about the package + 2. the upstream RPM summary collected by sync.sh into the + appstream-entries.tsv files under --entries-dir + 3. a generated fallback + + Only packages that actually made it into the mirror (i.e. that appear + in one of the TSV files) get a component - never pulled-in + dependencies, and never packages whose download failed. + + NOTE ON DISCOVER/GNOME SOFTWARE: + Components without an icon/screenshots/.desktop file make a package + installable and findable via SEARCH, but they will not appear on the + curated "Explore" front page or when browsing categories. That is a + frontend behaviour for non-desktop-app components and cannot be fixed + at the repo-metadata level. Only a real desktop-application component + (type: desktop-application + desktop_id: in packages.yaml, which + requires the package to ship a .desktop file) shows up there. """ -import sys -import json import argparse -from xml.sax.saxutils import escape +import glob +import json +import os +import re +import sys +from xml.sax.saxutils import escape, quoteattr + +CATALOG_VERSION = "0.14" +DEFAULT_NOTE = ( + "Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. " + "Fragen und Paketwünsche bitte an den IT-Service-Desk." +) -def cmd_parse_yaml(path: str) -> None: +def die(msg): + sys.stdout.flush() + sys.stderr.write(f"ERROR: {msg}\n") + sys.exit(1) + + +def warn(msg): + sys.stdout.flush() + sys.stderr.write(f"WARNING: {msg}\n") + + +def load_yaml(path): try: import yaml except ImportError: - sys.stderr.write( - "ERROR: python3-pyyaml (PyYAML) is not installed.\n" - "Install it with: dnf install -y python3-pyyaml\n" + die( + "python3-pyyaml (PyYAML) is not installed.\n" + "Install it with: dnf install -y python3-pyyaml" ) - sys.exit(1) + try: + with open(path, "r", encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + except OSError as exc: + die(f"could not read {path!r}: {exc}") + except Exception as exc: # yaml.YAMLError and friends + die(f"could not parse {path!r}: {exc}") - with open(path, "r", encoding="utf-8") as fh: - data = yaml.safe_load(fh) or {} +def normalise_package(entry, source_name, index): + """Accept both 'pkgname' and {name: pkgname, display_name: ...}.""" + if isinstance(entry, str): + return {"name": entry} + if isinstance(entry, dict): + if not entry.get("name"): + die( + f"package #{index} in source {source_name!r} has no 'name' key: " + f"{entry!r}" + ) + return dict(entry) + die( + f"package #{index} in source {source_name!r} must be a string or a " + f"mapping, got {type(entry).__name__}" + ) + + +def normalise_sources(data): sources = data.get("sources") or [] if not isinstance(sources, list): - sys.stderr.write("ERROR: 'sources' in YAML is not a list\n") - sys.exit(1) + die("'sources' in YAML is not a list") for i, src in enumerate(sources): + if not isinstance(src, dict): + die(f"source #{i} is not a mapping") for field in ("name", "baseurl", "packages"): if field not in src: - sys.stderr.write( - f"ERROR: source #{i} is missing required field '{field}'\n" - ) - sys.exit(1) + die(f"source #{i} is missing required field '{field}'") if not isinstance(src["packages"], list) or not src["packages"]: - sys.stderr.write( - f"ERROR: source '{src.get('name')}' has an empty/invalid packages list\n" + die( + f"source '{src.get('name')}' has an empty/invalid packages list" ) - sys.exit(1) - - json.dump(sources, sys.stdout) + src["packages"] = [ + normalise_package(p, src.get("name"), j) + for j, p in enumerate(src["packages"]) + ] + return sources -def cmd_gen_appstream(origin: str, entries_json: str) -> None: - entries = json.loads(entries_json) - print('') - print(f'') - for e in entries: - name = e.get("name", "") - summary = e.get("summary") or f"{name} (mirrored package)" - cid = f"{name}.generic" - print(' ') - print(f" {escape(cid)}") - print(f" {escape(name)}") - print(f" {escape(name)}") - print(f" {escape(summary)}") - print(" CC0-1.0") - print(" unknown") - print(" ") - print("") +def cmd_parse_yaml(path): + data = load_yaml(path) + json.dump(normalise_sources(data), sys.stdout) -def main() -> None: +# --------------------------------------------------------------------------- +# AppStream catalog generation +# --------------------------------------------------------------------------- + +def sanitize_id_part(value): + part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-") + if not part: + die(f"package name yields no usable AppStream id segment: {value!r}") + if part[0].isdigit(): + part = "pkg-" + part + return part + + +def as_list(value): + if value is None: + return [] + if isinstance(value, (list, tuple)): + return list(value) + return [value] + + +def pick(meta, defaults, key, fallback=None): + if meta.get(key) is not None: + return meta[key] + if defaults.get(key) is not None: + return defaults[key] + return fallback + + +def read_entries(entries_dir): + """Merge every source's appstream-entries.tsv into {name: summary}.""" + entries = {} + pattern = os.path.join(entries_dir, "*", "appstream-entries.tsv") + for path in sorted(glob.glob(pattern)): + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.rstrip("\n") + if not line: + continue + parts = line.split("\t", 1) + name = parts[0].strip() + summary = parts[1].strip() if len(parts) > 1 else "" + if not name: + continue + if name in entries and entries[name] and summary \ + and entries[name] != summary: + warn( + f"package '{name}' was contributed by more than one " + "source with differing summaries - keeping the first" + ) + continue + entries.setdefault(name, summary) + except OSError: + continue + return entries + + +def collect_metadata(yaml_path): + """Return (defaults, {pkgname: metadata}, id_prefix, yaml_origin).""" + data = load_yaml(yaml_path) + appstream_cfg = data.get("appstream") or {} + defaults = dict(appstream_cfg.get("defaults") or {}) + id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo" + yaml_origin = appstream_cfg.get("origin") + + if not defaults.get("developer_id"): + segments = id_prefix.split(".") + if len(segments) >= 2: + defaults["developer_id"] = ".".join(segments[:2]) + + by_name = {} + for src in normalise_sources(data): + for pkg in src["packages"]: + name = pkg["name"] + if name in by_name: + warn(f"package '{name}' listed more than once - keeping the first") + continue + by_name[name] = pkg + return defaults, by_name, id_prefix, yaml_origin + + +def emit(out, depth, text): + out.append(" " * depth + text) + + +def build_component(pkgname, meta, upstream_summary, defaults, id_prefix, out): + if meta.get("appstream") is False or meta.get("hidden") is True: + return False + + binaries = as_list(meta.get("binary") or meta.get("binaries")) + + # A console-application component without trips + # appstreamcli's 'console-app-no-binary' check, so only claim that type + # when a binary is actually declared. Runtimes and libraries stay + # 'generic', which Discover still finds via search. + comp_type = pick(meta, defaults, "type") + if not comp_type: + comp_type = "console-application" if binaries else "generic" + + comp_id = meta.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}" + + display_name = meta.get("display_name") or pkgname + + note = pick(meta, defaults, "note", DEFAULT_NOTE) + summary = meta.get("summary") or upstream_summary or note + + emit(out, 1, f"") + emit(out, 2, f"{escape(comp_id)}") + emit(out, 2, f"{escape(pkgname)}") + emit(out, 2, f"{escape(display_name)}") + emit(out, 2, f"{escape(summary)}") + + emit(out, 2, "") + body = meta.get("description") + wrote_paragraph = False + if body: + for para in [p.strip() for p in str(body).split("\n\n") if p.strip()]: + emit(out, 3, f"

{escape(para)}

") + wrote_paragraph = True + if note and (not wrote_paragraph or note != summary): + emit(out, 3, f"

{escape(note)}

") + elif not wrote_paragraph: + emit(out, 3, f"

{escape(summary)}

") + emit(out, 2, "") + + icon = pick(meta, defaults, "icon", "application-x-executable") + if icon: + emit(out, 2, f'{escape(str(icon))}') + + categories = as_list(pick(meta, defaults, "categories")) + if categories: + emit(out, 2, "") + for cat in categories: + emit(out, 3, f"{escape(str(cat))}") + emit(out, 2, "") + + keywords = as_list(meta.get("keywords")) + if keywords: + emit(out, 2, "") + for kw in keywords: + emit(out, 3, f"{escape(str(kw))}") + emit(out, 2, "") + + if binaries: + emit(out, 2, "") + for binary in binaries: + emit(out, 3, f"{escape(str(binary))}") + emit(out, 2, "") + + homepage = meta.get("homepage") or defaults.get("homepage") + if homepage: + emit(out, 2, f'{escape(str(homepage))}') + help_url = meta.get("help_url") or defaults.get("help_url") + if help_url: + emit(out, 2, f'{escape(str(help_url))}') + + developer = meta.get("developer") or defaults.get("developer") + if developer: + dev_id = meta.get("developer_id") or defaults.get("developer_id") + attr = f" id={quoteattr(str(dev_id))}" if dev_id else "" + emit(out, 2, f"") + emit(out, 3, f"{escape(str(developer))}") + emit(out, 2, "") + + project_license = meta.get("project_license") or defaults.get("project_license") + if project_license: + emit(out, 2, f"{escape(str(project_license))}") + + # A desktop-application component that Discover cannot launch is worse + # than a generic one, so insist on the desktop file id. + if comp_type == "desktop-application": + desktop_id = meta.get("desktop_id") + if desktop_id: + emit( + out, 2, + f'{escape(str(desktop_id))}', + ) + else: + warn( + f"{pkgname}: type=desktop-application without 'desktop_id' - " + "Discover will not be able to launch it after installation" + ) + + emit(out, 1, "") + return True + + +def cmd_gen_appstream(origin, yaml_path, entries_dir): + defaults, by_name, id_prefix, yaml_origin = collect_metadata(yaml_path) + entries = read_entries(entries_dir) + + if not entries: + warn( + f"no appstream-entries.tsv found under {entries_dir} - the catalog " + "will be empty and nothing will show up in Discover" + ) + + effective_origin = origin or yaml_origin or "rpm-mirror" + + out = [] + out.append('') + out.append( + f'' + ) + + written = 0 + for pkgname in sorted(entries): + meta = by_name.get(pkgname, {}) + if build_component(pkgname, meta, entries[pkgname], defaults, + id_prefix, out): + written += 1 + + out.append("") + sys.stdout.write("\n".join(out) + "\n") + sys.stderr.write( + f"gen-appstream: wrote {written} component(s), origin={effective_origin}\n" + ) + + +def main(): ap = argparse.ArgumentParser() sub = ap.add_subparsers(dest="cmd", required=True) @@ -90,16 +351,17 @@ def main() -> None: p1.add_argument("path") p2 = sub.add_parser("gen-appstream") - p2.add_argument("origin") - p2.add_argument("entries_json") + p2.add_argument("--origin", default="") + p2.add_argument("--yaml", required=True) + p2.add_argument("--entries-dir", required=True) args = ap.parse_args() if args.cmd == "parse-yaml": cmd_parse_yaml(args.path) elif args.cmd == "gen-appstream": - cmd_gen_appstream(args.origin, args.entries_json) + cmd_gen_appstream(args.origin, args.yaml, args.entries_dir) if __name__ == "__main__": - main() + main() \ No newline at end of file