From 67e5c714a1a395dbb0d8264033fd449f1510c057 Mon Sep 17 00:00:00 2001 From: "David M. Klein" <4+leander19961@noreply.localhost> Date: Thu, 27 Aug 2026 13:46:50 +0200 Subject: [PATCH] =?UTF-8?q?docker/syncer/sync=5Fhelpers.py=20gel=C3=B6scht?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/syncer/sync_helpers.py | 367 ---------------------------------- 1 file changed, 367 deletions(-) delete mode 100644 docker/syncer/sync_helpers.py diff --git a/docker/syncer/sync_helpers.py b/docker/syncer/sync_helpers.py deleted file mode 100644 index 0cd96f8..0000000 --- a/docker/syncer/sync_helpers.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/env python3 -""" -sync_helpers.py - small helper used by sync.sh - -Reliable YAML parsing and XML escaping in pure bash/sed is a losing game, so -these two jobs live here: - - parse-yaml - Reads the "sources:" list from the YAML file, NORMALISES it and prints - it as JSON on stdout, so sync.sh can loop over it safely. - - Normalisation: every entry under "packages:" becomes an object with at - least a "name" key, whether it was written as a plain string or as a - mapping with extra AppStream metadata. sync.sh can therefore always do - p["name"] without caring which form the YAML used. - - gen-appstream --origin O --yaml F --entries-dir D - Builds the AppStream catalog for the combined repo. - - Metadata precedence, highest first: - 1. what packages.yaml says about the package - 2. the upstream RPM summary collected by sync.sh into the - appstream-entries.tsv files under --entries-dir - 3. a generated fallback - - Only packages that actually made it into the mirror (i.e. that appear - in one of the TSV files) get a component - never pulled-in - dependencies, and never packages whose download failed. - - NOTE ON DISCOVER/GNOME SOFTWARE: - Components without an icon/screenshots/.desktop file make a package - installable and findable via SEARCH, but they will not appear on the - curated "Explore" front page or when browsing categories. That is a - frontend behaviour for non-desktop-app components and cannot be fixed - at the repo-metadata level. Only a real desktop-application component - (type: desktop-application + desktop_id: in packages.yaml, which - requires the package to ship a .desktop file) shows up there. -""" -import argparse -import glob -import json -import os -import re -import sys -from xml.sax.saxutils import escape, quoteattr - -CATALOG_VERSION = "0.14" -DEFAULT_NOTE = ( - "Bereitgestellt über das Unternehmens-Repo der IT-Abteilung. " - "Fragen und Paketwünsche bitte an den IT-Service-Desk." -) - - -def die(msg): - sys.stdout.flush() - sys.stderr.write(f"ERROR: {msg}\n") - sys.exit(1) - - -def warn(msg): - sys.stdout.flush() - sys.stderr.write(f"WARNING: {msg}\n") - - -def load_yaml(path): - try: - import yaml - except ImportError: - die( - "python3-pyyaml (PyYAML) is not installed.\n" - "Install it with: dnf install -y python3-pyyaml" - ) - try: - with open(path, "r", encoding="utf-8") as fh: - return yaml.safe_load(fh) or {} - except OSError as exc: - die(f"could not read {path!r}: {exc}") - except Exception as exc: # yaml.YAMLError and friends - die(f"could not parse {path!r}: {exc}") - - -def normalise_package(entry, source_name, index): - """Accept both 'pkgname' and {name: pkgname, display_name: ...}.""" - if isinstance(entry, str): - return {"name": entry} - if isinstance(entry, dict): - if not entry.get("name"): - die( - f"package #{index} in source {source_name!r} has no 'name' key: " - f"{entry!r}" - ) - return dict(entry) - die( - f"package #{index} in source {source_name!r} must be a string or a " - f"mapping, got {type(entry).__name__}" - ) - - -def normalise_sources(data): - sources = data.get("sources") or [] - if not isinstance(sources, list): - die("'sources' in YAML is not a list") - - for i, src in enumerate(sources): - if not isinstance(src, dict): - die(f"source #{i} is not a mapping") - for field in ("name", "baseurl", "packages"): - if field not in src: - die(f"source #{i} is missing required field '{field}'") - if not isinstance(src["packages"], list) or not src["packages"]: - die( - f"source '{src.get('name')}' has an empty/invalid packages list" - ) - src["packages"] = [ - normalise_package(p, src.get("name"), j) - for j, p in enumerate(src["packages"]) - ] - return sources - - -def cmd_parse_yaml(path): - data = load_yaml(path) - json.dump(normalise_sources(data), sys.stdout) - - -# --------------------------------------------------------------------------- -# AppStream catalog generation -# --------------------------------------------------------------------------- - -def sanitize_id_part(value): - part = re.sub(r"[^A-Za-z0-9]+", "-", value).strip("-") - if not part: - die(f"package name yields no usable AppStream id segment: {value!r}") - if part[0].isdigit(): - part = "pkg-" + part - return part - - -def as_list(value): - if value is None: - return [] - if isinstance(value, (list, tuple)): - return list(value) - return [value] - - -def pick(meta, defaults, key, fallback=None): - if meta.get(key) is not None: - return meta[key] - if defaults.get(key) is not None: - return defaults[key] - return fallback - - -def read_entries(entries_dir): - """Merge every source's appstream-entries.tsv into {name: summary}.""" - entries = {} - pattern = os.path.join(entries_dir, "*", "appstream-entries.tsv") - for path in sorted(glob.glob(pattern)): - try: - with open(path, encoding="utf-8") as fh: - for line in fh: - line = line.rstrip("\n") - if not line: - continue - parts = line.split("\t", 1) - name = parts[0].strip() - summary = parts[1].strip() if len(parts) > 1 else "" - if not name: - continue - if name in entries and entries[name] and summary \ - and entries[name] != summary: - warn( - f"package '{name}' was contributed by more than one " - "source with differing summaries - keeping the first" - ) - continue - entries.setdefault(name, summary) - except OSError: - continue - return entries - - -def collect_metadata(yaml_path): - """Return (defaults, {pkgname: metadata}, id_prefix, yaml_origin).""" - data = load_yaml(yaml_path) - appstream_cfg = data.get("appstream") or {} - defaults = dict(appstream_cfg.get("defaults") or {}) - id_prefix = appstream_cfg.get("id_prefix") or "de.firma.repo" - yaml_origin = appstream_cfg.get("origin") - - if not defaults.get("developer_id"): - segments = id_prefix.split(".") - if len(segments) >= 2: - defaults["developer_id"] = ".".join(segments[:2]) - - by_name = {} - for src in normalise_sources(data): - for pkg in src["packages"]: - name = pkg["name"] - if name in by_name: - warn(f"package '{name}' listed more than once - keeping the first") - continue - by_name[name] = pkg - return defaults, by_name, id_prefix, yaml_origin - - -def emit(out, depth, text): - out.append(" " * depth + text) - - -def build_component(pkgname, meta, upstream_summary, defaults, id_prefix, out): - if meta.get("appstream") is False or meta.get("hidden") is True: - return False - - binaries = as_list(meta.get("binary") or meta.get("binaries")) - - # A console-application component without trips - # appstreamcli's 'console-app-no-binary' check, so only claim that type - # when a binary is actually declared. Runtimes and libraries stay - # 'generic', which Discover still finds via search. - comp_type = pick(meta, defaults, "type") - if not comp_type: - comp_type = "console-application" if binaries else "generic" - - comp_id = meta.get("id") or f"{id_prefix}.{sanitize_id_part(pkgname)}" - - display_name = meta.get("display_name") or pkgname - - note = pick(meta, defaults, "note", DEFAULT_NOTE) - summary = meta.get("summary") or upstream_summary or note - - emit(out, 1, f"") - emit(out, 2, f"{escape(comp_id)}") - emit(out, 2, f"{escape(pkgname)}") - emit(out, 2, f"{escape(display_name)}") - emit(out, 2, f"{escape(summary)}") - - emit(out, 2, "") - body = meta.get("description") - wrote_paragraph = False - if body: - for para in [p.strip() for p in str(body).split("\n\n") if p.strip()]: - emit(out, 3, f"

{escape(para)}

") - wrote_paragraph = True - if note and (not wrote_paragraph or note != summary): - emit(out, 3, f"

{escape(note)}

") - elif not wrote_paragraph: - emit(out, 3, f"

{escape(summary)}

") - emit(out, 2, "") - - icon = pick(meta, defaults, "icon", "application-x-executable") - if icon: - emit(out, 2, f'{escape(str(icon))}') - - categories = as_list(pick(meta, defaults, "categories")) - if categories: - emit(out, 2, "") - for cat in categories: - emit(out, 3, f"{escape(str(cat))}") - emit(out, 2, "") - - keywords = as_list(meta.get("keywords")) - if keywords: - emit(out, 2, "") - for kw in keywords: - emit(out, 3, f"{escape(str(kw))}") - emit(out, 2, "") - - if binaries: - emit(out, 2, "") - for binary in binaries: - emit(out, 3, f"{escape(str(binary))}") - emit(out, 2, "") - - homepage = meta.get("homepage") or defaults.get("homepage") - if homepage: - emit(out, 2, f'{escape(str(homepage))}') - help_url = meta.get("help_url") or defaults.get("help_url") - if help_url: - emit(out, 2, f'{escape(str(help_url))}') - - developer = meta.get("developer") or defaults.get("developer") - if developer: - dev_id = meta.get("developer_id") or defaults.get("developer_id") - attr = f" id={quoteattr(str(dev_id))}" if dev_id else "" - emit(out, 2, f"") - emit(out, 3, f"{escape(str(developer))}") - emit(out, 2, "") - - project_license = meta.get("project_license") or defaults.get("project_license") - if project_license: - emit(out, 2, f"{escape(str(project_license))}") - - # A desktop-application component that Discover cannot launch is worse - # than a generic one, so insist on the desktop file id. - if comp_type == "desktop-application": - desktop_id = meta.get("desktop_id") - if desktop_id: - emit( - out, 2, - f'{escape(str(desktop_id))}', - ) - else: - warn( - f"{pkgname}: type=desktop-application without 'desktop_id' - " - "Discover will not be able to launch it after installation" - ) - - emit(out, 1, "") - return True - - -def cmd_gen_appstream(origin, yaml_path, entries_dir): - defaults, by_name, id_prefix, yaml_origin = collect_metadata(yaml_path) - entries = read_entries(entries_dir) - - if not entries: - warn( - f"no appstream-entries.tsv found under {entries_dir} - the catalog " - "will be empty and nothing will show up in Discover" - ) - - effective_origin = origin or yaml_origin or "rpm-mirror" - - out = [] - out.append('') - out.append( - f'' - ) - - written = 0 - for pkgname in sorted(entries): - meta = by_name.get(pkgname, {}) - if build_component(pkgname, meta, entries[pkgname], defaults, - id_prefix, out): - written += 1 - - out.append("") - sys.stdout.write("\n".join(out) + "\n") - sys.stderr.write( - f"gen-appstream: wrote {written} component(s), origin={effective_origin}\n" - ) - - -def main(): - ap = argparse.ArgumentParser() - sub = ap.add_subparsers(dest="cmd", required=True) - - p1 = sub.add_parser("parse-yaml") - p1.add_argument("path") - - p2 = sub.add_parser("gen-appstream") - p2.add_argument("--origin", default="") - p2.add_argument("--yaml", required=True) - p2.add_argument("--entries-dir", required=True) - - args = ap.parse_args() - - if args.cmd == "parse-yaml": - cmd_parse_yaml(args.path) - elif args.cmd == "gen-appstream": - cmd_gen_appstream(args.origin, args.yaml, args.entries_dir) - - -if __name__ == "__main__": - main() \ No newline at end of file