#!/usr/bin/env python # update-plist.py - Adds and removes lines in pkg-plist(s) based on # poudriere error logs. # Inspired by: https://codeberg.org/tcberner/poudlist, by tcberner@, # but this just requires base Python and doesn't use a config file. # ---------------------------------------------------------------------------- # "THE BEER-WARE LICENSE" (Revision 42, (c) Poul-Henning Kamp): # Jason E. Hale wrote this file. As long as you retain # this notice you can do whatever you want with this stuff. If we meet some # day, and you think this stuff is worth it, you can buy me a beer in return. # # Jason E. Hale # ---------------------------------------------------------------------------- # MAINTAINER= jhale@FreeBSD.org # SYNOPSIS # update-plist.py [-h] [-E] [-p ] # DESCRIPTION # Adds and removes lines in pkg-plist(s) based on poudriere error logs. # It is recommend to run this tool after building a port with all options # enabled with either `poudriere bulk -t` or `poudriere testport` as it # will prune all stale plist entires, including those prefixed by '%%', # but it does recognize where it has already been run and won't try to # duplicate the previous effort. # Use ports-mgmt/hs-panopticum for a more in-depth and CPU-intensive # analysis than this little script could ever provide, but with some # human brains behind it, using this takes a fraction of the time for # most ports. # You will still need to manually verify that all of the PLIST_SUBs # are correct. This tool follows the poudriere logs to a tee, but # sometimes the PLIST_SUBs for a port are more aggressive than they # really should be as generated by 'make makeplist'. # OPTIONS # -h, --help print helps and then exits # # -p, --portsdir path to the ports tree to operate upon # Overrides the PORTSDIR environment variable # if defined, required otherwise. # # -E, --extended (experimental) try to work with ports with no pkg-plist # or more complicated pkg-plist structures (PLIST+=foo). # Writes pkg-plist..add~ and pkg-plist..rm~ # files to the port directory manual intervention instead of # skipping. The appended '~' is meant to prevent accidental # commital of these temporary files. # ENVIRONMENT # PORTSDIR path to the ports tree to operate upon # EXAMPLES # update-plist.py /usr/local/poudriere/data/logs/bulk/150amd64-area51/2026-03-19_22h36m33s/logs/errors # update-plist.py -p ~/src/area51/ports /usr/local/poudriere/data/logs/bulk/150amd64-area51/2026-03-19_22h36m33s/logs/errors/qt6-webengine-6.11.0.log # BUGS # - Assumes that the existing pkg-plist is already somewhat properly sorted. import sys import re import os import argparse import subprocess def extract_plist_sub(log_content): sub_map = {} match = re.search(r"--PLIST_SUB--\n(.*?)\n--End PLIST_SUB--", log_content, re.DOTALL) if match: raw_subs = match.group(1) pairs = re.findall(r'(\w+)=("(?:[^"\\]|\\.)*"|[^\s]+)', raw_subs) for key, val in pairs: clean_val = val.strip('"') sub_map[f"%%{key}%%"] = "" if "@comment" in clean_val else clean_val return sub_map def get_expanded_key(line, sub_map): expanded = line.strip() for macro in sorted(sub_map.keys(), key=len, reverse=True): if macro in expanded: expanded = expanded.replace(macro, sub_map[macro]).replace("//", "/") clean = re.sub(r"%%.*?%%", "", expanded) clean = re.sub(r"@\w+\s+", "", clean) return clean.strip().lstrip('/') def get_plist_paths(ports_tree_root, origin): """Uses make to resolve all PLIST paths, handling multiple space-separated plists.""" port_dir = os.path.join(ports_tree_root, origin) try: result = subprocess.check_output( ['make', '-C', port_dir, '-V', 'PLIST'], text=True, stderr=subprocess.DEVNULL ).strip() if result: paths = result.split() return [p if os.path.isabs(p) else os.path.join(port_dir, p) for p in paths] except Exception: pass return [os.path.join(port_dir, "pkg-plist")] def process_log(log_path, ports_tree_root, extended=False): try: with open(log_path, 'r', encoding='utf-8', errors='ignore') as f: first_lines = [] for _ in range(15): line = f.readline() if not line: break first_lines.append(line) f.seek(0) log_content = f.read() except Exception as e: sys.stderr.write(f"![Error] Could not read {log_path}: {e}\n") return if "Error: Plist issues found" not in log_content: return origin = None for line in first_lines: if "port directory:" in line: parts = line.split(":", 1)[1].strip().rstrip('/').split('/') if len(parts) >= 2: origin = os.path.join(parts[-2], parts[-1]) break if not origin: return plist_paths = get_plist_paths(ports_tree_root, origin) existing_plists = [p for p in plist_paths if os.path.exists(p)] is_fallback_mode = False fallback_rm_path = None # Handle the case where no valid plists exist on disk if not existing_plists: if not extended: sys.stderr.write(f"![Skip] {origin}: No valid plist files found (port likely relies on PLIST_FILES). Try using with the '-E' flag.\n") return else: is_fallback_mode = True log_basename = os.path.basename(log_path) if log_basename.endswith(".log"): log_basename = log_basename[:-4] fallback_add_name = f"pkg-plist.{log_basename}.add~" fallback_rm_name = f"pkg-plist.{log_basename}.rm~" fallback_add_path = os.path.join(ports_tree_root, origin, fallback_add_name) fallback_rm_path = os.path.join(ports_tree_root, origin, fallback_rm_name) sys.stderr.write(f"-> Warning: {origin} has no existing plist. Manual intervention required.\n") existing_plists = [fallback_add_path] plist_data = {} all_existing_lines = set() for path in existing_plists: if os.path.exists(path): with open(path, 'r', encoding='utf-8') as f: lines = [l.strip() for l in f.readlines() if l.strip()] plist_data[path] = lines all_existing_lines.update(lines) else: # For the newly created pkg-plist.*.add~ plist_data[path] = [] sub_map = extract_plist_sub(log_content) orphans = re.findall(r"Error: Orphaned: (.*)", log_content) missing_from_log = set(re.findall(r"Error: Missing: (.*)", log_content)) removed_count = 0 added_count = 0 if is_fallback_mode: # We don't have a source plist to remove from, so dump the missing lines to the rm file if missing_from_log: plist_data[fallback_rm_path] = sorted(list(missing_from_log)) removed_count = len(missing_from_log) else: for path, lines in plist_data.items(): new_lines = [] for line in lines: clean_line = re.sub(r"%%.*?%%", "", line).strip().lstrip('/') if line in missing_from_log or clean_line in missing_from_log: removed_count += 1 continue new_lines.append(line) plist_data[path] = new_lines # Target the FIRST file in our list for orphans (either the base plist or the unique .add~ file) base_plist = existing_plists[0] for orphan in sorted(orphans, key=lambda x: get_expanded_key(x, sub_map)): if orphan in all_existing_lines: continue orphan_key = get_expanded_key(orphan, sub_map) inserted = False for i, line in enumerate(plist_data[base_plist]): if get_expanded_key(line, sub_map) > orphan_key: plist_data[base_plist].insert(i, orphan) inserted = True added_count += 1 break if not inserted: plist_data[base_plist].append(orphan) added_count += 1 all_existing_lines.add(orphan) if added_count == 0 and removed_count == 0: sys.stderr.write(f"-> {origin}: Already up to date.\n") return # Suppress the multi-plist warning if we are writing to a fallback intervention file if len(existing_plists) > 1 and added_count > 0 and "pkg-plist.add" not in base_plist: base_name = os.path.basename(base_plist) sys.stderr.write(f"-> Warning: {origin} uses multiple plists. Orphans appended to {base_name}. Please verify manually.\n") sys.stderr.write(f"-> Updating {origin}... Done (+{added_count}/-{removed_count})\n") for path, lines in plist_data.items(): with open(path, 'w', encoding='utf-8') as f: f.write("\n".join(lines) + "\n") if __name__ == "__main__": env_portsdir = os.environ.get("PORTSDIR") is_required = not bool(env_portsdir) parser = argparse.ArgumentParser(description="Updates pkg-plist based on poudriere error logs.") parser.add_argument( "logpath", help="Poudriere log file or directory of logs" ) parser.add_argument( "-p", "--portsdir", required=is_required, default=env_portsdir, help="Path to ports tree to operate upon (Overrides PORTSDIR env var if set)" ) parser.add_argument( "-E", "--extended", action="store_true", help="(experimental) If a port has no pkg-plist or a more complicated pkg-plist structure, write orphaned/missing files to pkg-plist..add~ and pkg-plist..rm~ for manual review instead of skipping" ) args = parser.parse_args() if os.path.isdir(args.logpath): for filename in sorted(os.listdir(args.logpath)): if filename.endswith(".log"): process_log(os.path.join(args.logpath, filename), args.portsdir, args.extended) else: process_log(args.logpath, args.portsdir, args.extended)