|
| 1 | +# License: GPL2 or later see COPYING |
| 2 | +""" |
| 3 | +Track various system statistics during the build phase. |
| 4 | +- Report maximum allocated memory in total and for the "biggest" process |
| 5 | +
|
| 6 | +The plugin requires systemd-nspawn as build container runner |
| 7 | +""" |
| 8 | + |
| 9 | +import threading |
| 10 | +import os |
| 11 | +import json |
| 12 | +import backoff |
| 13 | + |
| 14 | +from mockbuild.trace_decorator import getLog |
| 15 | +from mockbuild.util import get_machinectl_uuid, _safe_check_output, USE_NSPAWN |
| 16 | + |
| 17 | +requires_api_version = "1.1" |
| 18 | +run_in_bootstrap = False |
| 19 | + |
| 20 | +def init(plugins, conf, buildroot): |
| 21 | + """ Plugin entry point """ |
| 22 | + SystemMonitor(plugins, conf, buildroot) |
| 23 | + |
| 24 | +class SystemMonitor: |
| 25 | + """ Main plugin class """ |
| 26 | + |
| 27 | + def get_top_process_info(self, scope_path): |
| 28 | + """Finds the process in the cgroup with the highest RSS. |
| 29 | +
|
| 30 | + Args: |
| 31 | + scope_path (str): Path to the cgroup scope directory. |
| 32 | +
|
| 33 | + Returns: |
| 34 | + tuple[int, str]: A tuple containing the RSS of the top process in bytes |
| 35 | + and its command line. |
| 36 | + """ |
| 37 | + procs_path = os.path.join(scope_path, "cgroup.procs") |
| 38 | + max_rss = 0 |
| 39 | + max_cmdline = "" |
| 40 | + |
| 41 | + try: |
| 42 | + if not os.path.exists(procs_path): |
| 43 | + return (0, "unknown") |
| 44 | + with open(procs_path, 'r', encoding="utf-8") as f: |
| 45 | + pids = f.read().split() |
| 46 | + |
| 47 | + for pid in pids: |
| 48 | + try: |
| 49 | + with open(f"/proc/{pid}/statm", 'r', encoding="utf-8") as sm: |
| 50 | + data = sm.read().split() |
| 51 | + if not data: |
| 52 | + continue |
| 53 | + # RSS in pages * PAGE_SIZE bytes |
| 54 | + rss_bytes = int(data[1]) * os.sysconf('SC_PAGE_SIZE') |
| 55 | + |
| 56 | + if rss_bytes > max_rss: |
| 57 | + max_rss = rss_bytes |
| 58 | + with open(f"/proc/{pid}/cmdline", 'r', encoding="utf-8") as cmd: |
| 59 | + max_cmdline = cmd.read().replace('\0', ' ').strip() |
| 60 | + if not max_cmdline: |
| 61 | + max_cmdline = f"{pid}" |
| 62 | + |
| 63 | + except (FileNotFoundError, ProcessLookupError, IndexError): |
| 64 | + continue |
| 65 | + except OSError as e: |
| 66 | + getLog().error("SYSMON: Error: %s", e) |
| 67 | + |
| 68 | + return (max_rss, max_cmdline) |
| 69 | + |
| 70 | + @backoff.on_predicate(backoff.constant, jitter=None, interval=2, max_time=120) |
| 71 | + def get_machine_id(self, buildroot): |
| 72 | + """ Retry getting machine id until nspawn starts """ |
| 73 | + return get_machinectl_uuid(buildroot.make_chroot_path()) |
| 74 | + |
| 75 | + def sysmon_thread(self, buildroot, interval): |
| 76 | + """ Main monitoring thread """ |
| 77 | + current_peak = 0 |
| 78 | + |
| 79 | + machine_id = self.get_machine_id(buildroot) |
| 80 | + if machine_id is None: |
| 81 | + getLog().error("SYSMON: Failed to get nspawn container machine_id") |
| 82 | + return |
| 83 | + |
| 84 | + getLog().debug("SYSMON: Collecting data from machine_id: %s", machine_id) |
| 85 | + |
| 86 | + # The unit name depends on systemd version |
| 87 | + ustr = _safe_check_output(["/bin/machinectl", "show", "--property=Unit", f"{machine_id}"]) |
| 88 | + if isinstance(ustr, bytes): |
| 89 | + ustr = ustr.decode("utf-8") |
| 90 | + machine_id_unit = ustr.rstrip().split('=')[1] |
| 91 | + scope_dir = f"/sys/fs/cgroup/machine.slice/{machine_id_unit}" |
| 92 | + peak_file = os.path.join(scope_dir, "memory.peak") |
| 93 | + |
| 94 | + while not self.sysmon_stop_event.is_set(): |
| 95 | + max_status = "Current" |
| 96 | + pid_status = "Current" |
| 97 | + |
| 98 | + try: |
| 99 | + if os.path.exists(peak_file): |
| 100 | + with open(peak_file, 'r', encoding="utf-8") as f: |
| 101 | + current_peak = int(f.read().strip()) |
| 102 | + |
| 103 | + if current_peak > self.max_memory_peak: |
| 104 | + max_status = "NEW" |
| 105 | + self.max_memory_peak = current_peak |
| 106 | + |
| 107 | + top = self.get_top_process_info(f"{scope_dir}/payload") |
| 108 | + if top[0] > self.top_rss[0]: |
| 109 | + pid_status = "NEW" |
| 110 | + self.top_rss = top |
| 111 | + |
| 112 | + if f"{max_status}{pid_status}" != "CurrentCurrent": |
| 113 | + getLog().debug( |
| 114 | + "SYSMON: %s PEAK %.2f MiB | SYSMON: %s Top Process: RSS:%.2f MiB [%s]", |
| 115 | + max_status, self.max_memory_peak / 1048576, |
| 116 | + pid_status, self.top_rss[0] / 1048576, self.top_rss[1] |
| 117 | + ) |
| 118 | + except (IOError, ValueError): |
| 119 | + getLog().debug("SYSMON: memory.peak missing %s", scope_dir) |
| 120 | + |
| 121 | + if self.sysmon_stop_event.wait(timeout=interval): |
| 122 | + break |
| 123 | + |
| 124 | + def _on_postdeps(self): |
| 125 | + # Inject nspawn args |
| 126 | + nspawn_args = self.config.get('nspawn_args', []) |
| 127 | + prop = '--property=MemoryAccounting=on' |
| 128 | + if prop not in nspawn_args: |
| 129 | + nspawn_args.append(prop) |
| 130 | + self.config['nspawn_args'] = nspawn_args |
| 131 | + |
| 132 | + # Start thread |
| 133 | + interval = self.system_monitor_opts.get('interval', 2) |
| 134 | + self.sysmon_stop_event.clear() |
| 135 | + self.sysmon_timer_thread = threading.Thread(target=self.sysmon_thread, |
| 136 | + args=(self.buildroot, interval), |
| 137 | + daemon=True) |
| 138 | + self.sysmon_timer_thread.start() |
| 139 | + |
| 140 | + getLog().debug("SYSMON: Monitoring thread started via callback.") |
| 141 | + |
| 142 | + def _on_postbuild(self): |
| 143 | + self.sysmon_stop_event.set() |
| 144 | + self.sysmon_timer_thread.join() |
| 145 | + getLog().info( |
| 146 | + "SYSMON: Total Memory Peak %.2f MiB | Top process: RSS:%.2f MiB [%s]", |
| 147 | + self.max_memory_peak / 1048576, self.top_rss[0] / 1048576, self.top_rss[1] |
| 148 | + ) |
| 149 | + out_file = os.path.join(self.buildroot.resultdir, 'system_monior.json') |
| 150 | + with open(out_file, 'w', encoding="utf-8") as f: |
| 151 | + json.dump({"total_max_memory" : self.max_memory_peak, |
| 152 | + "top_process_memory" : self.top_rss[0], |
| 153 | + "top_process_cmdline" : self.top_rss[1]}, |
| 154 | + f) |
| 155 | + |
| 156 | + def __init__(self, plugins, conf, buildroot): |
| 157 | + self.max_memory_peak = 0 |
| 158 | + self.top_rss = (0, "") |
| 159 | + self.sysmon_timer_thread = None |
| 160 | + self.sysmon_stop_event = threading.Event() |
| 161 | + self.buildroot = buildroot |
| 162 | + self.system_monitor_opts = conf |
| 163 | + self.config = buildroot.config |
| 164 | + |
| 165 | + if not USE_NSPAWN: |
| 166 | + getLog().warning("SYSMON: build is not using nspawn. Statistics will not be available") |
| 167 | + return |
| 168 | + |
| 169 | + getLog().info("SYSMON: Starting system monitor") |
| 170 | + plugins.add_hook("postdeps", self._on_postdeps) |
| 171 | + plugins.add_hook("postbuild", self._on_postbuild) |
0 commit comments