diff --git a/zfs_probe.py b/zfs_probe.py index 8743ceb..8138706 100644 --- a/zfs_probe.py +++ b/zfs_probe.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# example: python3 zfs_probe.py --duration 10 --interval 1 --track-files +# example: python3 zfs_probe.py --duration 120 --interval 5 --track-files --path-prefix /home import argparse import ast @@ -223,6 +223,110 @@ def zfs_get_properties(pools): return rows + + +def get_zfs_mountpoints(): + """ + Zwraca mountpointy filesystemów typu zfs widoczne w namespace procesu. + /proc/self/mountinfo opisuje mounty w bieżącym mount namespace. + """ + mountinfo = "/proc/self/mountinfo" + mounts = [] + text = safe_read_text(mountinfo) + for line in text.splitlines(): + line = line.strip() + if not line or " - " not in line: + continue + left, right = line.split(" - ", 1) + lparts = left.split() + rparts = right.split() + if len(lparts) < 5 or len(rparts) < 1: + continue + fstype = rparts[0] + mountpoint = lparts[4] + if fstype == "zfs": + mounts.append(mountpoint) + + # najdłuższe prefiksy najpierw, bez duplikatów + uniq = sorted(set(mounts), key=lambda x: (-len(x), x)) + return uniq + + +def path_is_under_mountpoints(path, mountpoints): + if not path or not path.startswith("/"): + return False + norm = os.path.normpath(path) + for mp in mountpoints: + if mp == "/": + return True + if norm == mp or norm.startswith(mp.rstrip("/") + "/"): + return True + return False + + +def filter_bpf_opens_to_zfs(opens_map, mountpoints): + kept = {} + dropped_relative = 0 + dropped_non_zfs = 0 + + for path, value in opens_map.items(): + if not isinstance(path, str): + continue + if not path.startswith("/"): + dropped_relative += value + continue + if path_is_under_mountpoints(path, mountpoints): + kept[path] = kept.get(path, 0) + value + else: + dropped_non_zfs += value + + return kept, dropped_relative, dropped_non_zfs + + +def normalize_prefixes(prefix_arg): + prefixes = [] + for item in (prefix_arg or "").split(","): + item = item.strip() + if not item: + continue + norm = os.path.normpath(item) + if not norm.startswith("/"): + continue + prefixes.append(norm) + return sorted(set(prefixes), key=lambda x: (-len(x), x)) + + +def path_matches_prefixes(path, prefixes): + if not path or not path.startswith("/"): + return False + norm = os.path.normpath(path) + for prefix in prefixes: + if prefix == "/": + return True + if norm == prefix or norm.startswith(prefix.rstrip("/") + "/"): + return True + return False + + +def filter_bpf_opens_to_prefixes(opens_map, prefixes): + kept = {} + dropped_relative = 0 + dropped_outside = 0 + + for path, value in opens_map.items(): + if not isinstance(path, str): + continue + if not path.startswith("/"): + dropped_relative += value + continue + if path_matches_prefixes(path, prefixes): + kept[path] = kept.get(path, 0) + value + else: + dropped_outside += value + + return kept, dropped_relative, dropped_outside + + def parse_arcstats(): path = "/proc/spl/kstat/zfs/arcstats" if not os.path.exists(path): @@ -257,11 +361,6 @@ def arc_delta(prev, curr): def parse_zpool_iostat_once(pool_list, interval): - """ - Działa z różnymi wersjami zpool iostat. - Próbuje najpierw z -l, potem bez -l. - Nie zakłada obecności timestampa. - """ candidate_cmds = [ ["zpool", "iostat", "-H", "-p", "-y", "-l"] + pool_list + [str(interval), "1"], ["zpool", "iostat", "-H", "-p", "-y"] + pool_list + [str(interval), "1"], @@ -362,30 +461,75 @@ def top_entries(entries, key, topn): return vals[:topn] +def parse_bpftrace_version(text): + m = re.search(r'v?(\d+)\.(\d+)(?:\.(\d+))?', text) + if not m: + return None + major = int(m.group(1)) + minor = int(m.group(2)) + patch = int(m.group(3) or 0) + return (major, minor, patch) + + +def choose_bpftrace_field_style(): + """ + bpftrace 0.19 zmienił składnię z args->field na args.field. + Dla 0.16 używamy starej składni, dla nowszych nowych wersji - nowej. + """ + if not command_exists("bpftrace"): + return None, None, "bpftrace pominięty: brak polecenia bpftrace." + + rc, out, err = run_cmd(["bpftrace", "--version"], timeout=10) + text = (out or err or "").strip() + version = parse_bpftrace_version(text) + if version is None: + return None, text, "bpftrace wykryty, ale nie udało się odczytać wersji z: {0}".format(text) + + if version < (0, 19, 0): + return "arrow", text, None + return "dot", text, None + + +def build_bpftrace_program(track_files=False, field_style="dot"): + if field_style == "arrow": + ret_expr = "args->ret" + bytes_expr = "args->bytes" + filename_expr = "str(args->filename)" + else: + ret_expr = "args.ret" + bytes_expr = "args.bytes" + filename_expr = "str(args.filename)" + + lines = [] + lines.append("tracepoint:syscalls:sys_exit_read /{0} > 0/ {{ @read_bytes_by_comm[comm] = sum({0}); @read_calls_by_comm[comm] = count(); }}".format(ret_expr)) + lines.append("tracepoint:block:block_rq_issue {{ @block_bytes_by_comm[comm] = sum({0}); @block_ios_by_comm[comm] = count(); }}".format(bytes_expr)) + if track_files: + lines.append("tracepoint:syscalls:sys_enter_openat {{ @opens[{0}] = count(); @opens_by_comm[comm, {0}] = count(); }}".format(filename_expr)) + lines.append("END {") + lines.append(' printf("===READ_BYTES_BY_COMM===\\n"); print(@read_bytes_by_comm);') + lines.append(' printf("===READ_CALLS_BY_COMM===\\n"); print(@read_calls_by_comm);') + lines.append(' printf("===BLOCK_BYTES_BY_COMM===\\n"); print(@block_bytes_by_comm);') + lines.append(' printf("===BLOCK_IOS_BY_COMM===\\n"); print(@block_ios_by_comm);') + if track_files: + lines.append(' printf("===OPENS===\\n"); print(@opens);') + lines.append(' printf("===OPENS_BY_COMM===\\n"); print(@opens_by_comm);') + lines.append("}") + return "\n".join(lines) + "\n" + + def start_bpftrace(outdir, track_files=False): if os.geteuid() != 0: return None, "bpftrace pominięty: uruchom jako root." - if not command_exists("bpftrace"): - return None, "bpftrace pominięty: brak polecenia bpftrace." - program = [] - program.append('tracepoint:syscalls:sys_exit_read /args.ret > 0/ { @read_bytes_by_comm[comm] = sum(args.ret); @read_calls_by_comm[comm] = count(); }') - program.append('tracepoint:block:block_rq_issue { @block_bytes_by_comm[comm] = sum(args.bytes); @block_ios_by_comm[comm] = count(); }') - if track_files: - program.append('tracepoint:syscalls:sys_enter_openat { @opens[str(args.filename)] = count(); @opens_by_comm[comm, str(args.filename)] = count(); }') - program.append('END {') - program.append(' printf("===READ_BYTES_BY_COMM===\\n"); print(@read_bytes_by_comm);') - program.append(' printf("===READ_CALLS_BY_COMM===\\n"); print(@read_calls_by_comm);') - program.append(' printf("===BLOCK_BYTES_BY_COMM===\\n"); print(@block_bytes_by_comm);') - program.append(' printf("===BLOCK_IOS_BY_COMM===\\n"); print(@block_ios_by_comm);') - if track_files: - program.append(' printf("===OPENS===\\n"); print(@opens);') - program.append(' printf("===OPENS_BY_COMM===\\n"); print(@opens_by_comm);') - program.append('}') + field_style, version_text, version_note = choose_bpftrace_field_style() + if version_note: + return None, version_note + + program = build_bpftrace_program(track_files=track_files, field_style=field_style) bt_path = os.path.join(outdir, "trace.bt") with open(bt_path, "w") as f: - f.write("\n".join(program) + "\n") + f.write(program) out_path = os.path.join(outdir, "bpftrace.txt") out_f = open(out_path, "w") @@ -395,7 +539,19 @@ def start_bpftrace(outdir, track_files=False): stderr=subprocess.STDOUT, universal_newlines=True, ) - return {"proc": proc, "out_f": out_f, "out_path": out_path, "bt_path": bt_path}, None + + note = "bpftrace: {0}, skladnia: {1}".format( + version_text if version_text else "unknown", + "args->field" if field_style == "arrow" else "args.field", + ) + return { + "proc": proc, + "out_f": out_f, + "out_path": out_path, + "bt_path": bt_path, + "version_text": version_text, + "field_style": field_style, + }, note SECTION_RE = re.compile(r"^===([A-Z0-9_]+)===$") @@ -461,7 +617,10 @@ def stop_bpftrace(handle): return proc = handle["proc"] if proc.poll() is not None: - handle["out_f"].close() + try: + handle["out_f"].close() + except Exception: + pass return try: @@ -491,6 +650,8 @@ def main(): ap.add_argument("--top", type=int, default=15, help="Ile pozycji pokazać w topkach") ap.add_argument("--outdir", default="", help="Katalog na logi i JSON") ap.add_argument("--track-files", action="store_true", help="Śledź top otwierane pliki przez bpftrace") + ap.add_argument("--zfs-files-only", action="store_true", help="W topce plików pokazuj tylko absolutne ścieżki leżące na mountpointach ZFS") + ap.add_argument("--path-prefix", default="", help="Pokaż tylko pliki spod tego prefiksu lub kilku prefiksów rozdzielonych przecinkami, np. /home/tank,/mnt/data") ap.add_argument("--no-bpf", action="store_true", help="Nie uruchamiaj bpftrace nawet gdy jest dostępny") args = ap.parse_args() @@ -512,6 +673,12 @@ def main(): sys.exit(1) oprint("Pule: {0}".format(", ".join(pools))) + zfs_mountpoints = get_zfs_mountpoints() + if zfs_mountpoints: + oprint("Mountpointy ZFS: {0}".format(", ".join(zfs_mountpoints))) + path_prefixes = normalize_prefixes(args.path_prefix) + if path_prefixes: + oprint("Filtr ścieżek: {0}".format(", ".join(path_prefixes))) meta = { "started_at": datetime.now().isoformat(), "duration": args.duration, @@ -519,6 +686,8 @@ def main(): "pools": pools, "hostname": os.uname().nodename, "uid": os.geteuid(), + "zfs_mountpoints": zfs_mountpoints, + "path_prefixes": path_prefixes, } with open(os.path.join(outdir, "meta.json"), "w") as f: json.dump(meta, f, indent=2) @@ -529,6 +698,8 @@ def main(): f.write(zpool_history_text(pools)) with open(os.path.join(outdir, "zfs_properties.json"), "w") as f: json.dump(zfs_get_properties(pools), f, indent=2) + with open(os.path.join(outdir, "zfs_mountpoints.json"), "w") as f: + json.dump(zfs_mountpoints, f, indent=2) bpf_handle = None bpf_note = None @@ -618,6 +789,9 @@ def main(): oprint() oprint("Wyniki zapisane w: {0}".format(outdir)) oprint("Pule: {0}".format(", ".join(pools))) + zfs_mountpoints = get_zfs_mountpoints() + if zfs_mountpoints: + oprint("Mountpointy ZFS: {0}".format(", ".join(zfs_mountpoints))) oprint("Czas zbierania: {0}s, interwał: {1}s, próbek: {2}".format( args.duration, args.interval, sum(len(v) for v in samples_by_pool.values()) )) @@ -701,10 +875,30 @@ def main(): opens = bpf.get("OPENS", {}) if opens: + report_opens = opens + title = "Top otwierane pliki z bpftrace" + + if path_prefixes: + report_opens, dropped_relative, dropped_outside = filter_bpf_opens_to_prefixes(opens, path_prefixes) + title = "Top otwierane pliki z bpftrace dla wskazanych ścieżek" + oprint("Filtrowanie plików po prefiksie:") + oprint(" zostawione wpisy: {0}".format(sum(report_opens.values()) if report_opens else 0)) + oprint(" odrzucone ścieżki względne: {0}".format(dropped_relative)) + oprint(" odrzucone poza prefiksem: {0}".format(dropped_outside)) + oprint() + elif args.zfs_files_only: + report_opens, dropped_relative, dropped_non_zfs = filter_bpf_opens_to_zfs(opens, zfs_mountpoints) + title = "Top otwierane pliki na ZFS z bpftrace" + oprint("Filtrowanie plików:") + oprint(" zostawione wpisy na ZFS: {0}".format(sum(report_opens.values()) if report_opens else 0)) + oprint(" odrzucone ścieżki względne: {0}".format(dropped_relative)) + oprint(" odrzucone ścieżki poza ZFS: {0}".format(dropped_non_zfs)) + oprint() + rows = [] - for path, value in sorted(opens.items(), key=lambda kv: kv[1], reverse=True)[:args.top]: + for path, value in sorted(report_opens.items(), key=lambda kv: kv[1], reverse=True)[:args.top]: rows.append({"path": path, "opens": value}) - print_table("Top otwierane pliki z bpftrace", rows, [ + print_table(title, rows, [ ("path", "path"), ("opens", "opens"), ]) @@ -715,10 +909,12 @@ def main(): "zpool_status_end.txt", "zpool_history.txt", "zfs_properties.json", + "zfs_mountpoints.json", "samples_zpool.json", "samples_arc.json", "proc_totals.json", "bpftrace.txt", + "trace.bt", ]: path = os.path.join(outdir, name) if os.path.exists(path):