Skip to content

Commit 89253f8

Browse files
committed
Add objects subcommand (CPython 3.9+)
This command allows you to get a list of all GC-registered objects and to query specific atttribute values for debugging.
1 parent dfc47cd commit 89253f8

8 files changed

Lines changed: 432 additions & 8 deletions

File tree

README.md

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ can be installed with ```apk add py-spy --update-cache --repository http://dl-3.
3838
## Usage
3939

4040
py-spy works from the command line and takes either the PID of the program you want to sample from
41-
or the command line of the python program you want to run. py-spy has three subcommands
42-
```record```, ```top``` and ```dump```:
41+
or the command line of the python program you want to run. py-spy has four subcommands
42+
```record```, ```top```, ```dump``` and ```objects```:
4343

4444
### record
4545

@@ -95,6 +95,25 @@ This is useful for the case where you just need a single call stack to figure ou
9595
python program is hung on. This command also has the ability to print out the local variables
9696
associated with each stack frame by setting the ```--locals``` flag.
9797

98+
### objects
99+
100+
py-spy has limited support for object introspection (available for 3.9+). Objects must be
101+
user/library-defined and must be reachable by the GC. `--objectpath` can be used for filtering
102+
by specifying a type name followed by one or more attributes that will be walked.
103+
104+
```bash
105+
# Print all GC-reachable objects
106+
py-spy objects --pid 12345
107+
# Print value attribute of all objects of type Foo
108+
py-spy objects --pid 12345 --objectpath Foo.value
109+
# Print url attribute found in config attribute of all objects of type Foo
110+
py-spy objects --pid 12345 --objectpath Foo.config.url
111+
# Print all dicts
112+
py-spy objects --pid 12345 --objectpath dict
113+
# Inspect a core dump instead of live process
114+
py-spy objects -c core.elf --objectpath dict
115+
```
116+
98117
## Frequently Asked Questions
99118

100119
### Why do we need another Python profiler?

src/config.rs

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ pub struct Config {
6161
pub refresh_seconds: f64,
6262
#[doc(hidden)]
6363
pub core_filename: Option<String>,
64+
#[doc(hidden)]
65+
pub object_path: Option<String>,
6466
}
6567

6668
#[allow(non_camel_case_types)]
@@ -135,6 +137,7 @@ impl Default for Config {
135137
lineno: LineNo::LastInstruction,
136138
refresh_seconds: 1.0,
137139
core_filename: None,
140+
object_path: None,
138141
}
139142
}
140143
}
@@ -331,6 +334,22 @@ impl Config {
331334
.action(ArgAction::SetTrue))
332335
.arg(subprocesses.clone());
333336

337+
let objects = Command::new("objects")
338+
.about("Dumps objects for a target program to stdout")
339+
.arg(Arg::new("core")
340+
.short('c')
341+
.long("core")
342+
.help("Filename of coredump to display python objects from")
343+
.value_name("core")
344+
.action(ArgAction::Set))
345+
.arg(Arg::new("objectpath")
346+
.long("objectpath")
347+
.help("Name of a type or path of an attribute, e.g., MyType._history")
348+
.value_name("objectpath")
349+
.action(ArgAction::Set))
350+
.arg(subprocesses.clone())
351+
.arg(pid.clone().required_unless_present("core"));
352+
334353
let completions = Command::new("completions")
335354
.about("Generate shell completions")
336355
.hide(true)
@@ -353,6 +372,8 @@ impl Config {
353372
let top = top.arg(nonblocking.clone());
354373
#[cfg(not(target_os = "freebsd"))]
355374
let dump = dump.arg(nonblocking.clone());
375+
#[cfg(not(target_os = "freebsd"))]
376+
let objects = objects.arg(nonblocking.clone());
356377

357378
let styles = Styles::styled()
358379
.header(AnsiColor::Yellow.on_default())
@@ -370,6 +391,7 @@ impl Config {
370391
.subcommand(record)
371392
.subcommand(top)
372393
.subcommand(dump)
394+
.subcommand(objects)
373395
.subcommand(completions);
374396
let matches = app.clone().try_get_matches_from(args)?;
375397
debug!("Command line args: {:?}", matches);
@@ -379,7 +401,7 @@ impl Config {
379401
let (subcommand, matches) = matches.subcommand().unwrap();
380402

381403
// Check if `--native` was used on an unsupported platform
382-
if subcommand != "completions" && !cfg!(feature = "unwind") && matches.get_flag("native") {
404+
if subcommand != "completions" && subcommand != "objects" && !cfg!(feature = "unwind") && matches.get_flag("native") {
383405
eprintln!(
384406
"Collecting stack traces from native extensions (`--native`) is not supported on your platform."
385407
);
@@ -425,6 +447,10 @@ impl Config {
425447
config.core_filename = matches.get_one("core").cloned();
426448
}
427449
}
450+
"objects" => {
451+
config.core_filename = matches.get_one("core").cloned();
452+
config.object_path = matches.get_one("objectpath").cloned();
453+
}
428454
"completions" => {
429455
let shell = matches.get_one::<clap_complete::Shell>("shell").unwrap();
430456
let app_name = app.get_name().to_string();
@@ -457,7 +483,9 @@ impl Config {
457483
}
458484
});
459485

460-
config.full_filenames = matches.get_flag("full_filenames");
486+
if subcommand != "objects" {
487+
config.full_filenames = matches.get_flag("full_filenames");
488+
}
461489
if cfg!(feature = "unwind") {
462490
config.native = matches.get_flag("native");
463491
}

src/coredump.rs

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,13 @@ use crate::python_bindings::{
1919
v3_9_5,
2020
};
2121
use crate::python_data_access::format_variable;
22-
use crate::python_interpreters::InterpreterState;
22+
use crate::python_interpreters::{InterpreterState, HasGcGenerations};
2323
use crate::python_process_info::{
2424
get_interpreter_address, get_python_version, get_threadstate_address, is_python_lib,
2525
ContainsAddr, PythonProcessInfo,
2626
};
2727
use crate::python_threading::thread_names_from_interpreter;
28+
use crate::obj_walk::{walk_gc, get_object_attribute};
2829
use crate::stack_trace::{get_stack_traces, StackTrace};
2930
use crate::version::Version;
3031

@@ -396,6 +397,70 @@ impl PythonCoreDump {
396397
}
397398
Ok(())
398399
}
400+
401+
pub fn print_objects(&self, config: &Config) -> Result<(), Error> {
402+
match self.version {
403+
Version {
404+
major: 3, minor: 9, ..
405+
} => self._print_objects::<v3_9_5::_is>(config),
406+
Version {
407+
major: 3, minor: 10, ..
408+
} => self._print_objects::<v3_10_0::_is>(config),
409+
Version {
410+
major: 3, minor: 11, ..
411+
} => self._print_objects::<v3_11_0::_is>(config),
412+
Version {
413+
major: 3, minor: 12, ..
414+
} => self._print_objects::<v3_12_0::_is>(config),
415+
Version {
416+
major: 3, minor: 13, ..
417+
} => self._print_objects::<v3_13_0::_is>(config),
418+
Version {
419+
major: 3, minor: 14, ..
420+
} => self._print_objects::<v3_14_0::_is>(config),
421+
_ => Err(format_err!(
422+
"Unsupported version of Python: {}",
423+
self.version
424+
)),
425+
}
426+
}
427+
428+
fn _print_objects<I>(&self, config: &Config) -> Result<(), Error>
429+
where
430+
I: HasGcGenerations,
431+
{
432+
let (type_name, attrs): (Option<&str>, Vec<&str>) =
433+
match config.object_path.as_deref() {
434+
Some(s) => {
435+
let mut parts = s.split('.');
436+
let first = parts.next();
437+
let rest = parts.collect();
438+
(first, rest)
439+
}
440+
None => (None, Vec::new()),
441+
};
442+
443+
let objects = walk_gc::<I, CoreDump>(self.interpreter_address, type_name, &self.core)?;
444+
println!("Found {} object(s)", objects.len());
445+
446+
if !attrs.is_empty() {
447+
for obj in objects {
448+
println!("<{} at 0x{:x}>:", type_name.unwrap(), obj);
449+
match get_object_attribute::<I, CoreDump>(obj, &attrs, &self.core, &self.version) {
450+
Ok(attr) => println!("{}", attr),
451+
Err(err) => println!("{}", err),
452+
}
453+
}
454+
} else {
455+
for obj in objects {
456+
if let Ok(formatted) = format_variable::<I, CoreDump>(&self.core, &self.version, obj, 200) {
457+
println!("{}", formatted);
458+
}
459+
}
460+
}
461+
462+
Ok(())
463+
}
399464
}
400465

401466
mod elfcore {

src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ mod cython;
3939
pub mod dump;
4040
#[cfg(feature = "unwind")]
4141
mod native_stack_trace;
42+
mod obj_walk;
4243
mod python_bindings;
4344
mod python_data_access;
4445
mod python_interpreters;

src/main.rs

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ mod dump;
1515
mod flamegraph;
1616
#[cfg(feature = "unwind")]
1717
mod native_stack_trace;
18+
mod obj_walk;
1819
mod python_bindings;
1920
mod python_data_access;
2021
mod python_interpreters;
@@ -371,6 +372,10 @@ fn run_spy_command(pid: remoteprocess::Pid, config: &config::Config) -> Result<(
371372
"dump" => {
372373
dump::print_traces(pid, config, None)?;
373374
}
375+
"objects" => {
376+
let process = python_spy::PythonSpy::new(pid, config)?;
377+
process.print_objects()?;
378+
}
374379
"record" => {
375380
record_samples(pid, config)?;
376381
}
@@ -401,8 +406,18 @@ fn pyspy_main() -> Result<(), Error> {
401406
{
402407
if let Some(ref core_filename) = config.core_filename {
403408
let core = coredump::PythonCoreDump::new(std::path::Path::new(&core_filename))?;
404-
let traces = core.get_stack(&config)?;
405-
return core.print_traces(&traces, &config);
409+
match config.command.as_ref() {
410+
"dump" => {
411+
let traces = core.get_stack(&config)?;
412+
return core.print_traces(&traces, &config);
413+
}
414+
"objects" => {
415+
return core.print_objects(&config);
416+
}
417+
_ => {
418+
return Err(format_err!("Unknown command {}", config.command));
419+
}
420+
}
406421
}
407422
}
408423

src/obj_walk.rs

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
use anyhow::{Context, Error, Result};
2+
3+
use remoteprocess::ProcessMemory;
4+
5+
use crate::python_data_access::{format_variable, PY_TPFLAGS_MANAGED_DICT, DictIterator, copy_string};
6+
use crate::python_interpreters::{
7+
GcHead, HasGcGenerations, InterpreterState, Object, TypeObject,
8+
};
9+
use crate::version::Version;
10+
11+
pub fn walk_gc<I, P>(addr: usize, type_name: Option<&str>, process: &P) -> Result<Vec<usize>, Error>
12+
where
13+
I: HasGcGenerations,
14+
P: ProcessMemory,
15+
{
16+
let mut ret = Vec::new();
17+
18+
for gen in I::gc_generations(addr) {
19+
let head_addr = I::generation_head_addr(gen);
20+
let mut gc: I::GcHead = process.copy_pointer(head_addr).context("Failed to copy gen gc head")?;
21+
let mut gc_addr = gc.next();
22+
23+
while gc_addr as *const _ != head_addr as *const _ {
24+
gc = process.copy_pointer(gc_addr).context("Failed to copy gchead")?;
25+
26+
let obj_addr = gc.obj_addr(gc_addr as usize);
27+
28+
let should_add = match type_name {
29+
Some(name) => isinstance::<I, P>(obj_addr, name, process).unwrap_or(false),
30+
None => true,
31+
};
32+
if should_add {
33+
ret.push(obj_addr);
34+
}
35+
36+
gc_addr = gc.next();
37+
}
38+
}
39+
Ok(ret)
40+
}
41+
42+
fn isinstance<I, P>(addr: usize, type_name: &str, process: &P) -> Result<bool, Error>
43+
where
44+
I: InterpreterState,
45+
P: ProcessMemory,
46+
{
47+
let value: I::Object = process.copy_struct(addr)?;
48+
let value_type = process.copy_pointer(value.ob_type())?;
49+
50+
// get the typename (truncating to 128 bytes if longer)
51+
let max_type_len = 128;
52+
let value_type_name = process.copy(value_type.name() as usize, max_type_len)?;
53+
let length = value_type_name
54+
.iter()
55+
.position(|&x| x == 0)
56+
.unwrap_or(max_type_len);
57+
let value_type_name = std::str::from_utf8(&value_type_name[..length])?;
58+
59+
Ok(value_type_name == type_name)
60+
}
61+
62+
pub fn get_object_attribute<I, P>(addr: usize, path: &Vec<&str>, process: &P, version: &Version) -> Result<String, Error>
63+
where
64+
I: InterpreterState,
65+
P: ProcessMemory,
66+
{
67+
if path.len() == 0 {
68+
return Err(format_err!("Path passed to get_object_attribute must not be empty"));
69+
}
70+
71+
let mut cur_addr = addr;
72+
for elem in path {
73+
match resolve_attribute::<I, P>(cur_addr, elem, process, version) {
74+
Ok(resolved) => cur_addr = resolved,
75+
Err(e) => {
76+
println!("Unable to resolve attribute: {}", e);
77+
return Err(e);
78+
}
79+
};
80+
}
81+
82+
format_variable::<I, P>(process, version, cur_addr, 16384)
83+
}
84+
85+
pub fn resolve_attribute<I, P>(addr: usize, name: &str, process: &P, version: &Version) -> Result<usize, Error>
86+
where
87+
I: InterpreterState,
88+
P: ProcessMemory,
89+
{
90+
let value: I::Object = process.copy_struct(addr)?;
91+
let value_type = process.copy_pointer(value.ob_type())?;
92+
93+
let flags = value_type.flags();
94+
let dict_iter = if flags & PY_TPFLAGS_MANAGED_DICT != 0 {
95+
DictIterator::from_managed_dict(
96+
process,
97+
version,
98+
addr,
99+
value.ob_type() as usize,
100+
flags,
101+
)?
102+
} else {
103+
let dict_offset = value_type.dictoffset();
104+
let dict_addr = (addr as isize + dict_offset) as usize;
105+
let thread_dict_addr: usize = process.copy_struct(dict_addr)?;
106+
DictIterator::from(process, version, thread_dict_addr)?
107+
};
108+
109+
for i in dict_iter {
110+
let (key, value) = i?;
111+
let varname = copy_string(key as *const I::StringObject, process)?;
112+
if varname == name {
113+
return Ok(value);
114+
}
115+
}
116+
Err(format_err!("Attribute {} not found", name))
117+
}

0 commit comments

Comments
 (0)