6

Kernel Debugging

 Decoding an OOPS
List debugging
Memory debugging
Locking debugging
Profiling

Kernel oops
[ 378.675775] BUG: unable to handle kernel paging request at 6b6b6b6b
[ 378.681969] IP: [<c149dbcd>] rfcomm_process_dlcs+0x1b/0x15e
[ 378.687535] *pdpt = 000000002d67c001 *pde = 0000000000000000
[ 378.693272] Oops: 0000 [#1] PREEMPT SMP
[ 378.697188] Modules linked in: wl12xx_compat_sdio(C) btwilink wl12xx_compat(C) mac80211_compat(C)
cfg80211_compat(C) compat(C) fm_drv st_drv fuse snd_soc_mfld_machine snd_soc_sn95031 snd_soc_sst_platform
atomisp lm3554 mt9m114 mt9e013 videobuf_vmalloc videobuf_core atmel_mxt_ts pn544_nxp
[ 378.722657]
[ 378.724140] Pid: 946, comm: krfcommd Tainted: G C 3.0.16-mid8-dirty #34
[ 378.731702] EIP: 0060:[<c149dbcd>] EFLAGS: 00010246 CPU: 0
[ 378.737180] EIP is at rfcomm_process_dlcs+0x1b/0x15e
[ 378.742135] EAX: eb89d8c8 EBX: 6b6b6b6b ECX: eb89d8c8 EDX: eb89d544
[ 378.748394] ESI: eb89d4f0 EDI: ed4f9f70 EBP: ed4f9f68 ESP: ed4f9f50
[ 378.754656] DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
[ 378.760047] Process krfcommd (pid: 946, ti=ed4f8000 task=ed4265e0 task.ti=ed4f8000)
[ 378.767692] Stack:
[ 378.769694] c15302c8 eb89d584 00000000 eb89d4f0 ed519920 ed4f9f70 ed4f9f80 c149eb4d
[ 378.777429] 00000000 ed4265e0 00000000 ed4f9f90 ed4f9f9c c149ebc4 0000bf54 00000000
[ 378.785168] 00000000 ee03bf54 c149eb74 ed4f9fe4 c104fe01 00000000 00000000 00000000
[ 378.792908] Call Trace:
[ 378.795351] [<c149eb4d>] rfcomm_process_sessions+0xb7/0xde
[ 378.800912] [<c149ebc4>] rfcomm_run+0x50/0x6c
[ 378.805344] [<c149eb74>] ? rfcomm_process_sessions+0xde/0xde
[ 378.811085] [<c104fe01>] kthread+0x63/0x68
[ 378.815258] [<c104fd9e>] ? __init_kthread_worker+0x42/0x42
[ 378.820824] [<c14dad82>] kernel_thread_helper+0x6/0xd
[ 378.825943] Code: 89 d8 e8 0a fe ff ff 8d 65 f8 31 c0 5b 5e 5d c3 55 89 e5 57 56 89 c6 53 8d 64 24 f4 8b 98
d8 03 00 00 8d 56 54 8d 80 d8 03 00 00 <8b> 3b 89 45 f0 89 55 ec e9 24 01 00 00 8b 93 a0 00 00 00 8d 83
[ 378.844811] EIP: [<c149dbcd>] rfcomm_process_dlcs+0x1b/0x15e SS:ESP 0068:ed4f9f50
[ 378.852287] CR2: 000000006b6b6b6b

Fault & EIP address, process

BUG: unable to handle kernel paging request at 6b6b6b6b
IP: [<c149dbcd>] rfcomm_process_dlcs+0x1b/0x15e
*pdpt = 000000002d67c001 *pde = 0000000000000000
Oops: 0000 [#1] PREEMPT SMP
Modules linked in: wl12xx_compat_sdio(C) btwilink wl12xx_compat(C)
Pid: 946, comm: krfcommd Tainted: G C 3.0.16-mid8-dirty #34
EIP: 0060:[<c149dbcd>] EFLAGS: 00010246 CPU: 0
EIP is at rfcomm_process_dlcs+0x1b/0x15e
EAX: eb89d8c8 EBX: 6b6b6b6b ECX: eb89d8c8 EDX: eb89d544
ESI: eb89d4f0 EDI: ed4f9f70 EBP: ed4f9f68 ESP: ed4f9f50
DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
Process krfcommd (pid: 946, ti=ed4f8000 task=ed4265e0 task.ti=ed4f8000)

Stack trace
Call Trace:
[<c149eb4d>] rfcomm_process_sessions+0xb7/0xde
[<c149ebc4>] rfcomm_run+0x50/0x6c
[<c149eb74>] ? rfcomm_process_sessions+0xde/0xde
[<c104fe01>] kthread+0x63/0x68
[<c104fd9e>] ? __init_kthread_worker+0x42/0x42
[<c14dad82>] kernel_thread_helper+0x6/0xd

Translatarea unei adrese n informaii fiier:linie

Avem nevoie de informaii de debug

(CONFIG_DEBUG_INFO)
addr2line
gdb
objdump -dSr

addr2line
$ addr2line -ie vmlinux
0xc149d76d
/home/tavi/src/linux/net/bluetooth/rfcomm/core.c:1866

i este util atunci cnd funciile sunt inlined de

ctre compilator
se va afia att locaia n funcia inclus ct i
locaia n funcia ce include funcia inclus

Informaii despre surs folosid gdb i adresa EIP

$ (gdb) l *(0xc149d76d)
0xc149d76d is in rfcomm_process_dlcs
(/home/tavi/src/linux/net/bluetooth/rfcomm/core.c:1866).
warning: Source file is more recent than executable.
1861 struct list_head *p, *n;
1862
1863 BT_DBG("session %p state %ld", s, s->state);
1864
1865 list_for_each_safe(p, n, &s->dlcs) {
1866 d = list_entry(p, struct rfcomm_dlc, list);
1867
1868 if (test_bit(RFCOMM_TIMED_OUT, &d->flags)) {
1869

__rfcomm_dlc_close(d, ETIMEDOUT);

1870

continue;

Probleme cauzate de module

Modulele sunt ncrcate la adrese dinamice dar sunt
compilate pentru adresa 0
1185.368428] BUG: unable to handle kernel paging request at 00100108
[ 1185.374653] IP: [<f27db050>] crush_it+0x50/0x75 [crusher]
[ 1185.380043] *pdpt = 0000000024588001 *pde = 0000000000000000
[ 1185.385781] Oops: 0000 [#1] PREEMPT SMP
[ 1185.389698] Modules linked in: crusher(P+) wl12xx_compat_sdio(C)
btwilink wl12xx_compat(C) mac80211_compat(C) cfg80211_compat(C) compat(C)
fm_drv st_drv fuse snd_soc_mfld_machine snd_sop
[ 1185.416208]
[ 1185.417697] Pid: 2581, comm: insmod Tainted: P C 3.0.16-mid8-dirty #46
[ 1185.425171] EIP: 0060:[<f27db050>] EFLAGS: 00010217 CPU: 0
[ 1185.430652] EIP is at crush_it+0x50/0x75 [crusher]
[ 1185.435431] EAX: f27db0a4 EBX: f27db0c0 ECX: e45bdee4 EDX: 00000001
[ 1185.441692] ESI: f27db0c0 EDI: 00100100 EBP: e45bdef4 ESP: e45bded8
[ 1185.447949] DS: 007b ES: 007b FS: 00d8 GS: 00e0 SS: 0068
[ 1185.453340] Process insmod (pid: 2581, ti=e45bc000 task=edb565e0 task.

Adresa de ncrcare a modulelor

root@android:/ # cat /proc/modules
btwilink 2838 0 - Live 0xf2832000
wl12xx_compat 137273 0 - Live 0xf27d3000 (C)
mac80211_compat 170167 1 wl12xx_compat, Live 0xf2513000 (C)
compat 1073 0 - Live 0xf1c5d000 (C)
fm_drv 23893 0 - Live 0xf1b2d000
st_drv 14201 2 btwilink,fm_drv, Live 0xf13f9000
fuse 56553 2 - Live 0xf1acb000
snd_soc_mfld_machine 6643 0 - Live 0xefe77000

Informaii despre surs cu gdb + funcie i offset

$ (gdb) l *(crush_it+0x50)
0x74 is in crush_it (/home/tavi/src/linux/kernel/crusher.c:27).
22

list_add(&e2.lh, &list);

23
24

list_for_each(i, &list) {

25

struct list_m *x = list_entry(i, struct list_m, lh);

26
27

printk("list_for_each %d\n", x->a);

28

list_del(x->lh);

29
30}
31

objdump -dSr
list_add(&e1.lh, &list);
list_add(&e2.lh, &list);
list_for_each(i, &list) {
48: 8b 3d 00 00 00 00 mov 0x0,%edi
4a: R_386_32 list
4e: eb 18 jmp 68 <crush_it+0x68>
struct list_m *x = list_entry(i, struct list_m, lh);
printk("list_for_each %d\n", x->a);
50: ff 77 08 pushl 0x8(%edi)
53: 68 00 00 00 00 push $0x0
54: R_386_32 .rodata.str1.1
58: e8 fc ff ff ff call 59 <crush_it+0x59>
59: R_386_PC32 printk

List debugging
Operaiile cu liste folosesc valori poision
0x00100100 (next pointer) and 0x00200200 (prev
pointer) pentru a prinde accesri de elemente
neiniializate
[1185.368428] BUG: unable to handle kernel paging request at 00100108
[ 1185.374653] IP: [<f27db050>] crush_it+0x50/0x75 [crusher]
list_for_each(i, &list) {
struct list_m *x = list_entry(i, struct list_m, lh);
printk("list_for_each %d\n", x->a);
list_del(x);
}

Memory debugging
CONFIG_DEBUG_SLAB
CONFIG_SLUB_DEBUG_ON
Poison based memory debuggers

Poison

0x5a5a5a5a
0x5a5a5a5a

Allocated
buffer

Poison

0x6b6b6b6b
0x6b6b6b6b

Utilizarea nainte de iniializare

[ 49.826991] BUG: unable to handle kernel paging request at 5a5a5a5a
[ 49.833202] IP: [<c1225063>] __list_del_entry+0x37/0x71

[ 49.922309] Call Trace:

[ 49.924751] [<c12250a8>] list_del+0xb/0x1b
[ 49.928926] [<f1de81a2>] use_before_init+0x31/0x38 [crusher]
[ 49.934662] [<f1de8265>] crush_it+0x38/0xa9 [crusher]
[ 49.939791] [<f1de82de>] init_module+0x8/0xa [crusher]
[ 49.945007] [<c1001072>] do_one_initcall+0x72/0x119
[ 49.949962] [<f1de82d6>] ? crush_it+0xa9/0xa9 [crusher]
[ 49.955269] [<c106b8ae>] sys_init_module+0xc8d/0xe77
[ 49.960316] [<c14d7d18>] syscall_call+0x7/0xb

Utilizarea nainte de iniializare (2)

noinline void use_before_init(void)
{
struct list_m *m = kmalloc(sizeof(*m), GFP_KERNEL);
printk("%s\n", __func__);
list_del(&m->lh);
}

Utilizarea dup eliberare

[ 34.592438] BUG: unable to handle kernel paging request at 6b6b6b6b
[ 34.598647] IP: [<c1225063>] __list_del_entry+0x37/0x71

[ 34.687754] Call Trace:

[ 34.690197] [<c12250a8>] list_del+0xb/0x1b
[ 34.694373] [<f4c6816a>] use_after_free+0x38/0x3f [crusher]
[ 34.700022] [<f4c6827f>] crush_it+0x52/0xa9 [crusher]
[ 34.705151] [<f4c682de>] init_module+0x8/0xa [crusher]
[ 34.710368] [<c1001072>] do_one_initcall+0x72/0x119
[ 34.715323] [<f4c682d6>] ? crush_it+0xa9/0xa9 [crusher]
[ 34.720626] [<c106b8ae>] sys_init_module+0xc8d/0xe77
[ 34.725676] [<c14d7d18>] syscall_call+0x7/0xb

Utilizarea dup eliberare (2)

noinline void use_after_free(void)
{
struct list_m *m = kmalloc(sizeof(*m), GFP_KERNEL);
printk("%s\n", __func__);
kfree(m);
list_del(&m->lh);
}

Utilizarea dup eliberare (3)

root@android:/ # insmod /system/lib/modules/crusher.ko
test=use_before_init
[ 18.713899] Slab corruption: size-4096 start=ed612000, len=4096
[ 18.720009] 000: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
[ 18.727022] 010: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6b 6b
noinline void use_after_free2(void)
{
char *b = kmalloc(3000, GFP_KERNEL);
kfree(b);
memset(b, 0, 30);
b = kmalloc(3000, GFP_KERNEL);
kfree(b);
}

Buffer overflow
[ 20.628752] slab error in verify_redzone_free(): cache `dummy':
memory outside object was overwritten
[ 20.637983] Pid: 1282, comm: insmod Not tainted 3.0.16-mid10-00007ga4a6b62-dirty #70
[ 20.638003] Call Trace:
[ 20.638050] [<c10cc1de>] __slab_error+0x17/0x1c
[ 20.638087] [<c10cc7ca>] __cache_free+0x12c/0x317
[ 20.638128] [<f27f1138>] ? buffer_overflow+0x4c/0x57 [crusher]
[ 20.638166] [<c10ccaba>] kmem_cache_free+0x2b/0xaf
[ 20.638213] [<f27f1138>] buffer_overflow+0x4c/0x57 [crusher]
[ 20.638257] [<f27f12aa>] crush_it+0x6c/0xa9 [crusher]
[ 20.638292] [<f27f12ef>] init_module+0x8/0xd [crusher]
[ 20.638323] [<c1001072>] do_one_initcall+0x72/0x119
[ 20.638358] [<f27f12e7>] ? crush_it+0xa9/0xa9 [crusher]
[ 20.638394] [<c106b8ae>] sys_init_module+0xc8d/0xe77
[ 20.638446] [<c14d7d18>] syscall_call+0x7/0xb
[ 20.638478] eb002bf8: redzone 1:0xd84156c5635688c0, redzone 2:0x0

Buffer overflow (2)

noinline void buffer_overflow(void)
{
struct kmem_cache *km = kmem_cache_create("dummy", 3000, 0, 0,
NULL);
char *b = kmem_cache_alloc(km, GFP_KERNEL);
printk("%s\n", __func__);
memset(b, 0, 3016);
kmem_cache_free(km, b);
}

Lockdep checker
CONFIG_DEBUG_LOCKDEP
Detecteaz lock inversion, dependene circulare,
bug-uri de locking n contexte softirq sau hardirq
Menine starea dependenelor ntre clase de
lockuri pentru a reduce complexitatea si a permite
verificarea la run-time
Pentru a reduce complexitatea i mai mult, un
scenariu este verificat o singur dat i rezultatul
este meninut ntr-un hash table

Exemplu: deacklock AB BA

noinline int

noinline int

deadlock_thread_a(void *arg)

deadlock_thread_b(void *arg)

mutex_lock(&a);

mutex_lock(&b);

schedule_timeout(HZ);

schedule_timeout(HZ);

mutex_lock(&b);

mutex_lock(&a);

mutex_unlock(&b);

mutex_unlock(&a);

mutex_unlock(&a);

mutex_unlock(&b);

return 0;

return 0;
}

Exemplu: deacklock AB BA (2)

[ 38.038659] =======================================================
[ 38.045075] [ INFO: possible circular locking dependency detected ]
[ 38.051336] 3.0.16-mid10-00007-g282a5ae-dirty #54
[ 38.056020] ------------------------------------------------------[ 38.062286] deadlock_b/2334 is trying to acquire lock:
[ 38.067413] (a){+.+...}, at: [<f32a0205>] deadlock_thread_b+0x25/0x3d
[crusher]
[ 38.074802]
[ 38.074809] but task is already holding lock:
[ 38.080629] (b){+.+...}, at: [<f32a01ef>] deadlock_thread_b+0xf/0x3d [crusher]
[ 38.087930]
[ 38.087938] which lock already depends on the new lock.
[ 38.087947]
[ 38.096100]

Exemplu: deacklock AB BA (3)

[ 38.096107] the existing dependency chain (in reverse order) is:
[ 38.103582]
[ 38.103589] -> #1 (b){+.+...}:
[ 38.108097] [<c1062fd3>] lock_acquire+0x104/0x140
[ 38.113404] [<c14d7f88>] __mutex_lock_common+0x3b/0x33f
[ 38.119226] [<c14d8356>] mutex_lock_nested+0x2d/0x36
[ 38.124792] [<f32a0242>] deadlock_thread_a+0x25/0x3f [crusher]
[ 38.131226] [<c104fe01>] kthread+0x63/0x68
[ 38.135917] [<c14da482>] kernel_thread_helper+0x6/0xd
[ 38.141574]
[ 38.141581] -> #0 (a){+.+...}:
[ 38.146091] [<c106283a>] __lock_acquire+0x988/0xc00
[ 38.151570] [<c1062fd3>] lock_acquire+0x104/0x140
[ 38.156873] [<c14d7f88>] __mutex_lock_common+0x3b/0x33f
[ 38.162698] [<c14d8356>] mutex_lock_nested+0x2d/0x36
[ 38.168260] [<f32a0205>] deadlock_thread_b+0x25/0x3d [crusher]
[ 38.174697] [<c104fe01>] kthread+0x63/0x68
[ 38.179389] [<c14da482>] kernel_thread_helper+0x6/0xd

Exemplu: deacklock AB BA (4)

[ 38.185040]
[ 38.185047] other info that might help us debug this:
[ 38.185055]
[ 38.193041] Possible unsafe locking scenario:
[ 38.193050]
[ 38.198953] CPU0 CPU1
[ 38.203472] ---- ---[ 38.207992] lock(b);
[ 38.210339] lock(a);
[ 38.215206] lock(b);
[ 38.220077] lock(a);
[ 38.222422]
[ 38.222429] *** DEADLOCK ***
[ 38.222435]

Exemplu: deacklock AB BA (5)

[ 38.228343] 1 lock held by deadlock_b/2334:
[ 38.232515] #0: (b){+.+...}, at: [<f32a01ef>]
deadlock_thread_b+0xf/0x3d [crusher]
[ 38.240246]
[ 38.240253] stack backtrace:
[ 38.244605] Pid: 2334, comm: deadlock_b Tainted: G C 3.0.16-mid1000007-g282a5ae-dirty #54
[ 38.253639] Call Trace:
[ 38.256095] [<c1061e16>] print_circular_bug+0x18e/0x19b
[ 38.261399] [<c106283a>] __lock_acquire+0x988/0xc00
[ 38.278527] [<c1062fd3>] lock_acquire+0x104/0x140
[ 38.289390] [<c14d7f88>] __mutex_lock_common+0x3b/0x33f
[ 38.306166] [<c14d8356>] mutex_lock_nested+0x2d/0x36
[ 38.317299] [<f32a0205>] deadlock_thread_b+0x25/0x3d [crusher]
[ 38.323208] [<c104fe01>] kthread+0x63/0x68
[ 38.332948] [<c14da482>] kernel_thread_helper+0x6/0xd

Exemplu: softirq + spin_lock

DEFINE_SPINLOCK(l);
noinline void f(unsigned long x)
{
spin_lock(&l);
spin_unlock(&l);
}
noinline void deadlock_softirq(void)
{
DEFINE_TIMER(t, f, jiffies+HZ, 0);
spin_lock(&l);
add_timer(&t);
while (1);
}

Exemplu: softirq + spin_lock (2)

[ 591.259946] =================================
[ 591.265740] [ INFO: inconsistent lock state ]
[ 591.270105] 3.0.16-mid10-00007-ga4a6b62-dirty #70
[ 591.274784] --------------------------------[ 591.279130] inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W} usage.
[ 591.285136] insmod/3588 [HC0[0]:SC1[1]:HE1:SE0] takes:
[ 591.290258] (l){+.?...}, at: [<f2dfb088>] f+0xd/0x19 [crusher]
[ 591.296168] {SOFTIRQ-ON-W} state was registered at:
[ 591.301034] [<c10620d3>] __lock_acquire+0x2d1/0xc00
[ 591.306079] [<c1062f23>] lock_acquire+0x104/0x140
[ 591.310947] [<c14d7471>] _raw_spin_lock+0x25/0x34
[ 591.315814] [<f2dfb071>] deadlock_softirq+0x57/0x61 [crusher]
[ 591.321726] [<f2dfb3bf>] crush_it+0xc0/0xe3 [crusher]
[ 591.326943] [<f2dfb3ea>] init_module+0x8/0xa [crusher]
[ 591.332246] [<c1001072>] do_one_initcall+0x72/0x119
[ 591.337296] [<c106b8ae>] sys_init_module+0xc8d/0xe77
[ 591.342417] [<c14d7d18>] syscall_call+0x7/0xb

Exemplu: softirq + spin_lock (3)

[ 591.346941] irq event stamp: 11144
[ 591.350332] hardirqs last enabled at (11144): [<c14d7a73>]
_raw_spin_unlock_irq+0x22/0x43
[ 591.358592] hardirqs last disabled at (11143): [<c14d74d2>]
_raw_spin_lock_irq+0xc/0x3a
[ 591.366591] softirqs last enabled at (11138): [<c103c35b>]
__do_softirq+0x1c9/0x1dd
[ 591.374327] softirqs last disabled at (11141): [<c1003c1d>]
do_softirq+0x53/0xa2
[ 591.381719]
[ 591.381724] other info that might help us debug this:
[ 591.388244] Possible unsafe locking scenario:
[ 591.388254]
[ 591.394152] CPU0
[ 591.396586] ---[ 591.399192] lock(l);
[ 591.401544] <Interrupt>
[ 591.404153] lock(l);

Kmemleak
Un memory leak detector ce urmrete alocrile i
pointerii alocai ntr-un mod similar cu un garbage
collector
Cnd pointerii ctre zona alocat nu mai sunt
detectabili i zona nu a fost eliberat se detectaz
un memory leak
Scaneaz memoria alocat i stiva kernel a
proceselor pentru a detecta dac mai exist
pointeri folosii
CONFIG_DEBUG_KMEMLEAK

Kmemleak (2)

Setup
# mount -t debugfs nodev /sys/kernel/debug/

Trigger a memory scan

# echo scan > /sys/kernel/debug/kmemleak

Clear all possible leaks

# echo clear > /sys/kernel/debug/kmemleak

Read memory leaks

# cat /sys/kernel/debug/kmemleak

Kmemleak (3)
noinline void mem_leak(void)
{
kmalloc(10, GFP_KERNEL);
return;
}
unreferenced object 0xecf5c670 (size 32):
comm "insmod", pid 1806, jiffies 4294793093 (age 117.527s)
hex dump (first 32 bytes):
5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZZZZZZZZZ
5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a 5a a5 ZZZZZZZZZZZZZZZ.
backtrace:
[<c14c5c4a>] kmemleak_alloc+0x21/0x3f
[<c10cdffe>] kmem_cache_alloc_trace+0x48e/0x4ce
[<f130b018>] mem_leak+0x18/0x1a [crusher]
[<f130b3de>] crush_it+0xdf/0xe3 [crusher]
[<f130b3ea>] init_module+0x8/0xa [crusher]
[<c1001072>] do_one_initcall+0x72/0x119
[<c106b95b>] sys_init_module+0xcb3/0xea0

Kmemcheck
Paging based memory debugger
Nu folosete pagini de gard ci marcheaz toate
paginile alocate ca fiind invalide pentru a genera
PF
Page fault handler-ul ruleaza kmemchecker-ul
Dac verificarea trece, pagina se marcheaz ca
valid i se activeaz modul single stepping
Dup excepia generat de single stepping se
seteaz pagina ca fiind invalid

perf

Profiling tool pentru

Hardware events: utilizare CPU, TLB misses,
cache misses
Software events: page faults, context switches

Statistical profiler
Suport colectarea backtrace-urilor (kernel +
userspace)
Poate filtra evenimentele dup proces,
procesor dar funcioneaz i pe ntreg
sistemul

Perf top

PerfTop: 121 irqs/sec kernel:95.9% exact: 0.0% [1000Hz cycles], (all, 2 CPUs)
------------------------------------------------------------------------------samples pcnt function DSO
_______ _____ ___________________________ _____________________
31.00 17.4% dvmAsmInstructionStartCode /system/lib/libdvm.so
23.00 12.9% getdelim /system/bin/perf
15.00 8.4% update_iter [kernel.kallsyms]
13.00 7.3% format_decode [kernel.kallsyms]
12.00 6.7% map__process_kallsym_symbol /system/bin/perf
10.00 5.6% vsnprintf [kernel.kallsyms]
10.00 5.6% number [kernel.kallsyms]
7.00 3.9% hex2u64 /system/bin/perf
7.00 3.9% pthread_mutex_lock /system/lib/libc.so
6.00 3.4% lock_acquire [kernel.kallsyms]
5.00 2.8% strstr /system/lib/libc.so
5.00 2.8% __lock_acquire [kernel.kallsyms]

perf top -event page-faults

-----------------------------------------------------------------------------PerfTop: 205 irqs/sec kernel: 3.4% exact: 0.0% [1000Hz page-faults], (all, 2
CPUs)
-----------------------------------------------------------------------------samples pcnt function DSO
_______ _____ _______________________________________ __________
41.00 10.0% mspace_malloc libcutils.so
35.00 8.5% _int_malloc perf
28.00 6.8% loadClassFromDex0(DvmDex*, DexClassDef libdvm.so
23.00 5.6% parseZipArchive(ZipArchive*) libdvm.so
21.00 5.1% strcmp libc.so
20.00 4.9% memcpy libc.so
16.00 3.9% dvmAsmInstructionStartCode libdvm.so
15.00 3.6% dvmResolveMethod libdvm.so
14.00 3.4% file_read_actor [kernel]
14.00 3.4% memset libc.so
12.00 2.9% android::ResTable::parsePackage(android libutils.so

Perf record/report
# perf record g ls
# perf report
# Events: 42 cycles
#
# Overhead Command Shared Object Symbol
# ........ ....... ................. ......................
#
11.79% ls [kernel.kallsyms] [k] check_poison_obj
|
--- check_poison_obj
|
|--57.86%-- __kmalloc
|

kzalloc.clone.0

perf_event_mmap

mmap_region

do_mmap_pgoff

|--52.60%-- sys_mmap_pgoff

Alte tool-uri

ftrace
kprobes
sparse (make C=1)
coccinelle
patchwork
checkpatch.pl
printk
dump_stack