crash와 drgn 분석 — vmcore에서 커널 상태 복원하기

1
crash> bt                       # 현재 task의 콜스택
2
crash> bt -a                    # 모든 CPU
3
crash> ps                       # 프로세스 목록
4
crash> ps -k                    # 커널 스레드
5
crash> log                      # dmesg
6
crash> log | tail               # 마지막 로그
7
crash> sys                      # 시스템 정보
8
crash> mach                     # CPU 정보
9
crash> mount                    # 마운트
10
crash> net                      # 네트워크 디바이스
11
crash> files                    # 열린 파일
12
crash> vm <pid>                 # 그 task의 VM 정보
13
crash> task <pid>               # task_struct
14
crash> kmem -s                  # slab 통계
15
crash> kmem -i                  # 메모리 요약
16
crash> p <var>                  # 전역 변수
17
crash> rd <addr>                # 메모리 읽기
18
crash> dis <func>               # 디스어셈블
19
crash> struct task_struct <addr> # 구조체 출력
20
crash> list <addr>              # linked list 순회
21
crash> tree <addr>              # tree 순회 (rbtree)

1
crash> bt
2
PID: 1234     TASK: ffff... CPU: 2  COMMAND: "myprog"
3
 #0 [ffff...] machine_kexec at ffffffff8104a123
4
 #1 [ffff...] __crash_kexec at ffffffff8108b456
5
 #2 [ffff...] panic at ffffffff8108c789
6
 #3 [ffff...] oops_end at ffffffff8102d012
7
 #4 [ffff...] no_context at ffffffff8106f345
8
 #5 [ffff...] __do_page_fault at ffffffff8106f678
9
 #6 [ffff...] do_page_fault at ffffffff8106f9ab
10
 #7 [ffff...] page_fault at ffffffff819c5678
11
    [exception RIP: my_driver_handler+0x42]
12
    RIP: ffffffffc0123456 RSP: ffff...
13
    RAX: 0000000000000000 ...
14
 #8 [ffff...] __handle_irq_event_percpu at ffffffff810abc01
15
 ...
16

17
crash> dis my_driver_handler
18
0xffffffffc0123410 <my_driver_handler>:  push %rbp
19
...
20
0xffffffffc0123456 <my_driver_handler+0x42>:  mov 0x10(%rax),%rcx
21
                                              ^^^ %rax = 0 → NULL deref
22

23
crash> sym ffffffffc0123456
24
my_driver_handler+0x42 at drivers/my_module.c:42

1
crash> p init_task
2
init_task = $1 = {
3
  state = 0,
4
  ...
5
  tasks = {
6
    next = 0xffff...,
7
    prev = 0xffff...
8
  },
9
  ...
10
}
11

12
crash> list -H init_task.tasks task_struct.tasks -s task_struct.comm | head
13
ffff...
14
  comm = "init"
15
ffff...
16
  comm = "kthreadd"
17
...

1
>>> from drgn.helpers.linux import *
2

3
# 현재 init_task
4
>>> init = prog['init_task']
5
>>> init.comm
6
(char [16])"swapper/0"
7

8
# 모든 task 순회
9
>>> for task in for_each_task(prog):
10
...     print(task.pid.value_(), task.comm.string_().decode())
11
0 swapper/0
12
1 init
13
2 kthreadd
14
3 rcu_gp
15
...

1
from drgn.helpers.linux import (
2
    for_each_task,
3
    for_each_online_cpu,
4
    list_for_each_entry,
5
    find_task,
6
    pid_for_each_thread,
7
    find_inode_by_path,
8
    block_device_name,
9
    ...
10
)
11

12
# 특정 PID
13
task = find_task(prog, 1234)
14
print(task.comm.string_())
15

16
# 그 task의 fd
17
from drgn.helpers.linux.fs import fget
18
file = fget(task, 3)        # fd 3
19
print(file.f_path.dentry.d_name.name.string_())

1
>>> task = find_task(prog, 1234)
2
>>> for frame in task.stack_trace():
3
...     print(frame)
4
#0  page_fault+0x...  at arch/x86/kernel/...
5
#1  do_page_fault+0x...  at arch/x86/mm/...
6
#2  my_function+0x42  at drivers/foo.c:42
7
...

1
#!/usr/bin/env drgn
2
# crash-info.py — vmcore 진단 자동화
3

4
from drgn.helpers.linux import for_each_task
5

6
print(f"Kernel: {prog['linux_banner'].string_().decode().strip()}")
7
print(f"Uptime: ...")
8
print()
9

10
# CPU별 콜스택
11
print("=== Per-CPU active task ===")
12
for cpu in for_each_online_cpu(prog):
13
    task = per_cpu(prog['current_task'], cpu)
14
    print(f"CPU {cpu}: {task.comm.string_().decode()} pid={task.pid.value_()}")
15

16
# blocked task (D state)
17
print("\n=== Tasks in D state (uninterruptible) ===")
18
for task in for_each_task(prog):
19
    if task.__state.value_() == 0x2:  # TASK_UNINTERRUPTIBLE
20
        print(f"  {task.comm.string_().decode()} pid={task.pid.value_()}")

1
PANIC: "BUG: kernel NULL pointer dereference, address: 0000000000000010"

1
PANIC: "Kernel panic - not syncing: softlockup: hung tasks"

1
PANIC: "Kernel panic - not syncing: hung_task: blocked tasks"

1
PANIC: "BUG: KASAN: use-after-free in ..."

1
PANIC: "Out of memory: Killed process N (myprog) ..."