psatck
pstack命令是一个在Linux系统中用于查看进程堆栈信息的工具。
写了一个服务端死锁程序，如下：
#include <iostream> #include <thread> #include <mutex> #include <chrono> #include <sys/socket.h> #include <netinet/in.h> #include <unistd.h> #include <cstring> // 定义两个互斥锁 std::mutex mutex1; std::mutex mutex2; // 处理客户端请求的函数 void handle_client(int client_socket, int client_id) { std::cout << "Client " << client_id << ": Connected" << std::endl; // 模拟客户端请求处理 if (client_id == 1) { // 客户端 1：先获取 mutex1，再获取 mutex2 std::cout << "Client " << client_id << ": Trying to lock mutex1..." << std::endl; std::lock_guard<std::mutex> lock1(mutex1); std::this_thread::sleep_for(std::chrono::seconds(5)); // 增加锁的持有时间 std::cout << "Client " << client_id << ": Locked mutex1, now trying to lock mutex2..." << std::endl; // 尝试获取 mutex2 std::lock_guard<std::mutex> lock2(mutex2); // 死锁发生点 std::this_thread::sleep_for(std::chrono::seconds(5)); // 模拟更多工作 std::cout << "Client " << client_id << ": Locked both mutex1 and mutex2" << std::endl; } else if (client_id == 2) { // 客户端 2：先获取 mutex2，再获取 mutex1 std::cout << "Client " << client_id << ": Trying to lock mutex2..." << std::endl; std::lock_guard<std::mutex> lock2(mutex2); std::this_thread::sleep_for(std::chrono::seconds(5)); // 增加锁的持有时间 std::cout << "Client " << client_id << ": Locked mutex2, now trying to lock mutex1..." << std::endl; // 尝试获取 mutex1 std::lock_guard<std::mutex> lock1(mutex1); // 死锁发生点 std::this_thread::sleep_for(std::chrono::seconds(5)); // 模拟更多工作 std::cout << "Client " << client_id << ": Locked both mutex1 and mutex2" << std::endl; } // 关闭客户端连接 close(client_socket); std::cout << "Client " << client_id << ": Disconnected" << std::endl; } // TCP 服务器主函数 void start_server(int port) { int server_fd, new_socket; struct sockaddr_in address; int opt = 1; int addrlen = sizeof(address); // 创建 socket 文件描述符 if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) { perror("socket failed"); exit(EXIT_FAILURE); } // 设置 SO_REUSEADDR 选项 if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, sizeof(opt))) { perror("setsockopt failed"); exit(EXIT_FAILURE); } // 绑定 socket 到指定端口 address.sin_family = AF_INET; address.sin_addr.s_addr = INADDR_ANY; address.sin_port = htons(port); if (bind(server_fd, (struct sockaddr *)&address, sizeof(address)) < 0) { perror("bind failed"); exit(EXIT_FAILURE); } // 监听连接 if (listen(server_fd, 3) < 0) { perror("listen failed"); exit(EXIT_FAILURE); } std::cout << "Server started on port " << port << ". Waiting for connections..." << std::endl; int client_id = 1; // 用于区分不同客户端 while (true) { // 接受新的客户端连接 if ((new_socket = accept(server_fd, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) { perror("accept failed"); continue; } // 为每个客户端创建一个新线程 std::thread client_thread(handle_client, new_socket, client_id++); client_thread.detach(); // 分离线程，允许其独立运行 } } int main() { int port = 8080; start_server(port); return 0; }
tcp_deadlock_server.cpp
编译：g++ -std=c++11 -pthread -o tcp_deadlock_server tcp_deadlock_server.cpp -g
用telnet(telnet 127.1 8080)连两次就会死锁，服务端输出如下：
Server started on port 8080. Waiting for connections...
Client 1: Connected
Client 1: Trying to lock mutex1...
Client 2: Connected
Client 2: Trying to lock mutex2...
Client 1: Locked mutex1, now trying to lock mutex2...
Client 2: Locked mutex2, now trying to lock mutex1...
pstack调试死锁
ps查看进程ID，然后pstack + 进程ID : pstack 915 > pstack_out，将输出重定向到文件，好看一些：
Thread 3 (LWP 919): #0 0x0000fffcc23821dc in ?? () from /lib64/libpthread.so.0 #1 0x0000fffcc237b060 in pthread_mutex_lock () from /lib64/libpthread.so.0 #2 0x00000000004012c4 in __gthread_mutex_lock (__mutex=0x420240 <mutex1>) at /usr/include/c++/7.3.0/aarch64-linux-gnu/bits/gthr-default.h:748 #3 0x0000000000401a88 in std::mutex::lock (this=0x420240 <mutex1>) at /usr/include/c++/7.3.0/bits/std_mutex.h:103 #4 0x0000000000401b34 in std::lock_guard<std::mutex>::lock_guard (this=0xfffcc19ce810, __m=...) at /usr/include/c++/7.3.0/bits/std_mutex.h:162 #5 0x00000000004015ac in handle_client (client_socket=5, client_id=2) at tcp_deadlock_server.cpp:38 #6 0x0000000000402308 in std::__invoke_impl<void, void (*)(int, int), int, int> (__f=@0x248c23e0: 0x401310 <handle_client(int, int)>, __args#0=@0x248c23dc: 5, __args#1=@0x248c23d8: 2) at /usr/include/c++/7.3.0/bits/invoke.h:60 #7 0x0000000000401e18 in std::__invoke<void (*)(int, int), int, int> (__fn=@0x248c23e0: 0x401310 <handle_client(int, int)>, __args#0=@0x248c23dc: 5, __args#1=@0x248c23d8: 2) at /usr/include/c++/7.3.0/bits/invoke.h:95 #8 0x00000000004029cc in std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> >::_M_invoke<0ul, 1ul, 2ul> (this=0x248c23d8) at /usr/include/c++/7.3.0/thread:234 #9 0x0000000000402970 in std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> >::operator() (this=0x248c23d8) at /usr/include/c++/7.3.0/thread:243 #10 0x0000000000402950 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> > >::_M_run (this=0x248c23d0) at /usr/include/c++/7.3.0/thread:186 #11 0x0000fffcc257e134 in ?? () from /lib64/libstdc++.so.6 #12 0x0000fffcc23788cc in ?? () from /lib64/libpthread.so.0 #13 0x0000fffcc22ba1ec in ?? () from /lib64/libc.so.6 Thread 2 (LWP 917): #0 0x0000fffcc23821dc in ?? () from /lib64/libpthread.so.0 #1 0x0000fffcc237b060 in pthread_mutex_lock () from /lib64/libpthread.so.0 #2 0x00000000004012c4 in __gthread_mutex_lock (__mutex=0x420270 <mutex2>) at /usr/include/c++/7.3.0/aarch64-linux-gnu/bits/gthr-default.h:748 #3 0x0000000000401a88 in std::mutex::lock (this=0x420270 <mutex2>) at /usr/include/c++/7.3.0/bits/std_mutex.h:103 #4 0x0000000000401b34 in std::lock_guard<std::mutex>::lock_guard (this=0xfffcc21de820, __m=...) at /usr/include/c++/7.3.0/bits/std_mutex.h:162 #5 0x0000000000401450 in handle_client (client_socket=4, client_id=1) at tcp_deadlock_server.cpp:27 #6 0x0000000000402308 in std::__invoke_impl<void, void (*)(int, int), int, int> (__f=@0x248c2290: 0x401310 <handle_client(int, int)>, __args#0=@0x248c228c: 4, __args#1=@0x248c2288: 1) at /usr/include/c++/7.3.0/bits/invoke.h:60 #7 0x0000000000401e18 in std::__invoke<void (*)(int, int), int, int> (__fn=@0x248c2290: 0x401310 <handle_client(int, int)>, __args#0=@0x248c228c: 4, __args#1=@0x248c2288: 1) at /usr/include/c++/7.3.0/bits/invoke.h:95 #8 0x00000000004029cc in std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> >::_M_invoke<0ul, 1ul, 2ul> (this=0x248c2288) at /usr/include/c++/7.3.0/thread:234 #9 0x0000000000402970 in std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> >::operator() (this=0x248c2288) at /usr/include/c++/7.3.0/thread:243 #10 0x0000000000402950 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)(int, int), int, int> > >::_M_run (this=0x248c2280) at /usr/include/c++/7.3.0/thread:186 #11 0x0000fffcc257e134 in ?? () from /lib64/libstdc++.so.6 #12 0x0000fffcc23788cc in ?? () from /lib64/libpthread.so.0 #13 0x0000fffcc22ba1ec in ?? () from /lib64/libc.so.6 Thread 1 (LWP 915): #0 0x0000fffcc23827c4 in accept () from /lib64/libpthread.so.0 #1 0x0000000000401868 in start_server (port=8080) at tcp_deadlock_server.cpp:89 #2 0x00000000004018f8 in main () at tcp_deadlock_server.cpp:102
pstack 输出
能看到一共三个线程，Thread 3 (LWP 919)卡在pthread_mutex_lock：#5 0x00000000004015ac in handle_client (client_socket=5, client_id=2) at tcp_deadlock_server.cpp:38，
Thread 2 (LWP 917)卡在pthread_mutex_lock：#5 0x0000000000401450 in handle_client (client_socket=4, client_id=1) at tcp_deadlock_server.cpp:27
就发现了死锁在的位置
gdb
不用c++11又写了一个程序，如下：
#include <iostream> #include <cstring> #include <unistd.h> #include <sys/socket.h> #include <netinet/in.h> #include <arpa/inet.h> #include <pthread.h> #include <errno.h> #include <cstdlib> #include <fcntl.h> #include <stdio.h> // 定义两个互斥锁 pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_t mutex2 = PTHREAD_MUTEX_INITIALIZER; // 处理客户端请求的函数 void* handle_client(void* arg) { int client_socket = *(static_cast<int*>(arg)); free(arg); // 释放传递给线程的动态分配的内存 std::cout << "Client connected with socket: " << client_socket << std::endl; // 模拟客户端请求处理 if (client_socket == 4) { // 客户端 1：先获取 mutex1，再获取 mutex2 std::cout << "Client " << client_socket << ": Trying to lock mutex1..." << std::endl; pthread_mutex_lock(&mutex1); usleep(5000000); // 休眠 5 秒 std::cout << "Client " << client_socket << ": Locked mutex1, now trying to lock mutex2..." << std::endl; // 尝试获取 mutex2 pthread_mutex_lock(&mutex2); // 死锁发生点 usleep(5000000); // 休眠 5 秒 std::cout << "Client " << client_socket << ": Locked both mutex1 and mutex2" << std::endl; // 释放互斥锁 pthread_mutex_unlock(&mutex2); pthread_mutex_unlock(&mutex1); } else if (client_socket == 5) { // 客户端 2：先获取 mutex2，再获取 mutex1 std::cout << "Client " << client_socket << ": Trying to lock mutex2..." << std::endl; pthread_mutex_lock(&mutex2); usleep(5000000); // 休眠 5 秒 std::cout << "Client " << client_socket << ": Locked mutex2, now trying to lock mutex1..." << std::endl; // 尝试获取 mutex1 pthread_mutex_lock(&mutex1); // 死锁发生点 usleep(5000000); // 休眠 5 秒 std::cout << "Client " << client_socket << ": Locked both mutex1 and mutex2" << std::endl; // 释放互斥锁 pthread_mutex_unlock(&mutex1); pthread_mutex_unlock(&mutex2); } // 关闭客户端连接 close(client_socket); std::cout << "Client disconnected with socket: " << client_socket << std::endl; pthread_exit(NULL); } // TCP 服务器主函数 void start_server(int port) { int server_fd, new_socket; struct sockaddr_in address; int opt = 1; int addrlen = sizeof(address); // 创建 socket 文件描述符 if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) { perror("socket failed"); exit(EXIT_FAILURE); } // 设置 SO_REUSEADDR 选项 if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt))) { perror("setsockopt failed"); exit(EXIT_FAILURE); } // 绑定 socket 到指定端口 address.sin_family = AF_INET; address.sin_addr.s_addr = INADDR_ANY; address.sin_port = htons(port); if (bind(server_fd, (struct sockaddr *)&address, sizeof(address)) < 0) { perror("bind failed"); exit(EXIT_FAILURE); } // 监听连接 if (listen(server_fd, 3) < 0) { perror("listen failed"); exit(EXIT_FAILURE); } std::cout << "Server started on port " << port << ". Waiting for connections..." << std::endl; int client_id = 1; // 用于区分不同客户端 while (true) { // 接受新的客户端连接 if ((new_socket = accept(server_fd, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) { perror("accept failed"); continue; } // 为每个客户端创建一个新线程 pthread_t thread; int* client_socket_ptr = new int(new_socket); // 动态分配存储套接字描述符的内存 if (pthread_create(&thread, NULL, handle_client, static_cast<void*>(client_socket_ptr)) != 0) { perror("pthread_create failed"); delete client_socket_ptr; // 如果线程创建失败，释放内存 close(new_socket); continue; } // 分离线程，允许其独立运行 pthread_detach(thread); // 为了测试死锁，只接受前两个客户端连接 if (client_id >= 3) { close(new_socket); // 关闭多余的连接 continue; } client_id++; } } int main() { int port = 8080; start_server(port); return 0; }
tcp_deadlock_server_c++0x.cpp
编译运行telnet测试跟上面一样
gdb调试死锁
ps查看进程ID，然后gdb跟进程：gdb -p 11560
查看所有线程：info threads，进入线程：thread 2 ,然后bt查看线程堆栈，切换另一个线程如上，就能看到两个线程都卡在了lock，具体调试步骤如下：
(gdb) info threads 3 Thread 0x7fb6c0115700 (LWP 11562) 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 2 Thread 0x7fb6bf714700 (LWP 11564) 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 * 1 Thread 0x7fb6c0117720 (LWP 11560) 0x0000003b5200e7ed in accept () from /lib64/libpthread.so.0 (gdb) thread 2 [Switching to thread 2 (Thread 0x7fb6bf714700 (LWP 11564))]#0 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 (gdb) bt #0 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x0000003b52009328 in _L_lock_854 () from /lib64/libpthread.so.0 #2 0x0000003b520091f7 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000000400f9f in handle_client (arg=0x13a8010) at tcp_deadlock_server.cpp:48 #4 0x0000003b520077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003b51ce570d in clone () from /lib64/libc.so.6 (gdb) thread 3 [Switching to thread 3 (Thread 0x7fb6c0115700 (LWP 11562))]#0 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 (gdb) bt #0 0x0000003b5200dff4 in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x0000003b52009328 in _L_lock_854 () from /lib64/libpthread.so.0 #2 0x0000003b520091f7 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x0000000000400eb2 in handle_client (arg=0x13a8010) at tcp_deadlock_server.cpp:33 #4 0x0000003b520077f1 in start_thread () from /lib64/libpthread.so.0 #5 0x0000003b51ce570d in clone () from /lib64/libc.so.6
gdb调试死锁
总结
pstack和gdb都使用 ptrace() 系统调用来附着到目标进程。ptrace() 允许 GDB 暂停目标进程的执行，读取和修改其内存及寄存器，并捕获系统调用。
