Current Dir: /usr/sbin/
[FILE] a2disconf [ edit | delete | rename ]
[FILE] a2dismod [ edit | delete | rename ]
[FILE] a2dissite [ edit | delete | rename ]
[FILE] a2enconf [ edit | delete | rename ]
[FILE] a2enmod [ edit | delete | rename ]
[FILE] a2ensite [ edit | delete | rename ]
[FILE] a2query [ edit | delete | rename ]
[FILE] aa-load [ edit | delete | rename ]
[FILE] aa-remove-unknown [ edit | delete | rename ]
[FILE] aa-status [ edit | delete | rename ]
[FILE] aa-teardown [ edit | delete | rename ]
[FILE] accessdb [ edit | delete | rename ]
[FILE] add-shell [ edit | delete | rename ]
[FILE] addgnupghome [ edit | delete | rename ]
[FILE] addgroup [ edit | delete | rename ]
[FILE] adduser [ edit | delete | rename ]
[FILE] agetty [ edit | delete | rename ]
[FILE] apache2 [ edit | delete | rename ]
[FILE] apache2ctl [ edit | delete | rename ]
[FILE] apachectl [ edit | delete | rename ]
[FILE] apparmor_parser [ edit | delete | rename ]
[FILE] apparmor_status [ edit | delete | rename ]
[FILE] applygnupgdefaults [ edit | delete | rename ]
[FILE] argdist-bpfcc [ edit | delete | rename ]
[FILE] arpd [ edit | delete | rename ]
[FILE] arptables [ edit | delete | rename ]
[FILE] arptables-nft [ edit | delete | rename ]
[FILE] arptables-nft-restore [ edit | delete | rename ]
[FILE] arptables-nft-save [ edit | delete | rename ]
[FILE] arptables-restore [ edit | delete | rename ]
[FILE] arptables-save [ edit | delete | rename ]
[FILE] badblocks [ edit | delete | rename ]
[FILE] bashreadline-bpfcc [ edit | delete | rename ]
[FILE] bashreadline.bt [ edit | delete | rename ]
[FILE] bcache-super-show [ edit | delete | rename ]
[FILE] bindsnoop-bpfcc [ edit | delete | rename ]
[FILE] biolatency-bpfcc [ edit | delete | rename ]
[FILE] biolatency-kp.bt [ edit | delete | rename ]
[FILE] biolatency.bt [ edit | delete | rename ]
[FILE] biolatpcts-bpfcc [ edit | delete | rename ]
[FILE] biopattern-bpfcc [ edit | delete | rename ]
[FILE] biosdecode [ edit | delete | rename ]
[FILE] biosnoop-bpfcc [ edit | delete | rename ]
[FILE] biosnoop.bt [ edit | delete | rename ]
[FILE] biostacks.bt [ edit | delete | rename ]
[FILE] biotop-bpfcc [ edit | delete | rename ]
[FILE] bitesize-bpfcc [ edit | delete | rename ]
[FILE] bitesize.bt [ edit | delete | rename ]
[FILE] blkdeactivate [ edit | delete | rename ]
[FILE] blkdiscard [ edit | delete | rename ]
[FILE] blkid [ edit | delete | rename ]
[FILE] blkzone [ edit | delete | rename ]
[FILE] blockdev [ edit | delete | rename ]
[FILE] bpflist-bpfcc [ edit | delete | rename ]
[FILE] bpftool [ edit | delete | rename ]
[FILE] bridge [ edit | delete | rename ]
[FILE] btrfsdist-bpfcc [ edit | delete | rename ]
[FILE] btrfsslower-bpfcc [ edit | delete | rename ]
[FILE] cachestat-bpfcc [ edit | delete | rename ]
[FILE] cachetop-bpfcc [ edit | delete | rename ]
[FILE] cache_check [ edit | delete | rename ]
[FILE] cache_dump [ edit | delete | rename ]
[FILE] cache_metadata_size [ edit | delete | rename ]
[FILE] cache_repair [ edit | delete | rename ]
[FILE] cache_restore [ edit | delete | rename ]
[FILE] cache_writeback [ edit | delete | rename ]
[FILE] capable-bpfcc [ edit | delete | rename ]
[FILE] capable.bt [ edit | delete | rename ]
[FILE] capsh [ edit | delete | rename ]
[FILE] cfdisk [ edit | delete | rename ]
[FILE] cgdisk [ edit | delete | rename ]
[FILE] chcpu [ edit | delete | rename ]
[FILE] check_forensic [ edit | delete | rename ]
[FILE] chgpasswd [ edit | delete | rename ]
[FILE] chmem [ edit | delete | rename ]
[FILE] chpasswd [ edit | delete | rename ]
[FILE] chroot [ edit | delete | rename ]
[FILE] cobjnew-bpfcc [ edit | delete | rename ]
[FILE] compactsnoop-bpfcc [ edit | delete | rename ]
[FILE] cpgr [ edit | delete | rename ]
[FILE] cppw [ edit | delete | rename ]
[FILE] cpudist-bpfcc [ edit | delete | rename ]
[FILE] cpuunclaimed-bpfcc [ edit | delete | rename ]
[FILE] cpuwalk.bt [ edit | delete | rename ]
[FILE] criticalstat-bpfcc [ edit | delete | rename ]
[FILE] cron [ edit | delete | rename ]
[FILE] cryptdisks_start [ edit | delete | rename ]
[FILE] cryptdisks_stop [ edit | delete | rename ]
[FILE] cryptsetup [ edit | delete | rename ]
[FILE] ctrlaltdel [ edit | delete | rename ]
[FILE] dbslower-bpfcc [ edit | delete | rename ]
[FILE] dbstat-bpfcc [ edit | delete | rename ]
[FILE] dcb [ edit | delete | rename ]
[FILE] dcsnoop-bpfcc [ edit | delete | rename ]
[FILE] dcsnoop.bt [ edit | delete | rename ]
[FILE] dcstat-bpfcc [ edit | delete | rename ]
[FILE] deadlock-bpfcc [ edit | delete | rename ]
[FILE] debugfs [ edit | delete | rename ]
[FILE] delgroup [ edit | delete | rename ]
[FILE] deluser [ edit | delete | rename ]
[FILE] depmod [ edit | delete | rename ]
[FILE] devlink [ edit | delete | rename ]
[FILE] dhcpcd [ edit | delete | rename ]
[FILE] dirtop-bpfcc [ edit | delete | rename ]
[FILE] dmeventd [ edit | delete | rename ]
[FILE] dmidecode [ edit | delete | rename ]
[FILE] dmsetup [ edit | delete | rename ]
[FILE] dmstats [ edit | delete | rename ]
[FILE] dosfsck [ edit | delete | rename ]
[FILE] dosfslabel [ edit | delete | rename ]
[FILE] dpkg-preconfigure [ edit | delete | rename ]
[FILE] dpkg-reconfigure [ edit | delete | rename ]
[FILE] drsnoop-bpfcc [ edit | delete | rename ]
[FILE] dumpe2fs [ edit | delete | rename ]
[FILE] e2freefrag [ edit | delete | rename ]
[FILE] e2fsck [ edit | delete | rename ]
[FILE] e2image [ edit | delete | rename ]
[FILE] e2label [ edit | delete | rename ]
[FILE] e2mmpstatus [ edit | delete | rename ]
[FILE] e2scrub [ edit | delete | rename ]
[FILE] e2scrub_all [ edit | delete | rename ]
[FILE] e2undo [ edit | delete | rename ]
[FILE] e4crypt [ edit | delete | rename ]
[FILE] e4defrag [ edit | delete | rename ]
[FILE] ebtables [ edit | delete | rename ]
[FILE] ebtables-nft [ edit | delete | rename ]
[FILE] ebtables-nft-restore [ edit | delete | rename ]
[FILE] ebtables-nft-save [ edit | delete | rename ]
[FILE] ebtables-restore [ edit | delete | rename ]
[FILE] ebtables-save [ edit | delete | rename ]
[FILE] ebtables-translate [ edit | delete | rename ]
[FILE] era_check [ edit | delete | rename ]
[FILE] era_dump [ edit | delete | rename ]
[FILE] era_invalidate [ edit | delete | rename ]
[FILE] era_restore [ edit | delete | rename ]
[FILE] escapesrc [ edit | delete | rename ]
[FILE] ethtool [ edit | delete | rename ]
[FILE] execsnoop-bpfcc [ edit | delete | rename ]
[FILE] execsnoop.bt [ edit | delete | rename ]
[FILE] exitsnoop-bpfcc [ edit | delete | rename ]
[FILE] ext4dist-bpfcc [ edit | delete | rename ]
[FILE] ext4slower-bpfcc [ edit | delete | rename ]
[FILE] faillock [ edit | delete | rename ]
[FILE] fatlabel [ edit | delete | rename ]
[FILE] fdisk [ edit | delete | rename ]
[FILE] filefrag [ edit | delete | rename ]
[FILE] filegone-bpfcc [ edit | delete | rename ]
[FILE] filelife-bpfcc [ edit | delete | rename ]
[FILE] fileslower-bpfcc [ edit | delete | rename ]
[FILE] filetop-bpfcc [ edit | delete | rename ]
[FILE] findfs [ edit | delete | rename ]
[FILE] fixparts [ edit | delete | rename ]
[FILE] fsadm [ edit | delete | rename ]
[FILE] fsck [ edit | delete | rename ]
[FILE] fsck.btrfs [ edit | delete | rename ]
[FILE] fsck.cramfs [ edit | delete | rename ]
[FILE] fsck.ext2 [ edit | delete | rename ]
[FILE] fsck.ext3 [ edit | delete | rename ]
[FILE] fsck.ext4 [ edit | delete | rename ]
[FILE] fsck.fat [ edit | delete | rename ]
[FILE] fsck.minix [ edit | delete | rename ]
[FILE] fsck.msdos [ edit | delete | rename ]
[FILE] fsck.vfat [ edit | delete | rename ]
[FILE] fsck.xfs [ edit | delete | rename ]
[FILE] fsfreeze [ edit | delete | rename ]
[FILE] fstab-decode [ edit | delete | rename ]
[FILE] fstrim [ edit | delete | rename ]
[FILE] funccount-bpfcc [ edit | delete | rename ]
[FILE] funcinterval-bpfcc [ edit | delete | rename ]
[FILE] funclatency-bpfcc [ edit | delete | rename ]
[FILE] funcslower-bpfcc [ edit | delete | rename ]
[FILE] gdisk [ edit | delete | rename ]
[FILE] genccode [ edit | delete | rename ]
[FILE] gencmn [ edit | delete | rename ]
[FILE] genl [ edit | delete | rename ]
[FILE] gennorm2 [ edit | delete | rename ]
[FILE] gensprep [ edit | delete | rename ]
[FILE] getcap [ edit | delete | rename ]
[FILE] gethostlatency-bpfcc [ edit | delete | rename ]
[FILE] gethostlatency.bt [ edit | delete | rename ]
[FILE] getpcaps [ edit | delete | rename ]
[FILE] getty [ edit | delete | rename ]
[FILE] groupadd [ edit | delete | rename ]
[FILE] groupdel [ edit | delete | rename ]
[FILE] groupmems [ edit | delete | rename ]
[FILE] groupmod [ edit | delete | rename ]
[FILE] grpck [ edit | delete | rename ]
[FILE] grpconv [ edit | delete | rename ]
[FILE] grpunconv [ edit | delete | rename ]
[FILE] grub-bios-setup [ edit | delete | rename ]
[FILE] grub-install [ edit | delete | rename ]
[FILE] grub-macbless [ edit | delete | rename ]
[FILE] grub-mkconfig [ edit | delete | rename ]
[FILE] grub-mkdevicemap [ edit | delete | rename ]
[FILE] grub-probe [ edit | delete | rename ]
[FILE] grub-reboot [ edit | delete | rename ]
[FILE] grub-set-default [ edit | delete | rename ]
[FILE] halt [ edit | delete | rename ]
[FILE] hardirqs-bpfcc [ edit | delete | rename ]
[FILE] hdparm [ edit | delete | rename ]
[FILE] httxt2dbm [ edit | delete | rename ]
[FILE] iconvconfig [ edit | delete | rename ]
[FILE] icupkg [ edit | delete | rename ]
[FILE] init [ edit | delete | rename ]
[FILE] inject-bpfcc [ edit | delete | rename ]
[FILE] insmod [ edit | delete | rename ]
[FILE] install-sgmlcatalog [ edit | delete | rename ]
[FILE] installkernel [ edit | delete | rename ]
[FILE] integritysetup [ edit | delete | rename ]
[FILE] invoke-rc.d [ edit | delete | rename ]
[FILE] ip [ edit | delete | rename ]
[FILE] ip6tables [ edit | delete | rename ]
[FILE] ip6tables-apply [ edit | delete | rename ]
[FILE] ip6tables-legacy [ edit | delete | rename ]
[FILE] ip6tables-legacy-restore [ edit | delete | rename ]
[FILE] ip6tables-legacy-save [ edit | delete | rename ]
[FILE] ip6tables-nft [ edit | delete | rename ]
[FILE] ip6tables-nft-restore [ edit | delete | rename ]
[FILE] ip6tables-nft-save [ edit | delete | rename ]
[FILE] ip6tables-restore [ edit | delete | rename ]
[FILE] ip6tables-restore-translate [ edit | delete | rename ]
[FILE] ip6tables-save [ edit | delete | rename ]
[FILE] ip6tables-translate [ edit | delete | rename ]
[FILE] iptables [ edit | delete | rename ]
[FILE] iptables-apply [ edit | delete | rename ]
[FILE] iptables-legacy [ edit | delete | rename ]
[FILE] iptables-legacy-restore [ edit | delete | rename ]
[FILE] iptables-legacy-save [ edit | delete | rename ]
[FILE] iptables-nft [ edit | delete | rename ]
[FILE] iptables-nft-restore [ edit | delete | rename ]
[FILE] iptables-nft-save [ edit | delete | rename ]
[FILE] iptables-restore [ edit | delete | rename ]
[FILE] iptables-restore-translate [ edit | delete | rename ]
[FILE] iptables-save [ edit | delete | rename ]
[FILE] iptables-translate [ edit | delete | rename ]
[FILE] iscsi-iname [ edit | delete | rename ]
[FILE] iscsiadm [ edit | delete | rename ]
[FILE] iscsid [ edit | delete | rename ]
[FILE] iscsistart [ edit | delete | rename ]
[FILE] iscsi_discovery [ edit | delete | rename ]
[FILE] isosize [ edit | delete | rename ]
[FILE] javacalls-bpfcc [ edit | delete | rename ]
[FILE] javaflow-bpfcc [ edit | delete | rename ]
[FILE] javagc-bpfcc [ edit | delete | rename ]
[FILE] javaobjnew-bpfcc [ edit | delete | rename ]
[FILE] javastat-bpfcc [ edit | delete | rename ]
[FILE] javathreads-bpfcc [ edit | delete | rename ]
[FILE] kbdrate [ edit | delete | rename ]
[FILE] killall5 [ edit | delete | rename ]
[FILE] killsnoop-bpfcc [ edit | delete | rename ]
[FILE] killsnoop.bt [ edit | delete | rename ]
[FILE] klockstat-bpfcc [ edit | delete | rename ]
[FILE] kpartx [ edit | delete | rename ]
[FILE] kvmexit-bpfcc [ edit | delete | rename ]
[FILE] ldattach [ edit | delete | rename ]
[FILE] ldconfig [ edit | delete | rename ]
[FILE] ldconfig.real [ edit | delete | rename ]
[FILE] llcstat-bpfcc [ edit | delete | rename ]
[FILE] loads.bt [ edit | delete | rename ]
[FILE] locale-gen [ edit | delete | rename ]
[FILE] logrotate [ edit | delete | rename ]
[FILE] logsave [ edit | delete | rename ]
[FILE] losetup [ edit | delete | rename ]
[FILE] lsmod [ edit | delete | rename ]
[FILE] luksformat [ edit | delete | rename ]
[FILE] lvchange [ edit | delete | rename ]
[FILE] lvconvert [ edit | delete | rename ]
[FILE] lvcreate [ edit | delete | rename ]
[FILE] lvdisplay [ edit | delete | rename ]
[FILE] lvextend [ edit | delete | rename ]
[FILE] lvm [ edit | delete | rename ]
[FILE] lvmconfig [ edit | delete | rename ]
[FILE] lvmdiskscan [ edit | delete | rename ]
[FILE] lvmdump [ edit | delete | rename ]
[FILE] lvmpolld [ edit | delete | rename ]
[FILE] lvmsadc [ edit | delete | rename ]
[FILE] lvmsar [ edit | delete | rename ]
[FILE] lvreduce [ edit | delete | rename ]
[FILE] lvremove [ edit | delete | rename ]
[FILE] lvrename [ edit | delete | rename ]
[FILE] lvresize [ edit | delete | rename ]
[FILE] lvs [ edit | delete | rename ]
[FILE] lvscan [ edit | delete | rename ]
[FILE] lxc [ edit | delete | rename ]
[FILE] lxd [ edit | delete | rename ]
[FILE] make-bcache [ edit | delete | rename ]
[FILE] make-ssl-cert [ edit | delete | rename ]
[FILE] mdadm [ edit | delete | rename ]
[FILE] mdflush-bpfcc [ edit | delete | rename ]
[FILE] mdflush.bt [ edit | delete | rename ]
[FILE] mdmon [ edit | delete | rename ]
[FILE] memleak-bpfcc [ edit | delete | rename ]
[FILE] mkdosfs [ edit | delete | rename ]
[FILE] mke2fs [ edit | delete | rename ]
[FILE] mkfs [ edit | delete | rename ]
[FILE] mkfs.bfs [ edit | delete | rename ]
[FILE] mkfs.btrfs [ edit | delete | rename ]
[FILE] mkfs.cramfs [ edit | delete | rename ]
[FILE] mkfs.ext2 [ edit | delete | rename ]
[FILE] mkfs.ext3 [ edit | delete | rename ]
[FILE] mkfs.ext4 [ edit | delete | rename ]
[FILE] mkfs.fat [ edit | delete | rename ]
[FILE] mkfs.minix [ edit | delete | rename ]
[FILE] mkfs.msdos [ edit | delete | rename ]
[FILE] mkfs.ntfs [ edit | delete | rename ]
[FILE] mkfs.vfat [ edit | delete | rename ]
[FILE] mkfs.xfs [ edit | delete | rename ]
[FILE] mkhomedir_helper [ edit | delete | rename ]
[FILE] mkinitramfs [ edit | delete | rename ]
[FILE] mklost+found [ edit | delete | rename ]
[FILE] mkntfs [ edit | delete | rename ]
[FILE] mkswap [ edit | delete | rename ]
[FILE] ModemManager [ edit | delete | rename ]
[FILE] modinfo [ edit | delete | rename ]
[FILE] modprobe [ edit | delete | rename ]
[FILE] mount.fuse [ edit | delete | rename ]
[FILE] mount.fuse3 [ edit | delete | rename ]
[FILE] mount.lowntfs-3g [ edit | delete | rename ]
[FILE] mount.ntfs [ edit | delete | rename ]
[FILE] mount.ntfs-3g [ edit | delete | rename ]
[FILE] mountsnoop-bpfcc [ edit | delete | rename ]
[FILE] mpathpersist [ edit | delete | rename ]
[FILE] multipath [ edit | delete | rename ]
[FILE] multipathc [ edit | delete | rename ]
[FILE] multipathd [ edit | delete | rename ]
[FILE] mysqld [ edit | delete | rename ]
[FILE] mysqld_qslower-bpfcc [ edit | delete | rename ]
[FILE] naptime.bt [ edit | delete | rename ]
[FILE] needrestart [ edit | delete | rename ]
[FILE] netplan [ edit | delete | rename ]
[FILE] netqtop-bpfcc [ edit | delete | rename ]
[FILE] newusers [ edit | delete | rename ]
[FILE] nfnl_osf [ edit | delete | rename ]
[FILE] nfsdist-bpfcc [ edit | delete | rename ]
[FILE] nfsslower-bpfcc [ edit | delete | rename ]
[FILE] nft [ edit | delete | rename ]
[FILE] nginx [ edit | delete | rename ]
[FILE] nodegc-bpfcc [ edit | delete | rename ]
[FILE] nodestat-bpfcc [ edit | delete | rename ]
[FILE] nologin [ edit | delete | rename ]
[FILE] ntfsclone [ edit | delete | rename ]
[FILE] ntfscp [ edit | delete | rename ]
[FILE] ntfslabel [ edit | delete | rename ]
[FILE] ntfsresize [ edit | delete | rename ]
[FILE] ntfsundelete [ edit | delete | rename ]
[FILE] offcputime-bpfcc [ edit | delete | rename ]
[FILE] offwaketime-bpfcc [ edit | delete | rename ]
[FILE] on_ac_power [ edit | delete | rename ]
[FILE] oomkill-bpfcc [ edit | delete | rename ]
[FILE] oomkill.bt [ edit | delete | rename ]
[FILE] opendkim [ edit | delete | rename ]
[FILE] opendkim-atpszone [ edit | delete | rename ]
[FILE] opendkim-expire [ edit | delete | rename ]
[FILE] opendkim-gengraphs [ edit | delete | rename ]
[FILE] opendkim-genkey [ edit | delete | rename ]
[FILE] opendkim-genstats [ edit | delete | rename ]
[FILE] opendkim-genzone [ edit | delete | rename ]
[FILE] opendkim-importstats [ edit | delete | rename ]
[FILE] opendkim-stats [ edit | delete | rename ]
[FILE] opendkim-testkey [ edit | delete | rename ]
[FILE] opendkim-testmsg [ edit | delete | rename ]
[FILE] opensnoop-bpfcc [ edit | delete | rename ]
[FILE] opensnoop.bt [ edit | delete | rename ]
[FILE] overlayroot-chroot [ edit | delete | rename ]
[FILE] ownership [ edit | delete | rename ]
[FILE] pam-auth-update [ edit | delete | rename ]
[FILE] pam_extrausers_chkpwd [ edit | delete | rename ]
[FILE] pam_extrausers_update [ edit | delete | rename ]
[FILE] pam_getenv [ edit | delete | rename ]
[FILE] pam_namespace_helper [ edit | delete | rename ]
[FILE] pam_timestamp_check [ edit | delete | rename ]
[FILE] paperconfig [ edit | delete | rename ]
[FILE] parted [ edit | delete | rename ]
[FILE] partprobe [ edit | delete | rename ]
[FILE] pdata_tools [ edit | delete | rename ]
[FILE] perlcalls-bpfcc [ edit | delete | rename ]
[FILE] perlflow-bpfcc [ edit | delete | rename ]
[FILE] perlstat-bpfcc [ edit | delete | rename ]
[FILE] pg_updatedicts [ edit | delete | rename ]
[FILE] phpcalls-bpfcc [ edit | delete | rename ]
[FILE] phpdismod [ edit | delete | rename ]
[FILE] phpenmod [ edit | delete | rename ]
[FILE] phpflow-bpfcc [ edit | delete | rename ]
[FILE] phpquery [ edit | delete | rename ]
[FILE] phpstat-bpfcc [ edit | delete | rename ]
[FILE] pidpersec-bpfcc [ edit | delete | rename ]
[FILE] pidpersec.bt [ edit | delete | rename ]
[FILE] pivot_root [ edit | delete | rename ]
[FILE] plocate-build [ edit | delete | rename ]
[FILE] plymouthd [ edit | delete | rename ]
[FILE] postalias [ edit | delete | rename ]
[FILE] postcat [ edit | delete | rename ]
[FILE] postconf [ edit | delete | rename ]
[FILE] postdrop [ edit | delete | rename ]
[FILE] postfix [ edit | delete | rename ]
[FILE] postfix-add-filter [ edit | delete | rename ]
[FILE] postfix-add-policy [ edit | delete | rename ]
[FILE] postfix-collate [ edit | delete | rename ]
[FILE] postkick [ edit | delete | rename ]
[FILE] postlock [ edit | delete | rename ]
[FILE] postlog [ edit | delete | rename ]
[FILE] postmap [ edit | delete | rename ]
[FILE] postmulti [ edit | delete | rename ]
[FILE] postqueue [ edit | delete | rename ]
[FILE] postsuper [ edit | delete | rename ]
[FILE] posttls-finger [ edit | delete | rename ]
[FILE] poweroff [ edit | delete | rename ]
[FILE] ppchcalls-bpfcc [ edit | delete | rename ]
[FILE] profile-bpfcc [ edit | delete | rename ]
[FILE] pvchange [ edit | delete | rename ]
[FILE] pvck [ edit | delete | rename ]
[FILE] pvcreate [ edit | delete | rename ]
[FILE] pvdisplay [ edit | delete | rename ]
[FILE] pvmove [ edit | delete | rename ]
[FILE] pvremove [ edit | delete | rename ]
[FILE] pvresize [ edit | delete | rename ]
[FILE] pvs [ edit | delete | rename ]
[FILE] pvscan [ edit | delete | rename ]
[FILE] pwck [ edit | delete | rename ]
[FILE] pwconv [ edit | delete | rename ]
[FILE] pwhistory_helper [ edit | delete | rename ]
[FILE] pwunconv [ edit | delete | rename ]
[FILE] pythoncalls-bpfcc [ edit | delete | rename ]
[FILE] pythonflow-bpfcc [ edit | delete | rename ]
[FILE] pythongc-bpfcc [ edit | delete | rename ]
[FILE] pythonstat-bpfcc [ edit | delete | rename ]
[FILE] qemu-ga [ edit | delete | rename ]
[FILE] qmqp-sink [ edit | delete | rename ]
[FILE] qmqp-source [ edit | delete | rename ]
[FILE] qshape [ edit | delete | rename ]
[FILE] rdmaucma-bpfcc [ edit | delete | rename ]
[FILE] readahead-bpfcc [ edit | delete | rename ]
[FILE] readprofile [ edit | delete | rename ]
[FILE] reboot [ edit | delete | rename ]
[FILE] remove-shell [ edit | delete | rename ]
[FILE] reset-trace-bpfcc [ edit | delete | rename ]
[FILE] resize2fs [ edit | delete | rename ]
[FILE] resolvconf [ edit | delete | rename ]
[FILE] rmail [ edit | delete | rename ]
[FILE] rmmod [ edit | delete | rename ]
[FILE] rmt [ edit | delete | rename ]
[FILE] rmt-tar [ edit | delete | rename ]
[FILE] rsyslogd [ edit | delete | rename ]
[FILE] rtacct [ edit | delete | rename ]
[FILE] rtcwake [ edit | delete | rename ]
[FILE] rtmon [ edit | delete | rename ]
[FILE] rubycalls-bpfcc [ edit | delete | rename ]
[FILE] rubyflow-bpfcc [ edit | delete | rename ]
[FILE] rubygc-bpfcc [ edit | delete | rename ]
[FILE] rubyobjnew-bpfcc [ edit | delete | rename ]
[FILE] rubystat-bpfcc [ edit | delete | rename ]
[FILE] runlevel [ edit | delete | rename ]
[FILE] runqlat-bpfcc [ edit | delete | rename ]
[FILE] runqlat.bt [ edit | delete | rename ]
[FILE] runqlen-bpfcc [ edit | delete | rename ]
[FILE] runqlen.bt [ edit | delete | rename ]
[FILE] runqslower-bpfcc [ edit | delete | rename ]
[FILE] runuser [ edit | delete | rename ]
[FILE] sendmail [ edit | delete | rename ]
[FILE] service [ edit | delete | rename ]
[FILE] setcap [ edit | delete | rename ]
[FILE] setuids.bt [ edit | delete | rename ]
[FILE] setvesablank [ edit | delete | rename ]
[FILE] setvtrgb [ edit | delete | rename ]
[FILE] sfdisk [ edit | delete | rename ]
[FILE] sgdisk [ edit | delete | rename ]
[FILE] shadowconfig [ edit | delete | rename ]
[FILE] shmsnoop-bpfcc [ edit | delete | rename ]
[FILE] shutdown [ edit | delete | rename ]
[FILE] slabratetop-bpfcc [ edit | delete | rename ]
[FILE] smtp-sink [ edit | delete | rename ]
[FILE] smtp-source [ edit | delete | rename ]
[FILE] sofdsnoop-bpfcc [ edit | delete | rename ]
[FILE] softirqs-bpfcc [ edit | delete | rename ]
[FILE] solisten-bpfcc [ edit | delete | rename ]
[FILE] split-logfile [ edit | delete | rename ]
[FILE] sshd [ edit | delete | rename ]
[FILE] ssllatency.bt [ edit | delete | rename ]
[FILE] sslsniff-bpfcc [ edit | delete | rename ]
[FILE] sslsnoop.bt [ edit | delete | rename ]
[FILE] stackcount-bpfcc [ edit | delete | rename ]
[FILE] start-stop-daemon [ edit | delete | rename ]
[FILE] statsnoop-bpfcc [ edit | delete | rename ]
[FILE] statsnoop.bt [ edit | delete | rename ]
[FILE] sudo_logsrvd [ edit | delete | rename ]
[FILE] sudo_sendlog [ edit | delete | rename ]
[FILE] sulogin [ edit | delete | rename ]
[FILE] swapin.bt [ edit | delete | rename ]
[FILE] swaplabel [ edit | delete | rename ]
[FILE] swapoff [ edit | delete | rename ]
[FILE] swapon [ edit | delete | rename ]
[FILE] switch_root [ edit | delete | rename ]
[FILE] syncsnoop-bpfcc [ edit | delete | rename ]
[FILE] syncsnoop.bt [ edit | delete | rename ]
[FILE] syscount-bpfcc [ edit | delete | rename ]
[FILE] syscount.bt [ edit | delete | rename ]
[FILE] sysctl [ edit | delete | rename ]
[FILE] tarcat [ edit | delete | rename ]
[FILE] tc [ edit | delete | rename ]
[FILE] tclcalls-bpfcc [ edit | delete | rename ]
[FILE] tclflow-bpfcc [ edit | delete | rename ]
[FILE] tclobjnew-bpfcc [ edit | delete | rename ]
[FILE] tclstat-bpfcc [ edit | delete | rename ]
[FILE] tcpaccept-bpfcc [ edit | delete | rename ]
[FILE] tcpaccept.bt [ edit | delete | rename ]
[FILE] tcpcong-bpfcc [ edit | delete | rename ]
[FILE] tcpconnect-bpfcc [ edit | delete | rename ]
[FILE] tcpconnect.bt [ edit | delete | rename ]
[FILE] tcpconnlat-bpfcc [ edit | delete | rename ]
[FILE] tcpdrop-bpfcc [ edit | delete | rename ]
[FILE] tcpdrop.bt [ edit | delete | rename ]
[FILE] tcplife-bpfcc [ edit | delete | rename ]
[FILE] tcplife.bt [ edit | delete | rename ]
[FILE] tcpretrans-bpfcc [ edit | delete | rename ]
[FILE] tcpretrans.bt [ edit | delete | rename ]
[FILE] tcprtt-bpfcc [ edit | delete | rename ]
[FILE] tcpstates-bpfcc [ edit | delete | rename ]
[FILE] tcpsubnet-bpfcc [ edit | delete | rename ]
[FILE] tcpsynbl-bpfcc [ edit | delete | rename ]
[FILE] tcpsynbl.bt [ edit | delete | rename ]
[FILE] tcptop-bpfcc [ edit | delete | rename ]
[FILE] tcptracer-bpfcc [ edit | delete | rename ]
[FILE] telinit [ edit | delete | rename ]
[FILE] thin_check [ edit | delete | rename ]
[FILE] thin_delta [ edit | delete | rename ]
[FILE] thin_dump [ edit | delete | rename ]
[FILE] thin_ls [ edit | delete | rename ]
[FILE] thin_metadata_size [ edit | delete | rename ]
[FILE] thin_repair [ edit | delete | rename ]
[FILE] thin_restore [ edit | delete | rename ]
[FILE] thin_rmap [ edit | delete | rename ]
[FILE] thin_trim [ edit | delete | rename ]
[FILE] threadsnoop-bpfcc [ edit | delete | rename ]
[FILE] threadsnoop.bt [ edit | delete | rename ]
[FILE] tipc [ edit | delete | rename ]
[FILE] tplist-bpfcc [ edit | delete | rename ]
[FILE] trace-bpfcc [ edit | delete | rename ]
[FILE] ttysnoop-bpfcc [ edit | delete | rename ]
[FILE] tune2fs [ edit | delete | rename ]
[FILE] ucalls [ edit | delete | rename ]
[FILE] uflow [ edit | delete | rename ]
[FILE] ufw [ edit | delete | rename ]
[FILE] ugc [ edit | delete | rename ]
[FILE] umount.udisks2 [ edit | delete | rename ]
[FILE] undump.bt [ edit | delete | rename ]
[FILE] unix_chkpwd [ edit | delete | rename ]
[FILE] unix_update [ edit | delete | rename ]
[FILE] uobjnew [ edit | delete | rename ]
[FILE] update-ca-certificates [ edit | delete | rename ]
[FILE] update-catalog [ edit | delete | rename ]
[FILE] update-fonts-alias [ edit | delete | rename ]
[FILE] update-fonts-dir [ edit | delete | rename ]
[FILE] update-fonts-scale [ edit | delete | rename ]
[FILE] update-grub [ edit | delete | rename ]
[FILE] update-grub-gfxpayload [ edit | delete | rename ]
[FILE] update-grub2 [ edit | delete | rename ]
[FILE] update-gsfontmap [ edit | delete | rename ]
[FILE] update-icon-caches [ edit | delete | rename ]
[FILE] update-ieee-data [ edit | delete | rename ]
[FILE] update-info-dir [ edit | delete | rename ]
[FILE] update-initramfs [ edit | delete | rename ]
[FILE] update-locale [ edit | delete | rename ]
[FILE] update-mime [ edit | delete | rename ]
[FILE] update-passwd [ edit | delete | rename ]
[FILE] update-pciids [ edit | delete | rename ]
[FILE] update-rc.d [ edit | delete | rename ]
[FILE] update-secureboot-policy [ edit | delete | rename ]
[FILE] update-shells [ edit | delete | rename ]
[FILE] update-xmlcatalog [ edit | delete | rename ]
[FILE] updatedb.plocate [ edit | delete | rename ]
[FILE] upgrade-from-grub-legacy [ edit | delete | rename ]
[FILE] usb_modeswitch [ edit | delete | rename ]
[FILE] usb_modeswitch_dispatcher [ edit | delete | rename ]
[FILE] useradd [ edit | delete | rename ]
[FILE] userdel [ edit | delete | rename ]
[FILE] usermod [ edit | delete | rename ]
[FILE] ustat [ edit | delete | rename ]
[FILE] uthreads [ edit | delete | rename ]
[FILE] uuidd [ edit | delete | rename ]
[FILE] validlocale [ edit | delete | rename ]
[FILE] vcstime [ edit | delete | rename ]
[FILE] vdpa [ edit | delete | rename ]
[FILE] veritysetup [ edit | delete | rename ]
[FILE] vfscount-bpfcc [ edit | delete | rename ]
[FILE] vfscount.bt [ edit | delete | rename ]
[FILE] vfsstat-bpfcc [ edit | delete | rename ]
[FILE] vfsstat.bt [ edit | delete | rename ]
[FILE] vgcfgbackup [ edit | delete | rename ]
[FILE] vgcfgrestore [ edit | delete | rename ]
[FILE] vgchange [ edit | delete | rename ]
[FILE] vgck [ edit | delete | rename ]
[FILE] vgconvert [ edit | delete | rename ]
[FILE] vgcreate [ edit | delete | rename ]
[FILE] vgdisplay [ edit | delete | rename ]
[FILE] vgexport [ edit | delete | rename ]
[FILE] vgextend [ edit | delete | rename ]
[FILE] vgimport [ edit | delete | rename ]
[FILE] vgimportclone [ edit | delete | rename ]
[FILE] vgmerge [ edit | delete | rename ]
[FILE] vgmknodes [ edit | delete | rename ]
[FILE] vgreduce [ edit | delete | rename ]
[FILE] vgremove [ edit | delete | rename ]
[FILE] vgrename [ edit | delete | rename ]
[FILE] vgs [ edit | delete | rename ]
[FILE] vgscan [ edit | delete | rename ]
[FILE] vgsplit [ edit | delete | rename ]
[FILE] vigr [ edit | delete | rename ]
[FILE] vipw [ edit | delete | rename ]
[FILE] virtiostat-bpfcc [ edit | delete | rename ]
[FILE] visudo [ edit | delete | rename ]
[FILE] vpddecode [ edit | delete | rename ]
[FILE] wakeuptime-bpfcc [ edit | delete | rename ]
[FILE] wipefs [ edit | delete | rename ]
[FILE] writeback.bt [ edit | delete | rename ]
[FILE] xfsdist-bpfcc [ edit | delete | rename ]
[FILE] xfsdist.bt [ edit | delete | rename ]
[FILE] xfsslower-bpfcc [ edit | delete | rename ]
[FILE] xfs_admin [ edit | delete | rename ]
[FILE] xfs_bmap [ edit | delete | rename ]
[FILE] xfs_copy [ edit | delete | rename ]
[FILE] xfs_db [ edit | delete | rename ]
[FILE] xfs_estimate [ edit | delete | rename ]
[FILE] xfs_freeze [ edit | delete | rename ]
[FILE] xfs_fsr [ edit | delete | rename ]
[FILE] xfs_growfs [ edit | delete | rename ]
[FILE] xfs_info [ edit | delete | rename ]
[FILE] xfs_io [ edit | delete | rename ]
[FILE] xfs_logprint [ edit | delete | rename ]
[FILE] xfs_mdrestore [ edit | delete | rename ]
[FILE] xfs_metadump [ edit | delete | rename ]
[FILE] xfs_mkfile [ edit | delete | rename ]
[FILE] xfs_ncheck [ edit | delete | rename ]
[FILE] xfs_quota [ edit | delete | rename ]
[FILE] xfs_repair [ edit | delete | rename ]
[FILE] xfs_rtcp [ edit | delete | rename ]
[FILE] xfs_scrub [ edit | delete | rename ]
[FILE] xfs_scrub_all [ edit | delete | rename ]
[FILE] xfs_spaceman [ edit | delete | rename ]
[FILE] xtables-legacy-multi [ edit | delete | rename ]
[FILE] xtables-monitor [ edit | delete | rename ]
[FILE] xtables-nft-multi [ edit | delete | rename ]
[FILE] zerofree [ edit | delete | rename ]
[FILE] zfsdist-bpfcc [ edit | delete | rename ]
[FILE] zfsslower-bpfcc [ edit | delete | rename ]
[FILE] zic [ edit | delete | rename ]
[FILE] zramctl [ edit | delete | rename ]
Viewing: /usr/sbin/deadlock-bpfcc
#! /usr/bin/python3
#
# deadlock Detects potential deadlocks (lock order inversions)
# on a running process. For Linux, uses BCC, eBPF.
#
# USAGE: deadlock.py [-h] [--binary BINARY] [--dump-graph DUMP_GRAPH]
# [--verbose] [--lock-symbols LOCK_SYMBOLS]
# [--unlock-symbols UNLOCK_SYMBOLS]
# pid
#
# This traces pthread mutex lock and unlock calls to build a directed graph
# representing the mutex wait graph:
#
# - Nodes in the graph represent mutexes.
# - Edge (A, B) exists if there exists some thread T where lock(A) was called
# and lock(B) was called before unlock(A) was called.
#
# If the program finds a potential lock order inversion, the program will dump
# the cycle of mutexes and the stack traces where each mutex was acquired, and
# then exit.
#
# This program can only find potential deadlocks that occur while the program
# is tracing the process. It cannot find deadlocks that may have occurred
# before the program was attached to the process.
#
# Since this traces all mutex lock and unlock events and all thread creation
# events on the traced process, the overhead of this bpf program can be very
# high if the process has many threads and mutexes. You should only run this on
# a process where the slowdown is acceptable.
#
# Note: This tool does not work for shared mutexes or recursive mutexes.
#
# For shared (read-write) mutexes, a deadlock requires a cycle in the wait
# graph where at least one of the mutexes in the cycle is acquiring exclusive
# (write) ownership.
#
# For recursive mutexes, lock() is called multiple times on the same mutex.
# However, there is no way to determine if a mutex is a recursive mutex
# after the mutex has been created. As a result, this tool will not find
# potential deadlocks that involve only one mutex.
#
# Copyright 2017 Facebook, Inc.
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 01-Feb-2017 Kenny Yu Created this.
from __future__ import (
absolute_import, division, unicode_literals, print_function
)
from bcc import BPF
from collections import defaultdict
import argparse
import json
import os
import subprocess
import sys
import time
class DiGraph(object):
'''
Adapted from networkx: http://networkx.github.io/
Represents a directed graph. Edges can store (key, value) attributes.
'''
def __init__(self):
# Map of node -> set of nodes
self.adjacency_map = {}
# Map of (node1, node2) -> map string -> arbitrary attribute
# This will not be copied in subgraph()
self.attributes_map = {}
def neighbors(self, node):
return self.adjacency_map.get(node, set())
def edges(self):
edges = []
for node, neighbors in self.adjacency_map.items():
for neighbor in neighbors:
edges.append((node, neighbor))
return edges
def nodes(self):
return self.adjacency_map.keys()
def attributes(self, node1, node2):
return self.attributes_map[(node1, node2)]
def add_edge(self, node1, node2, **kwargs):
if node1 not in self.adjacency_map:
self.adjacency_map[node1] = set()
if node2 not in self.adjacency_map:
self.adjacency_map[node2] = set()
self.adjacency_map[node1].add(node2)
self.attributes_map[(node1, node2)] = kwargs
def remove_node(self, node):
self.adjacency_map.pop(node, None)
for _, neighbors in self.adjacency_map.items():
neighbors.discard(node)
def subgraph(self, nodes):
graph = DiGraph()
for node in nodes:
for neighbor in self.neighbors(node):
if neighbor in nodes:
graph.add_edge(node, neighbor)
return graph
def node_link_data(self):
'''
Returns the graph as a dictionary in a format that can be
serialized.
'''
data = {
'directed': True,
'multigraph': False,
'graph': {},
'links': [],
'nodes': [],
}
# Do one pass to build a map of node -> position in nodes
node_to_number = {}
for node in self.adjacency_map.keys():
node_to_number[node] = len(data['nodes'])
data['nodes'].append({'id': node})
# Do another pass to build the link information
for node, neighbors in self.adjacency_map.items():
for neighbor in neighbors:
link = self.attributes_map[(node, neighbor)].copy()
link['source'] = node_to_number[node]
link['target'] = node_to_number[neighbor]
data['links'].append(link)
return data
def strongly_connected_components(G):
'''
Adapted from networkx: http://networkx.github.io/
Parameters
----------
G : DiGraph
Returns
-------
comp : generator of sets
A generator of sets of nodes, one for each strongly connected
component of G.
'''
preorder = {}
lowlink = {}
scc_found = {}
scc_queue = []
i = 0 # Preorder counter
for source in G.nodes():
if source not in scc_found:
queue = [source]
while queue:
v = queue[-1]
if v not in preorder:
i = i + 1
preorder[v] = i
done = 1
v_nbrs = G.neighbors(v)
for w in v_nbrs:
if w not in preorder:
queue.append(w)
done = 0
break
if done == 1:
lowlink[v] = preorder[v]
for w in v_nbrs:
if w not in scc_found:
if preorder[w] > preorder[v]:
lowlink[v] = min([lowlink[v], lowlink[w]])
else:
lowlink[v] = min([lowlink[v], preorder[w]])
queue.pop()
if lowlink[v] == preorder[v]:
scc_found[v] = True
scc = {v}
while (
scc_queue and preorder[scc_queue[-1]] > preorder[v]
):
k = scc_queue.pop()
scc_found[k] = True
scc.add(k)
yield scc
else:
scc_queue.append(v)
def simple_cycles(G):
'''
Adapted from networkx: http://networkx.github.io/
Parameters
----------
G : DiGraph
Returns
-------
cycle_generator: generator
A generator that produces elementary cycles of the graph.
Each cycle is represented by a list of nodes along the cycle.
'''
def _unblock(thisnode, blocked, B):
stack = set([thisnode])
while stack:
node = stack.pop()
if node in blocked:
blocked.remove(node)
stack.update(B[node])
B[node].clear()
# Johnson's algorithm requires some ordering of the nodes.
# We assign the arbitrary ordering given by the strongly connected comps
# There is no need to track the ordering as each node removed as processed.
# save the actual graph so we can mutate it here
# We only take the edges because we do not want to
# copy edge and node attributes here.
subG = G.subgraph(G.nodes())
sccs = list(strongly_connected_components(subG))
while sccs:
scc = sccs.pop()
# order of scc determines ordering of nodes
startnode = scc.pop()
# Processing node runs 'circuit' routine from recursive version
path = [startnode]
blocked = set() # vertex: blocked from search?
closed = set() # nodes involved in a cycle
blocked.add(startnode)
B = defaultdict(set) # graph portions that yield no elementary circuit
stack = [(startnode, list(subG.neighbors(startnode)))]
while stack:
thisnode, nbrs = stack[-1]
if nbrs:
nextnode = nbrs.pop()
if nextnode == startnode:
yield path[:]
closed.update(path)
elif nextnode not in blocked:
path.append(nextnode)
stack.append((nextnode, list(subG.neighbors(nextnode))))
closed.discard(nextnode)
blocked.add(nextnode)
continue
# done with nextnode... look for more neighbors
if not nbrs: # no more nbrs
if thisnode in closed:
_unblock(thisnode, blocked, B)
else:
for nbr in subG.neighbors(thisnode):
if thisnode not in B[nbr]:
B[nbr].add(thisnode)
stack.pop()
path.pop()
# done processing this node
subG.remove_node(startnode)
H = subG.subgraph(scc) # make smaller to avoid work in SCC routine
sccs.extend(list(strongly_connected_components(H)))
def find_cycle(graph):
'''
Looks for a cycle in the graph. If found, returns the first cycle.
If nodes a1, a2, ..., an are in a cycle, then this returns:
[(a1,a2), (a2,a3), ... (an-1,an), (an, a1)]
Otherwise returns an empty list.
'''
cycles = list(simple_cycles(graph))
if cycles:
nodes = cycles[0]
nodes.append(nodes[0])
edges = []
prev = nodes[0]
for node in nodes[1:]:
edges.append((prev, node))
prev = node
return edges
else:
return []
def print_cycle(binary, graph, edges, thread_info, print_stack_trace_fn):
'''
Prints the cycle in the mutex graph in the following format:
Potential Deadlock Detected!
Cycle in lock order graph: M0 => M1 => M2 => M0
for (m, n) in cycle:
Mutex n acquired here while holding Mutex m in thread T:
[ stack trace ]
Mutex m previously acquired by thread T here:
[ stack trace ]
for T in all threads:
Thread T was created here:
[ stack trace ]
'''
# List of mutexes in the cycle, first and last repeated
nodes_in_order = []
# Map mutex address -> readable alias
node_addr_to_name = {}
for counter, (m, n) in enumerate(edges):
nodes_in_order.append(m)
# For global or static variables, try to symbolize the mutex address.
symbol = symbolize_with_objdump(binary, m)
if symbol:
symbol += ' '
node_addr_to_name[m] = 'Mutex M%d (%s0x%016x)' % (counter, symbol, m)
nodes_in_order.append(nodes_in_order[0])
print('----------------\nPotential Deadlock Detected!\n')
print(
'Cycle in lock order graph: %s\n' %
(' => '.join([node_addr_to_name[n] for n in nodes_in_order]))
)
# Set of threads involved in the lock inversion
thread_pids = set()
# For each edge in the cycle, print where the two mutexes were held
for (m, n) in edges:
thread_pid = graph.attributes(m, n)['thread_pid']
thread_comm = graph.attributes(m, n)['thread_comm']
first_mutex_stack_id = graph.attributes(m, n)['first_mutex_stack_id']
second_mutex_stack_id = graph.attributes(m, n)['second_mutex_stack_id']
thread_pids.add(thread_pid)
print(
'%s acquired here while holding %s in Thread %d (%s):' % (
node_addr_to_name[n], node_addr_to_name[m], thread_pid,
thread_comm
)
)
print_stack_trace_fn(second_mutex_stack_id)
print('')
print(
'%s previously acquired by the same Thread %d (%s) here:' %
(node_addr_to_name[m], thread_pid, thread_comm)
)
print_stack_trace_fn(first_mutex_stack_id)
print('')
# Print where the threads were created, if available
for thread_pid in thread_pids:
parent_pid, stack_id, parent_comm = thread_info.get(
thread_pid, (None, None, None)
)
if parent_pid:
print(
'Thread %d created by Thread %d (%s) here: ' %
(thread_pid, parent_pid, parent_comm)
)
print_stack_trace_fn(stack_id)
else:
print(
'Could not find stack trace where Thread %d was created' %
thread_pid
)
print('')
def symbolize_with_objdump(binary, addr):
'''
Searches the binary for the address using objdump. Returns the symbol if
it is found, otherwise returns empty string.
'''
try:
command = (
'objdump -tT %s | grep %x | awk {\'print $NF\'} | c++filt' %
(binary, addr)
)
output = subprocess.check_output(command, shell=True)
return output.decode('utf-8').strip()
except subprocess.CalledProcessError:
return ''
def strlist(s):
'''Given a comma-separated string, returns a list of substrings'''
return s.strip().split(',')
def main():
examples = '''Examples:
deadlock 181 # Analyze PID 181
deadlock 181 --binary /lib/x86_64-linux-gnu/libpthread.so.0
# Analyze PID 181 and locks from this binary.
# If tracing a process that is running from
# a dynamically-linked binary, this argument
# is required and should be the path to the
# pthread library.
deadlock 181 --verbose
# Analyze PID 181 and print statistics about
# the mutex wait graph.
deadlock 181 --lock-symbols my_mutex_lock1,my_mutex_lock2 \\
--unlock-symbols my_mutex_unlock1,my_mutex_unlock2
# Analyze PID 181 and trace custom mutex
# symbols instead of pthread mutexes.
deadlock 181 --dump-graph graph.json
# Analyze PID 181 and dump the mutex wait
# graph to graph.json.
'''
parser = argparse.ArgumentParser(
description=(
'Detect potential deadlocks (lock inversions) in a running binary.'
'\nMust be run as root.'
),
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=examples,
)
parser.add_argument('pid', type=int, help='Pid to trace')
# Binaries with `:` in the path will fail to attach uprobes on kernels
# running without this patch: https://lkml.org/lkml/2017/1/13/585.
# Symlinks to the binary without `:` in the path can get around this issue.
parser.add_argument(
'--binary',
type=str,
default='',
help='If set, trace the mutexes from the binary at this path. '
'For statically-linked binaries, this argument is not required. '
'For dynamically-linked binaries, this argument is required and '
'should be the path of the pthread library the binary is using. '
'Example: /lib/x86_64-linux-gnu/libpthread.so.0',
)
parser.add_argument(
'--dump-graph',
type=str,
default='',
help='If set, this will dump the mutex graph to the specified file.',
)
parser.add_argument(
'--verbose',
action='store_true',
help='Print statistics about the mutex wait graph.',
)
parser.add_argument(
'--lock-symbols',
type=strlist,
default=['pthread_mutex_lock'],
help='Comma-separated list of lock symbols to trace. Default is '
'pthread_mutex_lock. These symbols cannot be inlined in the binary.',
)
parser.add_argument(
'--unlock-symbols',
type=strlist,
default=['pthread_mutex_unlock'],
help='Comma-separated list of unlock symbols to trace. Default is '
'pthread_mutex_unlock. These symbols cannot be inlined in the binary.',
)
parser.add_argument(
'-t', '--threads', type=int, default=65536,
help='Specifies the maximum number of threads to trace. default 65536. '
'Note. 40 bytes per thread.'
)
parser.add_argument(
'-e', '--edges', type=int, default=65536,
help='Specifies the maximum number of edge cases that can be recorded. '
'default 65536. Note. 88 bytes per edge case.'
)
parser.add_argument(
'-s', '--stacktraces', type=int, default=65536,
help='Specifies the maximum number of stack traces that can be recorded. '
'This number is rounded up to the next power of two.'
'default 65536. Note. 1 kbytes vmalloced per stack trace.'
)
args = parser.parse_args()
if not args.binary:
try:
args.binary = os.readlink('/proc/%d/exe' % args.pid)
except OSError as e:
print('%s. Is the process (pid=%d) running?' % (str(e), args.pid))
sys.exit(1)
with open('/usr/share/bpfcc-tools/deadlock.c') as f:
text = f.read()
text = text.replace('MAX_THREADS', str(args.threads));
text = text.replace('MAX_EDGES', str(args.edges));
text = text.replace('MAX_TRACES', str(args.stacktraces));
bpf = BPF(text=text)
# Trace where threads are created
bpf.attach_kretprobe(event=bpf.get_syscall_fnname('clone'), fn_name='trace_clone')
# We must trace unlock first, otherwise in the time we attached the probe
# on lock() and have not yet attached the probe on unlock(), a thread can
# acquire mutexes and release them, but the release events will not be
# traced, resulting in noisy reports.
for symbol in args.unlock_symbols:
try:
bpf.attach_uprobe(
name=args.binary,
sym=symbol,
fn_name='trace_mutex_release',
pid=args.pid,
)
except Exception as e:
print('%s. Failed to attach to symbol: %s\nIs --binary argument missing?' % (str(e), symbol))
sys.exit(1)
for symbol in args.lock_symbols:
try:
bpf.attach_uprobe(
name=args.binary,
sym=symbol,
fn_name='trace_mutex_acquire',
pid=args.pid,
)
except Exception as e:
print('%s. Failed to attach to symbol: %s' % (str(e), symbol))
sys.exit(1)
def print_stack_trace(stack_id):
'''Closure that prints the symbolized stack trace.'''
for addr in bpf.get_table('stack_traces').walk(stack_id):
line = bpf.sym(addr, args.pid)
# Try to symbolize with objdump if we cannot with bpf.
if line == '[unknown]':
symbol = symbolize_with_objdump(args.binary, addr)
if symbol:
line = symbol
print('@ %016x %s' % (addr, line))
print('Tracing... Hit Ctrl-C to end.')
while True:
try:
# Map of child thread pid -> parent info
thread_info = {
child.value: (parent.parent_pid, parent.stack_id, parent.comm)
for child, parent in bpf.get_table('thread_to_parent').items()
}
# Mutex wait directed graph. Nodes are mutexes. Edge (A,B) exists
# if there exists some thread T where lock(A) was called and
# lock(B) was called before unlock(A) was called.
graph = DiGraph()
for key, leaf in bpf.get_table('edges').items():
graph.add_edge(
key.mutex1,
key.mutex2,
thread_pid=leaf.thread_pid,
thread_comm=leaf.comm.decode('utf-8'),
first_mutex_stack_id=leaf.mutex1_stack_id,
second_mutex_stack_id=leaf.mutex2_stack_id,
)
if args.verbose:
print(
'Mutexes: %d, Edges: %d' %
(len(graph.nodes()), len(graph.edges()))
)
if args.dump_graph:
with open(args.dump_graph, 'w') as f:
data = graph.node_link_data()
f.write(json.dumps(data, indent=2))
cycle = find_cycle(graph)
if cycle:
print_cycle(
args.binary, graph, cycle, thread_info, print_stack_trace
)
sys.exit(1)
time.sleep(1)
except KeyboardInterrupt:
break
if __name__ == '__main__':
main()