Complete report #788
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
2
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
3
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
4
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
5
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
6
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
7
|
?
__nf_nat_alloc_null_binding
|
nf_nat |
|
0x5a
|
|
8
|
?
nf_ct_del_from_dying_or_unconfirmed_list
|
nf_conntrack |
|
0x5e
|
|
9
|
?
nf_ct_del_from_dying_or_unconfirmed_list
|
nf_conntrack |
|
0x5e
|
|
10
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
11
|
|
vmlinux |
|
0x89
|
|
12
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
13
|
|
vmlinux |
|
0x89
|
|
14
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
15
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
16
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
17
|
|
vmlinux |
|
0x89
|
|
18
|
|
vmlinux |
|
0x89
|
|
19
|
|
vmlinux |
|
0x89
|
|
20
|
|
vmlinux |
|
0x89
|
|
21
|
|
vmlinux |
|
0x89
|
|
22
|
|
vmlinux |
|
0x89
|
|
23
|
|
vmlinux |
|
0x89
|
|
24
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
25
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
26
|
|
vmlinux |
|
0x89
|
|
27
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
28
|
|
vmlinux |
|
0x89
|
|
29
|
|
vmlinux |
|
0x89
|
|
30
|
|
vmlinux |
|
0x89
|
|
31
|
|
vmlinux |
|
0x89
|
|
32
|
|
vmlinux |
|
0x89
|
|
33
|
|
vmlinux |
|
0x89
|
|
34
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
35
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
36
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
watchdog: BUG: soft lockup - CPU#0 stuck for 23s! [chronyd:734]
Modules linked in: ip_set_hash_net xt_set ip_set_hash_ip tcp_diag inet_diag xt_multiport xt_CHECKSUM ipt_MASQUERADE xt_conntrack ipt_REJECT nft_compat nf_nat_tftp nft_objref nf_conntrack_tftp nft_counter bridge stp llc cfg80211 rfkill nft_fib_inet nft_fib_ipv4 nft_fib_ipv6 nft_fib nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nf_tables_set nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip_set nf_tables libcrc32c nfnetlink sunrpc crct10dif_pclmul crc32_pclmul ghash_clmulni_intel vmw_vsock_virtio_transport vmw_vsock_virtio_transport_common joydev vsock pcspkr virtio_balloon i2c_piix4 tcp_bbr sch_fq ext4 mbcache jbd2 sd_mod t10_pi sg ata_generic virtio_net cirrus net_failover failover drm_shmem_helper drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm ata_piix crc32c_intel libata serio_raw virtio_console virtio_scsi fuse
CPU: 0 PID: 734 Comm: chronyd Kdump: loaded Tainted: G L --------- - - 4.18.0-477.15.1.el8_8.x86_64 #1
Hardware name: Red Hat KVM, BIOS 1.16.3-5.el9.alma.1 11/05/2023
RIP: 0010:native_safe_halt+0xe/0x20
Code: 00 f0 80 48 02 20 48 8b 00 a8 08 75 c0 e9 79 ff ff ff 90 90 90 90 90 90 90 90 90 90 e9 07 00 00 00 0f 00 2d 06 a0 60 00 fb f4 <e9> 1d 12 40 00 66 66 2e 0f 1f 84 00 00 00 00 00 66 90 e9 07 00 00
RSP: 0018:ffffba39c1083898 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13
RAX: 0000000000000003 RBX: ffff94d9b4ac1cc0 RCX: 0000000000000008
RDX: 0000000000000000 RSI: 0000000000000003 RDI: ffff94d9b4ac1cc0
RBP: ffff94d9f5c33cc0 R08: 0000000000000008 R09: 0000000000000094
R10: ffff94d984a60000 R11: 000000004bb8b66b R12: 0000000000000000
R13: 0000000000000001 R14: 0000000000000100 R15: 0000000000040000
FS: 00007f8068710940(0000) GS:ffff94d9f5c00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 00007f8332abc150 CR3: 000000000300c004 CR4: 0000000000370ef0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
kvm_wait+0x58/0x60
__pv_queued_spin_lock_slowpath+0x268/0x2a0
_raw_spin_lock+0x1e/0x30
sch_direct_xmit+0x127/0x370
__qdisc_run+0x152/0x580
__dev_queue_xmit+0x47c/0xba0
? __nf_nat_alloc_null_binding+0x5a/0x80 [nf_nat]
? nf_ct_del_from_dying_or_unconfirmed_list+0x5e/0x70 [nf_conntrack]
? __nf_conntrack_confirm+0x394/0x4b0 [nf_conntrack]
ip_finish_output2+0x26d/0x430
? __ip_finish_output+0xdf/0x1d0
ip_output+0x70/0xf0
? __ip_finish_output+0x1d0/0x1d0
ip_send_skb+0x15/0x40
udp_send_skb.isra.51+0x152/0x360
udp_sendmsg+0xa87/0xd80
? context_struct_compute_av+0x270/0x450
? avc_compute_av+0x12c/0x180
? sock_has_perm+0x80/0xa0
? avc_alloc_node+0x2/0x130
? __check_object_size+0xac/0x173
? sock_sendmsg+0x42/0x60
? udp_push_pending_frames+0x70/0x70
sock_sendmsg+0x42/0x60
____sys_sendmsg+0x22a/0x250
? copy_msghdr_from_user+0x5c/0x90
___sys_sendmsg+0x7c/0xc0
? rt_cache_seq_stop+0x10/0x10
? ip4_datagram_release_cb+0x4b/0x1e0
? __queue_work+0x145/0x3f0
? release_sock+0x43/0x90
? _raw_spin_unlock_bh+0xa/0x20
? sock_setsockopt+0xec/0xc20
__sys_sendmsg+0x57/0xa0
do_syscall_64+0x5b/0x1b0
entry_SYSCALL_64_after_hwframe+0x61/0xc6
RIP: 0033:0x7f80675b7a27
Code: c3 66 90 41 54 41 89 d4 55 48 89 f5 53 89 fb 48 83 ec 10 e8 0b ed ff ff 44 89 e2 48 89 ee 89 df 41 89 c0 b8 2e 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 35 44 89 c7 48 89 44 24 08 e8 44 ed ff ff 48
RSP: 002b:00007ffe68f7d4e0 EFLAGS: 00000293 ORIG_RAX: 000000000000002e
RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 00007f80675b7a27
RDX: 0000000000000000 RSI: 00007ffe68f7d520 RDI: 0000000000000004
RBP: 00007ffe68f7d520 R08: 0000000000000000 R09: 0000000000000030
R10: 000000000be2a7a2 R11: 0000000000000293 R12: 0000000000000000
R13: 00007ffe68f7d520 R14: 0000000000000001 R15: 00007ffe68f7d930
Complete report #790
| Frame # |
Function |
Binary |
Source or offset |
Line |
|
1
|
|
vmlinux |
|
0x89
|
|
2
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
3
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
4
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
5
|
|
vmlinux |
|
0x89
|
|
6
|
|
vmlinux |
|
0x89
|
|
7
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
8
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
9
|
|
vmlinux |
|
0x89
|
|
10
|
|
vmlinux |
|
0x89
|
|
11
|
|
vmlinux |
|
0x89
|
|
12
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
13
|
|
vmlinux |
|
0x89
|
|
14
|
|
vmlinux |
|
0x89
|
|
15
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
16
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
17
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
18
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
19
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
20
|
|
vmlinux |
|
0x89
|
|
21
|
|
vmlinux |
|
0x89
|
|
22
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
23
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
24
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
25
|
|
vmlinux |
|
0x89
|
|
26
|
|
vmlinux |
|
0x89
|
|
27
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
28
|
|
vmlinux |
|
0x89
|
|
29
|
|
vmlinux |
|
0x89
|
|
30
|
|
vmlinux |
|
0x89
|
|
31
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
|
32
|
|
vmlinux |
|
0x89
|
|
33
|
__wake_up_common_lock
|
vmlinux |
|
0x89
|
watchdog: BUG: soft lockup - CPU#2 stuck for 21s! [khugepaged:41]
Modules linked in: ip_set_hash_net xt_set ip_set_hash_ip tcp_diag inet_diag xt_multiport xt_CHECKSUM ipt_MASQUERADE xt_conntrack ipt_REJECT nft_compat nf_nat_tftp nft_objref nf_conntrack_tftp nft_counter bridge stp llc cfg80211 rfkill nft_fib_inet nft_fib_ipv4 nft_fib_ipv6 nft_fib nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nf_tables_set nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip_set nf_tables libcrc32c nfnetlink sunrpc crct10dif_pclmul crc32_pclmul ghash_clmulni_intel vmw_vsock_virtio_transport vmw_vsock_virtio_transport_common joydev vsock pcspkr virtio_balloon i2c_piix4 tcp_bbr sch_fq ext4 mbcache jbd2 sd_mod t10_pi sg ata_generic virtio_net cirrus net_failover failover drm_shmem_helper drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops drm ata_piix crc32c_intel libata serio_raw virtio_console virtio_scsi fuse
CPU: 2 PID: 41 Comm: khugepaged Kdump: loaded Tainted: G L --------- - - 4.18.0-477.15.1.el8_8.x86_64 #1
Hardware name: Red Hat KVM, BIOS 1.16.3-5.el9.alma.1 11/05/2023
RIP: 0010:smp_call_function_single+0xd5/0xf0
Code: 00 00 00 75 3a c9 e9 aa 64 c5 00 48 89 74 24 10 48 89 e6 48 89 54 24 18 e8 a8 fe ff ff 8b 14 24 83 e2 01 74 0a f3 90 8b 14 24 <83> e2 01 75 f6 eb c4 8b 05 36 0b 50 02 85 c0 75 84 0f 0b eb 80 e8
RSP: 0018:ffffba39c07a3720 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13
RAX: 0000000000000000 RBX: ffff94d98b957658 RCX: 0000000000000830
RDX: 0000000000000011 RSI: 00000000000000fb RDI: 0000000000000830
RBP: ffffba39c07a3760 R08: 0000000000000000 R09: 0000000000000025
R10: 0000000000000007 R11: 00000000fffffff2 R12: ffff94d98b957200
R13: 0000000000000001 R14: 0000000000000009 R15: ffff94d982e67740
FS: 0000000000000000(0000) GS:ffff94d9f5d00000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 000055d50f557378 CR3: 0000000007c10005 CR4: 0000000000370ee0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
? flush_tlb_func_common.constprop.9+0x220/0x220
flush_tlb_mm_range+0xde/0x110
ptep_clear_flush+0x58/0x70
try_to_unmap_one+0x24e/0xc10
? __mod_memcg_lruvec_state+0x4a/0xc0
? remove_migration_pte+0x7f/0x420
rmap_walk_file+0xe6/0x280
try_to_unmap+0x9c/0xf0
? page_remove_rmap+0x320/0x320
? anon_vma_ctor+0x40/0x40
? page_get_anon_vma+0x80/0x80
migrate_pages+0x8c9/0xb80
? isolate_freepages_block+0x410/0x410
? __bpf_trace_mm_compaction_kcompactd_sleep+0x10/0x10
compact_zone+0x759/0xdb0
compact_zone_order+0xbe/0x100
try_to_compact_pages+0xaf/0x250
__alloc_pages_direct_compact+0x87/0x170
__alloc_pages_slowpath+0x1be/0xcd0
? reweight_entity+0x155/0x170
? ttwu_do_wakeup+0x19/0x170
__alloc_pages_nodemask+0x2e2/0x330
khugepaged_alloc_page+0x17/0x50
collapse_huge_page+0x93/0x1000
? set_next_entity+0xb5/0x1e0
? apic_timer_interrupt+0xa/0x20
khugepaged+0xed9/0x11e0
? __schedule+0x2d9/0x870
? finish_wait+0x80/0x80
? collapse_pte_mapped_thp+0x430/0x430
kthread+0x134/0x150
? set_kthread_struct+0x50/0x50
ret_from_fork+0x35/0x40