CPU Utilization by the mogilefsd process
Carl Forsythe
carl at immi.com
Wed Feb 7 23:52:28 UTC 2007
Brad Fitzpatrick wrote:
> So one of the child processes died and the parent process has two sockets
> open to it which are spinning writable...
>
> Interesting.
>
> We should be able to fix it without the info, but for my curiosity: can
> you tell which child process is dying? If you run it in the foreground,
> does it say "Unexpected death from ........" or similar. Showing pid and
> function of child.
>
I restarted it in the foreground on one of the two trackers and we're
doing a 10000 file insert to see if we can get it back into the same
state. On the second tracker, one of the queryworker threads is out of
sequence with the other pids, so perhaps it was the one who died and the
parent recreated a new one to keep it at the correct levels? It appears
as if the child at pid 6464 is one the that is missing on this box.
500 6453 19.8 0.8 100964 16664 ? R Feb03 1411:39
/usr/bin/perl /usr/bin/mogilefsd -c /etc/mogilefs/mogilefsd.conf --daemon
500 6454 0.0 0.8 100964 16848 ? S Feb03 0:41
/usr/bin/mogilefsd [checker]
500 6455 0.0 0.8 101332 17300 ? S Feb03 3:11
/usr/bin/mogilefsd [replicate]
500 6456 0.0 0.8 101332 17324 ? S Feb03 3:20
/usr/bin/mogilefsd [replicate]
500 6457 0.0 0.8 101448 17308 ? S Feb03 3:03
/usr/bin/mogilefsd [replicate]
500 6458 0.0 0.8 101332 17304 ? S Feb03 3:07
/usr/bin/mogilefsd [replicate]
500 6459 0.0 0.8 101444 17304 ? S Feb03 3:09
/usr/bin/mogilefsd [replicate]
500 6460 0.0 0.8 101096 17232 ? S Feb03 0:46
/usr/bin/mogilefsd [delete]
500 6461 0.0 0.8 100964 16972 ? S Feb03 0:37
/usr/bin/mogilefsd [queryworker]
500 6462 0.0 0.8 100964 16948 ? S Feb03 0:30
/usr/bin/mogilefsd [queryworker]
500 6463 0.0 0.8 100964 16956 ? S Feb03 0:26
/usr/bin/mogilefsd [queryworker]
500 6465 0.0 0.8 100964 16944 ? S Feb03 0:26
/usr/bin/mogilefsd [queryworker]
500 6466 0.0 0.8 100964 16956 ? S Feb03 0:40
/usr/bin/mogilefsd [queryworker]
500 6467 0.0 0.8 100964 16956 ? S Feb03 0:32
/usr/bin/mogilefsd [queryworker]
500 6468 0.0 0.8 100964 16960 ? S Feb03 0:30
/usr/bin/mogilefsd [queryworker]
500 6469 0.0 0.8 100964 16988 ? S Feb03 0:36
/usr/bin/mogilefsd [queryworker]
500 6470 0.0 0.8 100964 16972 ? S Feb03 0:25
/usr/bin/mogilefsd [queryworker]
500 6471 0.0 1.0 110424 22468 ? S Feb03 4:38
/usr/bin/mogilefsd [monitor]
500 6472 0.0 0.8 100964 16896 ? S Feb03 0:45
/usr/bin/mogilefsd [reaper]
500 3454 0.0 0.8 101076 17012 ? S Feb05 0:17
/usr/bin/mogilefsd [queryworker] <-------
>
> On Wed, 7 Feb 2007, Carl Forsythe wrote:
>
>
>> Brad Fitzpatrick wrote:
>>
>>> What version?
>>>
>> Latest one from svn just installed on the nodes on Friday of last week
>> (727?). Version 2 as a more general version number.
>>
>>> strace it and tell me what it's doing? And show output of lsof -p <pid>
>>> as well?
>>>
>> strace output:
>> epoll_wait(7, {{EPOLLOUT, {u32=10, u64=10}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=11, u64=11}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>> epoll_wait(7, {{EPOLLOUT, {u32=11, u64=11}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=10, u64=10}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>> epoll_wait(7, {{EPOLLOUT, {u32=10, u64=10}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=11, u64=11}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>> epoll_wait(7, {{EPOLLOUT, {u32=11, u64=11}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=10, u64=10}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>> epoll_wait(7, {{EPOLLOUT, {u32=10, u64=10}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=11, u64=11}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>> epoll_wait(7, {{EPOLLOUT, {u32=11, u64=11}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=10, u64=10}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>> epoll_wait(7, {{EPOLLOUT, {u32=10, u64=10}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=11, u64=11}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>> epoll_wait(7, {{EPOLLOUT, {u32=11, u64=11}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=10, u64=10}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>> epoll_wait(7, {{EPOLLOUT, {u32=10, u64=10}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=11, u64=11}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>> epoll_wait(7, {{EPOLLOUT, {u32=11, u64=11}}, {EPOLLOUT, {u32=9, u64=9}},
>> {EPOLLOUT, {u32=10, u64=10}}}, 1000, 250) = 3
>> time(NULL) = 1170886869
>>
>> and so on and so on and so on.
>>
>>
>> lsof output:
>> [root at mogdb00 ~]# lsof -p 6558
>> COMMAND PID USER FD TYPE DEVICE SIZE NODE
>> NAME
>> mogilefsd 6558 mogilefsd cwd DIR 8,2 4096 2 /
>> mogilefsd 6558 mogilefsd rtd DIR 8,2 4096 2 /
>> mogilefsd 6558 mogilefsd txt REG 8,2 21024 4978702
>> /usr/bin/perl
>> mogilefsd 6558 mogilefsd mem REG 8,2 48520720 4977414
>> /usr/lib/locale/locale-archive
>> mogilefsd 6558 mogilefsd mem REG 8,2 19520 5040752
>> /usr/lib64/perl5/5.8.5/x86_64-linux-thread-multi/auto/IO/IO.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 23632 5040927
>> /usr/lib64/perl5/5.8.5/x86_64-linux-thread-multi/auto/Socket/Socket.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 123664 5040765
>> /usr/lib64/perl5/5.8.5/x86_64-linux-thread-multi/auto/POSIX/POSIX.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 52044 5042813
>> /usr/lib64/perl5/5.8.5/x86_64-linux-thread-multi/auto/Cwd/Cwd.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 8040 5040954
>> /usr/lib64/perl5/5.8.5/x86_64-linux-thread-multi/auto/Sys/Hostname/Hostname.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 16128 5040957
>> /usr/lib64/perl5/5.8.5/x86_64-linux-thread-multi/auto/Sys/Syslog/Syslog.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 84080 5320464
>> /usr/lib64/perl5/5.8.5/x86_64-linux-thread-multi/auto/Time/HiRes/HiRes.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 34352 5040758
>> /usr/lib64/perl5/5.8.5/x86_64-linux-thread-multi/auto/List/Util/Util.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 68390 5320352
>> /usr/lib64/perl5/site_perl/5.8.5/x86_64-linux-thread-multi/auto/BSD/Resource/Resource.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 315483 5203911
>> /usr/lib64/perl5/site_perl/5.8.5/x86_64-linux-thread-multi/auto/DBI/DBI.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 330140 5320581
>> /usr/lib64/perl5/site_perl/5.8.5/x86_64-linux-thread-multi/auto/DBD/mysql/mysql.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 10384 6156230
>> /lib64/libcom_err.so.2.1
>> mogilefsd 6558 mogilefsd mem REG 8,2 56902 6156030
>> /lib64/libnss_files-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 23009 6156028
>> /lib64/libnss_dns-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 105080 6156227
>> /lib64/ld-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 1493409 6156228
>> /lib64/tls/libc-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 17943 6156232
>> /lib64/libdl-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 613297 6156236
>> /lib64/tls/libm-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 79336 4987377
>> /usr/lib64/libz.so.1.2.1.2
>> mogilefsd 6558 mogilefsd mem REG 8,2 30070 6156238
>> /lib64/libcrypt-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 91412 6156231
>> /lib64/libresolv-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 107187 6156054
>> /lib64/libnsl-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 106203 6156073
>> /lib64/tls/libpthread-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 1385848 5042812
>> /usr/lib64/perl5/5.8.5/x86_64-linux-thread-multi/CORE/libperl.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 464040 4987375
>> /usr/lib64/libkrb5.so.3.2
>> mogilefsd 6558 mogilefsd mem REG 8,2 145456 4983599
>> /usr/lib64/libk5crypto.so.3.0
>> mogilefsd 6558 mogilefsd mem REG 8,2 17367 6156075
>> /lib64/libutil-2.3.4.so
>> mogilefsd 6558 mogilefsd mem REG 8,2 93832 4987376
>> /usr/lib64/libgssapi_krb5.so.2.2
>> mogilefsd 6558 mogilefsd mem REG 8,2 244288 6156234
>> /lib64/libssl.so.0.9.7a
>> mogilefsd 6558 mogilefsd mem REG 8,2 1230168 6156233
>> /lib64/libcrypto.so.0.9.7a
>> mogilefsd 6558 mogilefsd mem REG 8,2 1494376 5318825
>> /usr/lib64/mysql/libmysqlclient.so.15.0.0
>> mogilefsd 6558 mogilefsd 0u CHR 1,3 2133
>> /dev/null
>> mogilefsd 6558 mogilefsd 1u CHR 1,3 2133
>> /dev/null
>> mogilefsd 6558 mogilefsd 2u CHR 1,3 2133
>> /dev/null
>> mogilefsd 6558 mogilefsd 3r REG 8,2 2131 4987369
>> /usr/bin/mogilefsd
>> mogilefsd 6558 mogilefsd 4u sock 0,4 16156
>> can't identify protocol
>> mogilefsd 6558 mogilefsd 5u unix 0x000001007cfb46c0 16165
>> socket
>> mogilefsd 6558 mogilefsd 6u IPv4 16166 TCP
>> *:afs3-callback (LISTEN)
>> mogilefsd 6558 mogilefsd 7r 0000 0,8 0 16167
>> eventpoll
>> mogilefsd 6558 mogilefsd 8u unix 0x000001000a1f0c40 16168
>> socket
>> mogilefsd 6558 mogilefsd 9u unix 0x000001000a1f0640 16170
>> socket
>> mogilefsd 6558 mogilefsd 10u unix 0x000001007cfb49c0 16172
>> socket
>> mogilefsd 6558 mogilefsd 11u unix 0x000001007c826980 16174
>> socket
>> mogilefsd 6558 mogilefsd 12u unix 0x000001007c826c80 16176
>> socket
>> mogilefsd 6558 mogilefsd 13u unix 0x000001007c860380 16178
>> socket
>> mogilefsd 6558 mogilefsd 14u unix 0x000001007cffc640 16180
>> socket
>> mogilefsd 6558 mogilefsd 15u unix 0x000001007cffc040 16182
>> socket
>> mogilefsd 6558 mogilefsd 16u unix 0x0000010066740040 34322
>> socket
>> mogilefsd 6558 mogilefsd 17u unix 0x000001006f820c80 16186
>> socket
>> mogilefsd 6558 mogilefsd 18u unix 0x000001000938ac80 16188
>> socket
>> mogilefsd 6558 mogilefsd 19u unix 0x000001000938a680 16190
>> socket
>> mogilefsd 6558 mogilefsd 20u unix 0x000001000938a080 16192
>> socket
>> mogilefsd 6558 mogilefsd 21u unix 0x00000100088039c0 16194
>> socket
>> mogilefsd 6558 mogilefsd 22u unix 0x00000100088033c0 16196
>> socket
>> mogilefsd 6558 mogilefsd 23u unix 0x00000100084dcc40 16198
>> socket
>> mogilefsd 6558 mogilefsd 24u unix 0x00000100084dc640 16200
>> socket
>> mogilefsd 6558 mogilefsd 25u unix 0x000001000a1f0040 16202
>> socket
>> mogilefsd 6558 mogilefsd 26u unix 0x000001000805d980 16204
>> socket
>>
>>>
>>> On Wed, 7 Feb 2007, Carl Forsythe wrote:
>>>
>>>
>>>> Hi all,
>>>>
>>>> On our trackers I see one mogilefsd process in top (the parent process)
>>>> sitting at 100% cpu utilization, is this typical behavior for the parent
>>>> mogilefsd? The tracker platform is CentOS 4.4 64 bit running on AMD
>>>> Opteron processors.
>>>>
>>>> Thanks,
>>>>
>>>> -Carl
>>>>
>>>>
>>>>
>>>>
>>>>
>>
More information about the mogilefs
mailing list