Segfault in scheduler thread

[Mike Rice <mikerice1969@…>]

Running SVN trunk 16313. May just be memory corruption but hopefully someone can see something wrong from the trace. The full stack trace is attached.

Core was generated by `/usr/local/bin/mythbackend --noupnp --daemon -v most,noschedule,nojobqueue --lo'.
Program terminated with signal 11, Segmentation fault.
#0  0x06ffbc61 in malloc_consolidate () from /lib/libc.so.6
#1  0x06ffdd8d in _int_malloc () from /lib/libc.so.6
#2  0x06fffb7b in malloc () from /lib/libc.so.6
#3  0x04d3dbfd in my_malloc () from /usr/lib/mysql/libmysqlclient.so.15
#4  0x04d40d7a in alloc_root () from /usr/lib/mysql/libmysqlclient.so.15
#5  0x04d63300 in cli_read_rows () from /usr/lib/mysql/libmysqlclient.so.15
#6  0x04d63d41 in ?? () from /usr/lib/mysql/libmysqlclient.so.15
#7  0x04d62244 in mysql_real_query () from /usr/lib/mysql/libmysqlclient.so.15
#8  0x04974557 in QMYSQLResult::reset () from /usr/lib/qt-3.3/plugins/sqldrivers/libqsqlmysql.so
#9  0x0220200b in QSqlQuery::exec () from /usr/lib/qt-3.3/lib/libqt-mt.so.3
#10 0x00962706 in MSqlQuery::exec (this=0xb35fa85c, query=@0xb35fa7f0) at mythdbcon.cpp:390
        result = 2
#11 0x022026b1 in QSqlQuery::exec () from /usr/lib/qt-3.3/lib/libqt-mt.so.3
#12 0x08076f7d in MSqlQuery::exec (this=0xb35fa85c) at ../../libs/libmyth/mythdbcon.h:111
No locals.
#13 0x00ef2a42 in CardUtil::GetInputGroups (inputid=2) at cardutil.cpp:1133
        list = (
    std::vector<unsigned int,std::allocator<unsigned int> > &) @0xb35fa9f0: {<std::_Vector_base<unsigned int,std::allocator<unsigned int> >> = {_M_impl = {<std::allocator<unsigned int>> = {<__gnu_cxx::new_allocator<unsigned int>> = {<No data fields>}, <No data fields>},
      _M_start = 0x0, _M_finish = 0x0, _M_end_of_storage = 0x0}}, <No data fields>}
        list = {<std::_Vector_base<unsigned int,std::allocator<unsigned int> >> = {
    _M_impl = {<std::allocator<unsigned int>> = {<__gnu_cxx::new_allocator<unsigned int>> = {<No data fields>}, <No data fields>},
      _M_start = 0x2284ecd, _M_finish = 0x10, _M_end_of_storage = 0x70ea128}}, <No data fields>}
        query = {<> = {<No data fields>}, m_db = 0xac650a50, m_isConnected = true, m_returnConnection = true}
#14 0x00ef5dcd in CardUtil::GetConflictingCards (inputid=2, exclude_cardid=2) at cardutil.cpp:1198
        cardids = (
    std::vector<unsigned int,std::allocator<unsigned int> > &) @0xb35fab78: {<std::_Vector_base<unsigned int,std::allocator<unsigned int> >> = {_M_impl = {<std::allocator<unsigned int>> = {<__gnu_cxx::new_allocator<unsigned int>> = {<No data fields>}, <No data fields>},
      _M_start = 0xb35fab98, _M_finish = 0x80743c2, _M_end_of_storage = 0xb35fabc8}}, <No data fields>}
        inputgroupids = {<std::_Vector_base<unsigned int,std::allocator<unsigned int> >> = {
    _M_impl = {<std::allocator<unsigned int>> = {<__gnu_cxx::new_allocator<unsigned int>> = {<No data fields>}, <No data fields>},
      _M_start = 0x0, _M_finish = 0x0, _M_end_of_storage = 0x0}}, <No data fields>}
        cardids = {<std::_Vector_base<unsigned int,std::allocator<unsigned int> >> = {
    _M_impl = {<std::allocator<unsigned int>> = {<__gnu_cxx::new_allocator<unsigned int>> = {<No data fields>}, <No data fields>},
      _M_start = 0x251e93c, _M_finish = 0xb35faa18, _M_end_of_storage = 0x8074e62}}, <No data fields>}
#15 0x080e387c in Scheduler::IsBusyRecording (this=0x8395880, rcinfo=0x8439a48) at scheduler.cpp:1476
        rctv = (EncoderLink *) 0x83be680
        busy_input = {<InputInfo> = {_vptr.InputInfo = 0x81612a8, name = {static null = {
        static null = <same as static member of an already seen type>, d = 0x82960c0, static shared_null = 0x82960c0}, d = 0x82960c0,
      static shared_null = 0x82960c0}, sourceid = 0, inputid = 0, cardid = 0, mplexid = 0}, chanid = 0}
        inputid = 2
        cardids = {<std::_Vector_base<unsigned int,std::allocator<unsigned int> >> = {
    _M_impl = {<std::allocator<unsigned int>> = {<__gnu_cxx::new_allocator<unsigned int>> = {<No data fields>}, <No data fields>},
      _M_start = 0xb35fab98, _M_finish = 0x80743c2, _M_end_of_storage = 0xb35fabc8}}, <No data fields>}
#16 0x080f4b00 in Scheduler::RunScheduler (this=0x8395880) at scheduler.cpp:1741
        msg = {static null = {static null = <same as static member of an already seen type>, d = 0x82960c0,
    static shared_null = 0x82960c0}, d = 0x82960c0, static shared_null = 0x82960c0}
        fsID = -1
        lockit = {mtx = 0x8388c30}
        subtitle = {static null = {static null = <same as static member of an already seen type>, d = 0x82960c0,
    static shared_null = 0x82960c0}, d = 0x8c87700, static shared_null = 0x82960c0}
        details = {static null = {static null = <same as static member of an already seen type>, d = 0x82960c0,
    static shared_null = 0x82960c0}, d = 0x82960c0, static shared_null = 0x82960c0}
        doSchedAfterStart = false
        is_rec = true
        statuschanged = false
        recIter = {_M_cur = 0x876b690, _M_first = 0x876b538, _M_last = 0x876b738, _M_node = 0xac32cd6c}
        prerollseconds = 60
        secsleft = 86
        nexttv = (EncoderLink *) 0x83be680
        nextRecording = (ProgramInfo *) 0x8439a48
        nextrectime = {d = {jd = 2454528}, t = {ds = 72000000}}
        schedid = {static null = {static null = <same as static member of an already seen type>, d = 0x82960c0,
    static shared_null = 0x82960c0}, d = 0x93ada50, static shared_null = 0x82960c0}
        curtime = {d = {jd = 2454528}, t = {ds = 71913120}}
        lastupdate = {d = {jd = 2454528}, t = {ds = 71904035}}
        startIter = {_M_cur = 0x876b67c, _M_first = 0x876b538, _M_last = 0x876b738, _M_node = 0xac32cd6c}
        blockShutdown = false
        idleSince = {d = {jd = 0}, t = {ds = 0}}
        idleTimeoutSecs = 0
        idleWaitForRecordingTime = 15
        firstRun = false
        fillstart = {tv_sec = 1204516704, tv_usec = 36836}
        fillend = {tv_sec = 1204516712, tv_usec = 31287}
        matchTime = 0.00151199999
        placeTime = 7.99445105
        query = {<> = {<No data fields>}, m_db = 0x83885c8, m_isConnected = true, m_returnConnection = false}
#17 0x080f6e6a in Scheduler::SchedulerThread (param=0x8395880) at scheduler.cpp:2083
        sched = (Scheduler *) 0x8395880
#18 0x03f4e50b in start_thread () from /lib/libpthread.so.0

[Mike Rice <mikerice1969@…>]

backtrace

[cg@…]

When do you see this? I see something similar when surfing channels, but not every time.

[tim@…]

Replying to Mike Rice <mikerice1969@gmail.com>:

Running SVN trunk 16313. May just be memory corruption but hopefully someone can see something wrong from the trace. The full stack trace is attached.

I'm seeing an abort from glibc when channel hopping in LiveTV. It looks to me like a memory corruption somewhere. I added

if (me->Message().left(6) == "SIGNAL") {

QString crashme=me->ExtraDataList?().join(" ");

}

at the top of the various checks in MainServer::customEvent() in mythbackend/mainserver.cpp and this line triggers the crash according to kdbg and the resulting coredump. The QStringList is complete gibberish at this point.

However, I also added comparable QString crashme=slist.join(" ") statements into the 3 places where the SIGNAL message is generated and none of them fired, which suggests to me that something is scribbling on memory somewhere else.

valgrind is hard to use in this case but it does have several complaints I'm investigating. I may be able to do some more poking this evening. It looks like the backend is often calling RemoteFile::openSocket() while RemoteFile::path is either NULL or undefined.

I'm running with separate back and front ends, both on x86_64, and have gone down to -O0 in the compile flags to make debugging easier/possible :-)

[Mike Rice <mikerice1969@…>]

Replying to tim@electronghost.co.uk:

Replying to Mike Rice <mikerice1969@gmail.com>:

Running SVN trunk 16313. May just be memory corruption but hopefully someone can see something wrong from the trace. The full stack trace is attached.

I'm seeing an abort from glibc when channel hopping in LiveTV. It looks to me like a memory corruption somewhere. I added

We don't usually use LiveTV and there was no LiveTV going on during the crash I posted originally.

Today though one of my kids did use LiveTV on a slow front end. The SD tuners turned out to be busy and it was trying to use the HDHR tuner. I see a bunch of:

2008-03-03 17:44:37.690 write -> 22 290     BACKEND_MESSAGE[]:[]SIGNAL 3[]:[]Signal Lock[]:[]slock 1 1 0 1 0 ...
2008-03-03 17:44:37.691 MythSocket(ae241820:22): DownRef: 1
2008-03-03 17:44:37.691 MythSocket(ae23f6d0:25): UpRef: 2
2008-03-03 17:44:37.692 MythSocket(ae23f6d0:25): DownRef: 1
2008-03-03 17:44:37.720 MythEvent: SIGNAL 3
2008-03-03 17:44:37.721 MythSocket(b7c05650:12): UpRef: 2
2008-03-03 17:44:37.722 MythSocket(b7c05650:12): DownRef: 1
2008-03-03 17:44:37.722 MythSocket(a7c5cca0:23): UpRef: 2
2008-03-03 17:44:37.723 MythSocket(a7c5cca0:23): DownRef: 1
2008-03-03 17:44:37.724 MythSocket(a7e18978:24): UpRef: 2
2008-03-03 17:44:37.725 write -> 24 290     BACKEND_MESSAGE[]:[]SIGNAL 3[]:[]Signal Lock[]:[]slock 1 1 0 1 0 ...

Then a memory corruption crash.

======= Backtrace: =========
/lib/libc.so.6[0x3edf8ac]
/lib/libc.so.6(__libc_malloc+0x7b)[0x3ee0b7b]
/usr/lib/libstdc++.so.6(_Znwj+0x27)[0x32b3ba7]
/usr/lib/qt-3.3/lib/libqt-mt.so.3(_ZN7QString9setLengthEj+0x93)[0x6a3d573]
/usr/lib/qt-3.3/lib/libqt-mt.so.3[0x6a40464]
/usr/lib/qt-3.3/lib/libqt-mt.so.3(_ZNK7QString3argExii+0x112)[0x6a42ac2]
/usr/local/bin/mythbackend(_ZNK7QString3argEiii+0x3c)[0x80783dc]
/usr/local/lib/libmythtv-0.21.so.0(_ZNK18SignalMonitorValue9GetStatusEv+0x212)[0xf3e68c]
/usr/local/lib/libmythtv-0.21.so.0(_ZN16DTVSignalMonitor13GetStatusListEb+0x247)[0x12b28fb]
/usr/local/lib/libmythtv-0.21.so.0(_ZN13SignalMonitor11MonitorLoopEv+0x76)[0x12a9f26]
/usr/local/lib/libmythtv-0.21.so.0(_ZN13SignalMonitor16SpawnMonitorLoopEPv+0x18)[0x12a9b96]

I think there is another ticket more related to this one but I don't know the number.

[anonymous]

Replying to Mike Rice <mikerice1969@gmail.com>:

Today though one of my kids did use LiveTV on a slow front end. The SD tuners turned out to be busy and it was trying to use the HDHR tuner. I see a bunch of:

Yup, that's my crash alright :-) Frontend speed doesn't seem to matter (I have quite a fast one myself for testing; I use my desktop system)

2008-03-03 17:44:37.690 write -> 22 290     BACKEND_MESSAGE[]:[]SIGNAL 3[]:[]Signal Lock[]:[]slock 1 1 0 1 0 ...
2008-03-03 17:44:37.691 MythSocket(ae241820:22): DownRef: 1
2008-03-03 17:44:37.691 MythSocket(ae23f6d0:25): UpRef: 2
2008-03-03 17:44:37.692 MythSocket(ae23f6d0:25): DownRef: 1
2008-03-03 17:44:37.720 MythEvent: SIGNAL 3
2008-03-03 17:44:37.721 MythSocket(b7c05650:12): UpRef: 2
2008-03-03 17:44:37.722 MythSocket(b7c05650:12): DownRef: 1
2008-03-03 17:44:37.722 MythSocket(a7c5cca0:23): UpRef: 2
2008-03-03 17:44:37.723 MythSocket(a7c5cca0:23): DownRef: 1
2008-03-03 17:44:37.724 MythSocket(a7e18978:24): UpRef: 2
2008-03-03 17:44:37.725 write -> 24 290     BACKEND_MESSAGE[]:[]SIGNAL 3[]:[]Signal Lock[]:[]slock 1 1 0 1 0 ...

Then a memory corruption crash.

======= Backtrace: =========
/lib/libc.so.6[0x3edf8ac]
/lib/libc.so.6(__libc_malloc+0x7b)[0x3ee0b7b]
/usr/lib/libstdc++.so.6(_Znwj+0x27)[0x32b3ba7]
/usr/lib/qt-3.3/lib/libqt-mt.so.3(_ZN7QString9setLengthEj+0x93)[0x6a3d573]
/usr/lib/qt-3.3/lib/libqt-mt.so.3[0x6a40464]
/usr/lib/qt-3.3/lib/libqt-mt.so.3(_ZNK7QString3argExii+0x112)[0x6a42ac2]
/usr/local/bin/mythbackend(_ZNK7QString3argEiii+0x3c)[0x80783dc]
/usr/local/lib/libmythtv-0.21.so.0(_ZNK18SignalMonitorValue9GetStatusEv+0x212)[0xf3e68c]
/usr/local/lib/libmythtv-0.21.so.0(_ZN16DTVSignalMonitor13GetStatusListEb+0x247)[0x12b28fb]
/usr/local/lib/libmythtv-0.21.so.0(_ZN13SignalMonitor11MonitorLoopEv+0x76)[0x12a9f26]
/usr/local/lib/libmythtv-0.21.so.0(_ZN13SignalMonitor16SpawnMonitorLoopEPv+0x18)[0x12a9b96]

I think there is another ticket more related to this one but I don't know the number.

I looked and couldn't find it, so have opened #4854

[Marc Randolph <mrandtx@…>]

#4855 (closed as duplicate of this one) has an additional gdb trace and backend log.

[cg@…]

I just upgraded to 16384, to see if the changes reverted by danielk do any help. Unfortunately I still see the crash, immediately when I enter LiveTV. Let me know if you need another backtrace

[anonymous]

cg, submit a new ticket with a backtrace.

[danielk]

valgrind is the only tool that will help us here. The backtrace does not tell us what caused the memory corruption.