Invalid read followed by crash at dct36

[zlai88@…]

The fussed file t811_full.mp3 (generated by Zzuf) caused Mplayer to crash by bad usage of CPU/FPU/RAM. Valgrind reports invalid read of size 4 followed by crash at dct36.

This is reproducible on Linux Debian Etch, with the latest Subversion head mplayer (r27133). The machine used is VMWare Player.

Reproduce as follows:
wget ​http://www.eecs.berkeley.edu/~zhl210/t811_full.mp3
Valgrind mplayer t811_full.mp3

Here is the report by Valgrind:

==16360== Invalid read of size 4
==16360== Stack hash: 3479153806
==16360== at 0x81E03DB: dct36 (dct36.c:169)
==16360== by 0x81E492D: do_layer3 (layer3.c:1212)
==16360== by 0x81E6015: MP3_DecodeFrame (sr1.c:539)
==16360== by 0x80D83B4: decode_audio (dec_audio.c:383)
==16360== by 0x8075D39: main (mplayer.c:2044)
==16360== Address 0x40bb3aac is not stack'd, malloc'd or (recently) free'd

==16360==
==16360== ERROR SUMMARY: 361 errors from 5 contexts (suppressed: 17 from 1)
==16360== malloc/free: in use at exit: 202,405 bytes in 2,205 blocks.
==16360== malloc/free: 15,620 allocs, 13,415 frees, 6,689,446 bytes allocated.
==16360== For counts of detected errors, rerun with: -v
==16360== searching for pointers to 2,205 not-freed blocks.
==16360== checked 2,919,872 bytes.
==16360==
==16360== LEAK SUMMARY:
==16360== definitely lost: 0 bytes in 0 blocks.
==16360== possibly lost: 0 bytes in 0 blocks.
==16360== still reachable: 202,405 bytes in 2,205 blocks.
==16360== suppressed: 0 bytes in 0 blocks.
==16360== Rerun with --leak-check=full to see details of leaked memory.

Here is the backtrace using gdb:

Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread -1209591584 (LWP 16035)]
0x081e03db in dct36 (inbuf=0xbfffc290, o1=0x8705500, o2=0x8704300, wintab=0x40bb3a40,

tsbuf=0xbfffb090) at mp3lib/dct36.c:169

169 MACRO(0);
(gdb) bt
#0 0x081e03db in dct36 (inbuf=0xbfffc290, o1=0x8705500, o2=0x8704300, wintab=0x40bb3a40,

tsbuf=0xbfffb090) at mp3lib/dct36.c:169

#1 0x081e492e in do_layer3 (fr=0x8708640, single=-1) at mp3lib/layer3.c:1212
#2 0x081e6016 in MP3_DecodeFrame (

hova=0x8979512 "&#65533;&#65533;&#65533;1&#65533;&#65533;\236<g&#65533;*E&#65533;&#65533;<M3\005\211T&#65533;\n\177Z\v\v&#65533;
\210\t\021b&\a\004k&#65533;\a&#65533;o\026\b", single=-1) at mp3lib/sr1.c:539

#3 0x080d83b5 in decode_audio (sh_audio=0x8977ea0, minlen=8192)

at libmpcodecs/dec_audio.c:383

#4 0x08075d3a in main (argc=3, argv=0xbffff754) at mplayer.c:2044

Dump of assembler code from 0x81e03bb to 0x81e03fb:
0x081e03bb <dct36+891>: cmp $0xb4,%al
0x081e03bd <dct36+893>: xchg %eax,%ebp
0x081e03be <dct36+894>: or %bl,%cl
0x081e03c0 <dct36+896>: inc %ebp
0x081e03c1 <dct36+897>: test $0xd8,%al
0x081e03c3 <dct36+899>: les (bad),%ebx
0x081e03c4 <dct36+900>: fmuls 0x895b428
0x081e03ca <dct36+906>: fxch %st(4)
0x081e03cc <dct36+908>: fsubrs 0xffffffa8(%ebp)
0x081e03cf <dct36+911>: fmuls 0x895b438
0x081e03d5 <dct36+917>: flds 0xffffffe0(%ebp)
0x081e03d8 <dct36+920>: fadds 0xffffffbc(%ebp)
0x081e03db <dct36+923>: flds 0x6c(%edx)
0x081e03de <dct36+926>: fmul %st(1),%st
0x081e03e0 <dct36+928>: fstps 0x24(%ecx)
0x081e03e3 <dct36+931>: fmuls 0x68(%edx)
0x081e03e6 <dct36+934>: fstps 0x20(%ecx)
0x081e03e9 <dct36+937>: flds 0xffffffe0(%ebp)
0x081e03ec <dct36+940>: fsubs 0xffffffbc(%ebp)
0x081e03ef <dct36+943>: flds 0x20(%edx)
0x081e03f2 <dct36+946>: fmul %st(1),%st
0x081e03f4 <dct36+948>: fadds 0x20(%esi)
0x081e03f7 <dct36+951>: fstps 0x400(%ebx)
End of assembler dump.

eax 0xbfffc290 -1073757552
ecx 0x8704300 141574912
edx 0x40bb3a40 1086011968
ebx 0xbfffb090 -1073762160
esp 0xbfffaedc 0xbfffaedc
ebp 0xbfffaf78 0xbfffaf78
esi 0x8705500 141579520
edi 0x8704300 141574912
eip 0x81e03db 0x81e03db <dct36+923>
eflags 0x210292 [ AF SF IF RF ID ]
cs 0x73 115
ss 0x7b 123
ds 0x7b 123
es 0x7b 123
fs 0x0 0
gs 0x33 51
st0 -2.9774419374462013365700840950012207e-08 (raw 0xbfe5ffc2a40000000000)
st1 -7.378469011685906119585818392948986e-07 (raw 0xbfeac61073f528ca7f69)
st2 -5.7552754825868197459111212654914169e-07 (raw 0xbfea9a7df3b3e116e000)
st3 -1.2546106206212573822695015984818312e-07 (raw 0xbfe886b6796168f4c000)
st4 -9.195324863394668092787713539099407e-07 (raw 0xbfeaf6d5ca92d55f7f5a)
st5 -4.0034542605586182116658329009335429e-07 (raw 0xbfe9d6ef0e6f359639d7)
st6 -nan(0x47214ff147214ff1) (raw 0xffff47214ff147214ff1)
st7 0.866025388240814208984375 (raw 0x3ffeddb3d70000000000)
fctrl 0x37f 895
fstat 0x5021 20513
ftag 0xf 15
fiseg 0x73 115
fioff 0x81e03d8 136184792
foseg 0x7b 123
fooff 0xbfffaf34 -1073762508
fop 0x45 69
xmm0 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {

0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, uint128 = 0x00000000000000000000000000000000}

xmm1 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {

0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, uint128 = 0x00000000000000000000000000000000}

xmm2 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {

0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, uint128 = 0x00000000000000000000000000000000}

xmm3 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {

0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, uint128 = 0x00000000000000000000000000000000}

xmm4 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {

0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, uint128 = 0x00000000000000000000000000000000}

xmm5 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {

0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, uint128 = 0x00000000000000000000000000000000}

xmm6 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {

0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, uint128 = 0x00000000000000000000000000000000}

xmm7 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0}, v16_int8 = {

0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {
0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0}, uint128 = 0x00000000000000000000000000000000}

mxcsr 0x1f80 [ IM DM ZM OM UM PM ]
mm0 {uint64 = 0x9a7df3b3e116e000, v2_int32 = {0xe116e000, 0x9a7df3b3}, v4_int16 = {

0xe000, 0xe116, 0xf3b3, 0x9a7d}, v8_int8 = {0x0, 0xe0, 0x16, 0xe1, 0xb3, 0xf3, 0x7d,
0x9a}}

mm1 {uint64 = 0x86b6796168f4c000, v2_int32 = {0x68f4c000, 0x86b67961}, v4_int16 = {

0xc000, 0x68f4, 0x7961, 0x86b6}, v8_int8 = {0x0, 0xc0, 0xf4, 0x68, 0x61, 0x79, 0xb6,
0x86}}

mm2 {uint64 = 0xf6d5ca92d55f7f5a, v2_int32 = {0xd55f7f5a, 0xf6d5ca92}, v4_int16 = {

0x7f5a, 0xd55f, 0xca92, 0xf6d5}, v8_int8 = {0x5a, 0x7f, 0x5f, 0xd5, 0x92, 0xca, 0xd5,
0xf6}}

mm3 {uint64 = 0xd6ef0e6f359639d7, v2_int32 = {0x359639d7, 0xd6ef0e6f}, v4_int16 = {

0x39d7, 0x3596, 0xe6f, 0xd6ef}, v8_int8 = {0xd7, 0x39, 0x96, 0x35, 0x6f, 0xe, 0xef, 0xd6}}

mm4 {uint64 = 0x47214ff147214ff1, v2_int32 = {0x47214ff1, 0x47214ff1}, v4_int16 = {

0x4ff1, 0x4721, 0x4ff1, 0x4721}, v8_int8 = {0xf1, 0x4f, 0x21, 0x47, 0xf1, 0x4f, 0x21,
0x47}}

mm5 {uint64 = 0xddb3d70000000000, v2_int32 = {0x0, 0xddb3d700}, v4_int16 = {0x0,

0x0, 0xd700, 0xddb3}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0xd7, 0xb3, 0xdd}}

mm6 {uint64 = 0xffc2a40000000000, v2_int32 = {0x0, 0xffc2a400}, v4_int16 = {0x0,

0x0, 0xa400, 0xffc2}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0xc2, 0xff}}

mm7 {uint64 = 0xc61073f528ca7f69, v2_int32 = {0x28ca7f69, 0xc61073f5}, v4_int16 = {

0x7f69, 0x28ca, 0x73f5, 0xc610}, v8_int8 = {0x69, 0x7f, 0xca, 0x28, 0xf5, 0x73, 0x10,
0xc6}}

This bug was found as part of the SUPERB-TRUST 2008 project.

[zlai88@…]

In the second paragraph of my description, I have mistakenly reported "r27133" as the version for Mplayer. The correct version number should be r27138.