Re: [issue1504] hammer crash on cleanups

[Dennis Melentyev <dennis.melentyev@xxxxxxxxx>]

Hi!

I had something very similar on 95% full hammer FS with version of FS = 1.
After upgrading it to v2 panics vent away.

dennis@dfly (xterm) > uname -srip
DragonFly 2.3.1-DEVELOPMENT i386 GENERIC
As of 05/Jul/2009

Like:
# hammer version-upgrade /mnt/big_plate 2

Also here are some interesting lines in /var/log/messages, possibly
indicating corrupted media:

Sep 16 01:22:42 dfly kernel: ad4: WARNING - WRITE_DMA UDMA ICRC error
(retrying request) LBA=92046464
Sep 16 01:26:18 dfly syslogd: kernel boot file is /boot/kernel
Sep 16 01:26:18 dfly kernel: ad4: FAILURE - device detached
Sep 16 01:26:18 dfly kernel: subdisk4: detached
Sep 16 01:26:18 dfly kernel: HAMMER(Share_Big): Critical error
inode=-1 while flushing meta-data
Sep 16 01:26:18 dfly kernel: HAMMER(Share_Big): Forcing read-only mode
Sep 16 01:26:18 dfly kernel: HAMMER(Share_Big): Critical error
inode=-1 while flushing meta-data
Sep 16 01:26:18 dfly last message repeated 6 times
Sep 16 01:26:18 dfly kernel: ad4: detached
. .....
Sep 16 01:26:18 dfly kernel: Fatal trap 12: page fault while in kernel mode
Sep 16 01:26:18 dfly kernel: fault virtual address      = 0xa4
Sep 16 01:26:18 dfly kernel: fault code         = supervisor write,
page not present
. .....
Sep 16 01:26:18 dfly kernel: processor eflags   = interrupt enabled,
resume, IOPL = 0
Sep 16 01:26:18 dfly kernel: current process            = Idle
Sep 16 01:26:18 dfly kernel: current thread          = pri 46 (CRIT)
. .....
Sep 16 01:26:18 dfly kernel:
Sep 16 01:26:18 dfly kernel: syncing disks... 439 HAMMER(Share_Big):
Critical error inode=-1 while flushing meta-data

2009/9/16 Eugene <dfuser@medcom.com.ua>:
> Hello again.
>
> Simon 'corecode' Schubert wrote:
>>
>> Best update to latest master, there were many fixes since August.
>
> Now I've an updated kernel with sources from 13-sep-2009:
> DragonFly diana.medcom.com.ua 2.3.2-DEVELOPMENT DragonFly 2.3.2-DEVELOPMENT
> #1: Tue Sep 15 17:23:22 EEST 2009
> root@diana.medcom.com.ua:/usr/obj/usr/src/sys/CUSTOM  i386
>
> but the "panic" message greeted me this morning:
>
> panic: assertion: cursor->trans->sync_lock_refs > 0 in hammer_recover_cursor
> Trace beginning at frame 0xd75ca708
> panic(d75ca72c,d75ca7a8,d75ca88c,d75ca804,d75ca738) at panic+0x8c
> panic(c05ac924,c0617e20,c059441c,d75ca7a8,d75ca99c) at panic+0x8c
> hammer_recover_cursor(d75ca7a8,b,8,d75ca768,c04acd05) at
> hammer_recover_cursor+0x2c
> hammer_ioc_mirror_write(d75caa74,d2d28550,c2a727f8,d75ca9d8,c0394328) at
> hammer_ioc_mirror_write+0x947
> hammer_ioctl(d2d28550,c0c46808,c2a727f8,1,c29a7768) at hammer_ioctl+0x8f8
> hammer_vop_ioctl(d75caad0,c067a1e0,d4ae7050,d75caaec,c0345eaa) at
> hammer_vop_ioctl+0x2f
> vop_ioctl(d4ae7050,c2b145e8,c0c46808,c2a727f8,1) at vop_ioctl+0x3e
> vn_ioctl(d0291928,c0c46808,c2a727f8,c29a7768,d75cacf0) at vn_ioctl+0xe0
> mapped_ioctl(3,c0c46808,bfbffa14,0,d75cacf0) at mapped_ioctl+0x3e7
> sys_ioctl(d75cacf0,6,1b88a2c3,0,d2aed218) at sys_ioctl+0x17
> syscall2(d75cad40) at syscall2+0x1ef
> Xint0x80_syscall() at Xint0x80_syscall+0x36
> Debugger("panic")
>
> a partly backtrace is:
>
> #11 0xc04b0f8c in hammer_recover_cursor (cursor=0xd75ca7a8)
>   at /usr/src/sys/vfs/hammer/hammer_cursor.c:591
> 591             KKASSERT(cursor->trans->sync_lock_refs > 0);
> (kgdb) list
> 586     hammer_recover_cursor(hammer_cursor_t cursor)
> 587     {
> 588             int error;
> 589
> 590             hammer_unlock_cursor(cursor);
> 591             KKASSERT(cursor->trans->sync_lock_refs > 0);
> 592
> 593             /*
> 594              * Wait for the deadlock to clear
> 595              */
>
> (kgdb) print cursor
> $1 = (hammer_cursor_t) 0xd75ca7a8
> (kgdb) print cursor->trans
> $2 = (hammer_transaction_t) 0xd75caa74
> (kgdb) print cursor->trans->sync_lock_refs
> $3 = 0
>
> #12 0xc04ba517 in hammer_ioc_mirror_write (trans=0xd75caa74, ip=0xd2d28550,
>   mirror=0xc2a727f8) at /usr/src/sys/vfs/hammer/hammer_mirror.c:469
> 469                                     hammer_recover_cursor(&cursor);
> (kgdb) list
> 464                      * Retry the current record on deadlock, otherwise
> setup
> 465                      * for the next loop.
> 466                      */
> 467                     if (error == EDEADLK) {
> 468                             while (error == EDEADLK) {
> 469                                     hammer_recover_cursor(&cursor);
> 470                                     error =
> hammer_cursor_upgrade(&cursor);
> 471                             }
> 472                     } else {
> 473                             if (error == EALREADY)
>
> #13 0xc04b97dc in hammer_ioctl (ip=0xd2d28550, com=3234097160,
>   data=0xc2a727f8 "", fflag=1, cred=0xc29a7768)
>   at /usr/src/sys/vfs/hammer/hammer_ioctl.c:134
> 134                             error = hammer_ioc_mirror_write(&trans, ip,
> (kgdb) list
> 129                                         (struct hammer_ioc_mirror_rw
> *)data);
> 130                     }
> 131                     break;
> 132             case HAMMERIOC_MIRROR_WRITE:
> 133                     if (error == 0) {
> 134                             error = hammer_ioc_mirror_write(&trans, ip,
> 135                                         (struct hammer_ioc_mirror_rw
> *)data);
> 136                     }
> 137                     break;
> 138             case HAMMERIOC_GET_VERSION:
>
> #14 0xc04c94de in hammer_vop_ioctl (ap=0xd75caad0)
>   at /usr/src/sys/vfs/hammer/hammer_vnops.c:2305
> 2305            return(hammer_ioctl(ip, ap->a_command, ap->a_data,
> (kgdb) list
> 2300    hammer_vop_ioctl(struct vop_ioctl_args *ap)
> 2301    {
> 2302            struct hammer_inode *ip = ap->a_vp->v_data;
> 2303
> 2304            ++hammer_stats_file_iopsr;
> 2305            return(hammer_ioctl(ip, ap->a_command, ap->a_data,
> 2306                                ap->a_fflag, ap->a_cred));
> 2307    }
> 2308
>
> #15 0xc03940ae in vop_ioctl (ops=0xd4ae7050, vp=0xc2b145e8,
>   command=3234097160, data=0xc2a727f8 "", fflag=1, cred=0xc29a7768,
>   msg=0xd75cacf0) at /usr/src/sys/kern/vfs_vopops.c:376
> 376             DO_OPS(ops, error, &ap, vop_ioctl);
> (kgdb) list
> 371             ap.a_data = data;
> 372             ap.a_fflag = fflag;
> 373             ap.a_cred = cred;
> 374             ap.a_sysmsg = msg;
> 375
> 376             DO_OPS(ops, error, &ap, vop_ioctl);
> 377             return(error);
> 378     }
> 379
>
> #16 0xc0393bed in vn_ioctl (fp=0xd0291928, com=3234097160, data=0xc2a727f8
> "",
>   ucred=0xc29a7768, msg=0xd75cacf0) at /usr/src/sys/kern/vfs_vnops.c:938
> 938                     error = VOP_IOCTL(vp, com, data, fp->f_flag, ucred,
> msg);
> (kgdb) list
> 933                             }
> 934                             *(int *)data = dev_dflags(vp->v_rdev) &
> D_TYPEMASK;
> 935                             error = 0;
> 936                             break;
> 937                     }
> 938                     error = VOP_IOCTL(vp, com, data, fp->f_flag, ucred,
> msg);
> 939                     if (error == 0 && com == TIOCSCTTY) {
> 940                             struct proc *p = curthread->td_proc;
> 941                             struct session *sess;
>
> --
> Sorry for my poor English.
>



-- 
Dennis Melentyev