www.fabiankeil.de/bilder/freebsd/kernel-panic-r275748-zfs/

FreeBSD (r275748 with custom patches) panic

Dump header from device /dev/ada0s1b
Architecture: amd64
Architecture Version: 2
Dump Length: 370741248B (353 MB)
Blocksize: 512
Dumptime: Thu Dec 25 22:44:44 2014
Hostname: r500.local
Magic: FreeBSD Kernel Dump
Version String: FreeBSD 11.0-CURRENT #673 r275748+dcbb16a(fk): Sat Dec 13 21:33:19 CET 2014
  fk@r500.local:/usr/obj/usr/src/sys/ZOEY
Panic String: solaris assert: zap_update(scn->scn_dp->dp_meta_objset, 1, "scan", sizeof (uint64_t), (sizeof (dsl_scan_phys_t) / sizeof (uint64_t)), &scn->scn_phys, tx) == 0 (0x6 == 0x0), file: /usr/src/sys/EÇ FreeBSD Kernel Dump
Dump Parity: 549952069
Bounds: 6
Dump Status: good

[Foto: 7109-solaris-assert-zap_update-return-code-6]

Apparently spa->spa_state is POOL_STATE_ACTIVE after the pool has gone:

(kgdb) where
#0  doadump (textdump=0) at pcpu.h:219
#1  0xffffffff80313cae in db_dump (dummy=<value optimized out>, dummy2=0, dummy3=0, dummy4=0x0) at /usr/src/sys/ddb/db_command.c:533
#2  0xffffffff8031378d in db_command (cmd_table=0x0) at /usr/src/sys/ddb/db_command.c:440
#3  0xffffffff80313504 in db_command_loop () at /usr/src/sys/ddb/db_command.c:493
#4  0xffffffff80316010 in db_trap (type=<value optimized out>, code=0) at /usr/src/sys/ddb/db_main.c:251
#5  0xffffffff805f46d1 in kdb_trap (type=3, code=0, tf=<value optimized out>) at /usr/src/sys/kern/subr_kdb.c:654
#6  0xffffffff80876c57 in trap (frame=0xfffffe0095157500) at /usr/src/sys/amd64/amd64/trap.c:542
#7  0xffffffff8085a9b2 in calltrap () at /usr/src/sys/amd64/amd64/exception.S:231
#8  0xffffffff805f3dce in kdb_enter (why=0xffffffff80993d2d "panic", msg=<value optimized out>) at cpufunc.h:63
#9  0xffffffff805b1e11 in panic (fmt=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:739
#10 0xffffffff81bd922f in assfail3 (a=<value optimized out>, lv=<value optimized out>, op=<value optimized out>, rv=<value optimized out>, f=<value optimized out>, l=<value optimized out>)
    at /usr/src/sys/cddl/compat/opensolaris/kern/opensolaris_cmn_err.c:91
#11 0xffffffff81940313 in dsl_scan_sync (dp=<value optimized out>, tx=<value optimized out>) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c:411
#12 0xffffffff819573a1 in spa_sync (spa=0xfffff8005114a000, txg=5496) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6644
#13 0xffffffff81960766 in txg_sync_thread (arg=0xfffff80006697800) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:517
#14 0xffffffff80577afa in fork_exit (callout=0xffffffff819603c0 <txg_sync_thread>, arg=0xfffff80006697800, frame=0xfffffe0095157c00) at /usr/src/sys/kern/kern_fork.c:978
#15 0xffffffff8085aeee in fork_trampoline () at /usr/src/sys/amd64/amd64/exception.S:605
#16 0x0000000000000000 in ?? ()
(kgdb) f 12
#12 0xffffffff819573a1 in spa_sync (spa=0xfffff8005114a000, txg=5496) at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6644
6644				    spa->spa_uberblock.ub_rootbp.blk_birth);
(kgdb) p *spa
$1 = {spa_name = "extreme", '\0' <repeats 248 times>, spa_comment = 0x0, spa_avl = {avl_child = {0x0, 0x0}, avl_pcb = 18446735277722042633}, spa_config = 0xfffff80040f6c1c0, spa_config_syncing = 0x0, 
  spa_config_splitting = 0x0, spa_load_info = 0xfffff800713dfbe0, spa_config_txg = 5480, spa_sync_pass = 1, spa_state = POOL_STATE_ACTIVE, spa_inject_ref = 0, spa_sync_on = 1 '\001', 
  spa_load_state = SPA_LOAD_NONE, spa_import_flags = 0, spa_zio_taskq = {{{stqs_count = 1, stqs_taskq = 0xfffff8006496b770}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 1, 
        stqs_taskq = 0xfffff800365efd40}, {stqs_count = 0, stqs_taskq = 0x0}}, {{stqs_count = 1, stqs_taskq = 0xfffff8006e0e6050}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 8, 
        stqs_taskq = 0xfffff80014868e80}, {stqs_count = 0, stqs_taskq = 0x0}}, {{stqs_count = 1, stqs_taskq = 0xfffff8005135a7b0}, {stqs_count = 1, stqs_taskq = 0xfffff8005135a800}, {stqs_count = 1, 
        stqs_taskq = 0xfffff8005135a6e0}, {stqs_count = 1, stqs_taskq = 0xfffff80051eca7c0}}, {{stqs_count = 8, stqs_taskq = 0xfffff800111ff7c0}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 1, 
        stqs_taskq = 0xfffff8006496b7f0}, {stqs_count = 0, stqs_taskq = 0x0}}, {{stqs_count = 1, stqs_taskq = 0xfffff8005197d7a0}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 1, 
        stqs_taskq = 0xfffff8006e0e6760}, {stqs_count = 0, stqs_taskq = 0x0}}, {{stqs_count = 1, stqs_taskq = 0xfffff8005dd1d1a0}, {stqs_count = 0, stqs_taskq = 0x0}, {stqs_count = 1, 
        stqs_taskq = 0xfffff8001dd80d40}, {stqs_count = 0, stqs_taskq = 0x0}}}, spa_dsl_pool = 0xfffff80006697800, spa_is_initializing = 0, spa_normal_class = 0xfffff80036499c00, 
  spa_log_class = 0xfffff8005d14a000, spa_first_txg = 5416, spa_final_txg = 18446744073709551615, spa_freeze_txg = 18446744073709551615, spa_load_max_txg = 18446744073709551615, spa_claim_max_txg = 5416, 
  spa_loaded_ts = {tv_sec = 1419543190, tv_nsec = 985278826}, spa_meta_objset = 0xfffff80043374800, spa_vdev_txg_list = {tl_lock = {lock_object = {lo_name = 0xffffffff81a16183 "tl->tl_lock", 
        lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, tl_offset = 624, tl_head = {0xfffff800061a5a70, 0x0, 0x0, 0x0}}, spa_root_vdev = 0xfffff800028d4000, 
  spa_config_guid = 17131816209720905815, spa_load_guid = 1565021790, spa_last_synced_guid = 17131816209720905815, spa_config_dirty_list = {list_size = 1624, list_offset = 672, list_head = {
      list_next = 0xfffff8005114a3c0, list_prev = 0xfffff8005114a3c0}}, spa_state_dirty_list = {list_size = 1624, list_offset = 688, list_head = {list_next = 0xfffff800061a5ab0, 
      list_prev = 0xfffff800061a5ab0}}, spa_spares = {sav_object = 0, sav_config = 0x0, sav_vdevs = 0x0, sav_count = 0, sav_sync = 0, sav_pending = 0x0, sav_npending = 0}, spa_l2cache = {sav_object = 0, 
    sav_config = 0x0, sav_vdevs = 0x0, sav_count = 0, sav_sync = 0, sav_pending = 0x0, sav_npending = 0}, spa_label_features = 0xfffff8005dcba2c0, spa_config_object = 27, spa_config_generation = 0, 
  spa_syncing_txg = 5496, spa_deferred_bpobj = {bpo_lock = {lock_object = {lo_name = 0xffffffff81a091f9 "bpo->bpo_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, 
    bpo_os = 0xfffff80043374800, bpo_object = 31, bpo_epb = 128, bpo_havecomp = 1 '\001', bpo_havesubobj = 1 '\001', bpo_phys = 0xfffff800518b0200, bpo_dbuf = 0xfffff8005148de80, 
    bpo_cached_dbuf = 0xfffff800141d8000}, spa_free_bplist = {{bpl_lock = {lock_object = {lo_name = 0xffffffff81a08f69 "bpl->bpl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, 
      bpl_list = {list_size = 144, list_offset = 128, list_head = {list_next = 0xfffff8005114a4f0, list_prev = 0xfffff8005114a4f0}}}, {bpl_lock = {lock_object = {lo_name = 0xffffffff81a08f69 "bpl->bpl_lock", 
          lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, bpl_list = {list_size = 144, list_offset = 128, list_head = {list_next = 0xfffff8005114a530, list_prev = 0xfffff8005114a530}}}, {
      bpl_lock = {lock_object = {lo_name = 0xffffffff81a08f69 "bpl->bpl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, bpl_list = {list_size = 144, list_offset = 128, list_head = {
          list_next = 0xfffff8005114a570, list_prev = 0xfffff8005114a570}}}, {bpl_lock = {lock_object = {lo_name = 0xffffffff81a08f69 "bpl->bpl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, 
        sx_lock = 1}, bpl_list = {list_size = 144, list_offset = 128, list_head = {list_next = 0xfffff8005114a5b0, list_prev = 0xfffff8005114a5b0}}}}, spa_ubsync = {ub_magic = 12235020, ub_version = 5000, 
    ub_txg = 5495, ub_guid_sum = 569402782283225051, ub_timestamp = 1419543868, ub_rootbp = {blk_dva = {{dva_word = {1, 5314489}}, {dva_word = {1, 3617574}}, {dva_word = {1, 1785983}}}, 
      blk_prop = 9226476022604496899, blk_pad = {0, 0}, blk_phys_birth = 0, blk_birth = 5495, blk_fill = 281, blk_cksum = {zc_word = {38259425213, 4030330449781, 215699799542786, 7821546357561732}}}, 
    ub_software_version = 5000}, spa_uberblock = {ub_magic = 12235020, ub_version = 5000, ub_txg = 5495, ub_guid_sum = 569402782283225051, ub_timestamp = 1419543868, ub_rootbp = {blk_dva = {{dva_word = {1, 
            5314528}}, {dva_word = {1, 3617613}}, {dva_word = {1, 1786006}}}, blk_prop = 9226476022604496899, blk_pad = {0, 0}, blk_phys_birth = 0, blk_birth = 5496, blk_fill = 281, blk_cksum = {zc_word = {
          39112394345, 4117668979717, 220219642409083, 7979154812190673}}}, ub_software_version = 5000}, spa_extreme_rewind = 0, spa_last_io = 6494674, spa_scrub_lock = {lock_object = {
      lo_name = 0xffffffff81a157e8 "spa->spa_scrub_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_scrub_inflight = 0, spa_scrub_io_cv = {
    cv_description = 0xffffffff81a15851 "spa->spa_scrub_io_cv", cv_waiters = 0}, spa_scrub_active = 1 '\001', spa_scrub_type = 0 '\0', spa_scrub_finished = 0 '\0', spa_scrub_started = 1 '\001', 
  spa_scrub_reopen = 0 '\0', spa_scan_pass_start = 1419543713, spa_scan_pass_exam = 508732928, spa_async_lock = {lock_object = {lo_name = 0xffffffff81a15766 "spa->spa_async_lock", lo_flags = 40960000, 
      lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_async_thread = 0x0, spa_async_thread_vd = 0x0, spa_async_suspended = 0, spa_async_cv = {cv_description = 0xffffffff81a1582c "spa->spa_async_cv", 
    cv_waiters = 0}, spa_async_tasks = 4, spa_root = 0x0, spa_ena = 0, spa_last_open_failed = 0, spa_last_ubsync_txg = 0, spa_last_ubsync_txg_ts = 0, spa_load_txg = 0, spa_load_txg_ts = 1414959257, 
  spa_load_meta_errors = 0, spa_load_data_errors = 0, spa_verify_min_txg = 5412, spa_errlog_lock = {lock_object = {lo_name = 0xffffffff81a15792 "spa->spa_errlog_lock", lo_flags = 40960000, lo_data = 0, 
      lo_witness = 0x0}, sx_lock = 1}, spa_errlog_last = 0, spa_errlog_scrub = 0, spa_errlist_lock = {lock_object = {lo_name = 0xffffffff81a1577b "spa->spa_errlist_lock", lo_flags = 40960000, lo_data = 0, 
      lo_witness = 0x0}, sx_lock = 1}, spa_errlist_last = {avl_root = 0x0, avl_compar = 0xffffffff8194e4e0 <spa_error_entry_compare>, avl_offset = 40, avl_numnodes = 0, avl_size = 64}, spa_errlist_scrub = {
    avl_root = 0xfffff8004f2a8028, avl_compar = 0xffffffff8194e4e0 <spa_error_entry_compare>, avl_offset = 40, avl_numnodes = 42, avl_size = 64}, spa_deflate = 1, spa_history = 32, spa_history_lock = {
    lock_object = {lo_name = 0xffffffff81a157a8 "spa->spa_history_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_pending_vdev = 0x0, spa_props_lock = {lock_object = {
      lo_name = 0xffffffff81a157d3 "spa->spa_props_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_pool_props_object = 33, spa_bootfs = 0, spa_failmode = 1, spa_delegation = 1, 
  spa_config_list = {list_size = 24, list_offset = 0, list_head = {list_next = 0xfffff80040f6d400, list_prev = 0xfffff80040f6d400}}, spa_async_zio_root = 0xfffff80051ecae60, spa_suspend_zio_root = 0x0, 
  spa_suspend_lock = {lock_object = {lo_name = 0xffffffff81a157fd "spa->spa_suspend_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_suspend_cv = {
    cv_description = 0xffffffff81a15867 "spa->spa_suspend_cv", cv_waiters = 0}, spa_suspended = 0 '\0', spa_claiming = 0 '\0', spa_debug = 0, spa_is_root = 0, spa_minref = 11, spa_mode = 3, 
  spa_log_state = SPA_LOG_GOOD, spa_autoexpand = 0, spa_ddt = {0xfffffe001f2b8000, 0xfffffe00342ed000, 0xfffffe002cc88000, 0xfffffe004daee000, 0xfffffe0022a6c000, 0xfffffe0007aa6000, 0xfffffe0025142000, 
    0xfffffe0015d07000, 0xfffffe001210b000, 0xfffffe001f1b2000, 0xfffffe000b4f2000}, spa_ddt_stat_object = 0, spa_dedup_ditto = 0, spa_dedup_checksum = 8, spa_dspace = 3992977408, spa_vdev_top_lock = {
    lock_object = {lo_name = 0xffffffff81a15814 "spa->spa_vdev_top_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_proc_lock = {lock_object = {
      lo_name = 0xffffffff81a157bf "spa->spa_proc_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, spa_proc_cv = {cv_description = 0xffffffff81a1583f "spa->spa_proc_cv", 
    cv_waiters = 0}, spa_proc_state = SPA_PROC_NONE, spa_proc = 0xffffffff816673e8, spa_did = 0, spa_trim_thread = 0x0, spa_trim_lock = {lock_object = {lo_name = 0x0, lo_flags = 0, lo_data = 0, 
      lo_witness = 0x0}, sx_lock = 0}, spa_trim_cv = {cv_description = 0x0, cv_waiters = 0}, spa_autoreplace = 1, spa_vdev_locks = 22, spa_creation_version = 5000, spa_prev_software_version = 5000, 
  spa_feat_for_write_obj = 29, spa_feat_for_read_obj = 28, spa_feat_desc_obj = 30, spa_feat_enabled_txg_obj = 203, spa_feat_refcount_cache = {0, 57, 1, 0, 84, 7, 1, 0, 1, 0, 0, 0}, spa_deadman_cycid = {
    c_links = {le = {le_next = 0x0, le_prev = 0xfffffe0000b77b68}, sle = {sle_next = 0x0}, tqe = {tqe_next = 0x0, tqe_prev = 0xfffffe0000b77b68}}, c_time = 32184028184784, c_precision = 268435437500, 
    c_arg = 0xfffff8005114a000, c_func = 0xffffffff8195bdd0 <spa_deadman>, c_lock = 0x0, c_flags = 22, c_cpu = 0}, spa_deadman_calls = 0, spa_sync_starttime = 6493428075431, 
  spa_deadman_synctime = 1000000000000, spa_ccw_fail_time = 0, spa_config_lock = {{scl_lock = {lock_object = {lo_name = 0xffffffff81a15b11 "scl->scl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, 
        sx_lock = 1}, scl_writer = 0x0, scl_write_wanted = 0, scl_cv = {cv_description = 0xffffffff81a15b20 "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 1}}, {scl_lock = {lock_object = {
          lo_name = 0xffffffff81a15b11 "scl->scl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0x0, scl_write_wanted = 0, scl_cv = {
        cv_description = 0xffffffff81a15b20 "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 0}}, {scl_lock = {lock_object = {lo_name = 0xffffffff81a15b11 "scl->scl_lock", lo_flags = 40960000, 
          lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0x0, scl_write_wanted = 0, scl_cv = {cv_description = 0xffffffff81a15b20 "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 0}}, {
      scl_lock = {lock_object = {lo_name = 0xffffffff81a15b11 "scl->scl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0x0, scl_write_wanted = 0, scl_cv = {
---Type <return> to continue, or q <return> to quit---
        cv_description = 0xffffffff81a15b20 "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 0}}, {scl_lock = {lock_object = {lo_name = 0xffffffff81a15b11 "scl->scl_lock", lo_flags = 40960000, 
          lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0x0, scl_write_wanted = 0, scl_cv = {cv_description = 0xffffffff81a15b20 "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 0}}, {
      scl_lock = {lock_object = {lo_name = 0xffffffff81a15b11 "scl->scl_lock", lo_flags = 40960000, lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0x0, scl_write_wanted = 0, scl_cv = {
        cv_description = 0xffffffff81a15b20 "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 0}}, {scl_lock = {lock_object = {lo_name = 0xffffffff81a15b11 "scl->scl_lock", lo_flags = 40960000, 
          lo_data = 0, lo_witness = 0x0}, sx_lock = 1}, scl_writer = 0x0, scl_write_wanted = 0, scl_cv = {cv_description = 0xffffffff81a15b20 "scl->scl_cv", cv_waiters = 0}, scl_count = {rc_count = 0}}}, 
  spa_refcount = {rc_count = 46}, spa_splitting_newspa = 0}