Ceph

Logs: /a/teuthology-2012-10-02_19:00:06-regression-master-testing-gcov/731

2012-10-02T20:18:42.774 DEBUG:teuthology.orchestra.run:Running: 'LD_LIBRARY_PRELOAD=/tmp/cephtest/binary/usr/local/lib /tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local/bin/ceph-coverage /tmp/cephtest/archive/coverage /tmp/cephtest/binary/usr/local/bin/ceph -k /tmp/cephtest/ceph.keyring -c /tmp/cephtest/ceph.conf --concise quorum_status'
2012-10-02T20:18:45.733 INFO:teuthology.task.ceph.mon.c.err:mon/PaxosService.cc: In function 'void PaxosService::propose_pending()' thread 7f45a75a5700 time 2012-10-02 20:18:45.726714
2012-10-02T20:18:45.733 INFO:teuthology.task.ceph.mon.c.err:mon/PaxosService.cc: 110: FAILED assert(have_pending)
2012-10-02T20:18:45.733 INFO:teuthology.task.ceph.mon.c.err: ceph version 0.52-938-g9e8f69a (commit:9e8f69a0da243566fd400cbc17dc557a4f0a57f8)
2012-10-02T20:18:45.733 INFO:teuthology.task.ceph.mon.c.err: 1: /tmp/cephtest/binary/usr/local/bin/ceph-mon() [0x4cb1fc]
2012-10-02T20:18:45.733 INFO:teuthology.task.ceph.mon.c.err: 2: (AuthMonitor::check_rotate()+0x24d) [0x5704dd]
2012-10-02T20:18:45.733 INFO:teuthology.task.ceph.mon.c.err: 3: (AuthMonitor::tick()+0x180) [0x570770]
2012-10-02T20:18:45.733 INFO:teuthology.task.ceph.mon.c.err: 4: (Monitor::tick()+0x8d) [0x49b00d]
2012-10-02T20:18:45.733 INFO:teuthology.task.ceph.mon.c.err: 5: (C_Mon_Tick::finish(int)+0x15) [0x4b5935]
2012-10-02T20:18:45.734 INFO:teuthology.task.ceph.mon.c.err: 6: (SafeTimer::timer_thread()+0x7a5) [0x670b15]
2012-10-02T20:18:45.734 INFO:teuthology.task.ceph.mon.c.err: 7: (SafeTimerThread::entry()+0x15) [0x673f45]
2012-10-02T20:18:45.734 INFO:teuthology.task.ceph.mon.c.err: 8: (Thread::_entry_func(void*)+0x12) [0x669fe2]
2012-10-02T20:18:45.734 INFO:teuthology.task.ceph.mon.c.err: 9: (()+0x7e9a) [0x7f45ad60ce9a]
2012-10-02T20:18:45.734 INFO:teuthology.task.ceph.mon.c.err: 10: (clone()+0x6d) [0x7f45abdc54bd]
2012-10-02T20:18:45.734 INFO:teuthology.task.ceph.mon.c.err: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2012-10-02T20:18:45.734 INFO:teuthology.task.ceph.mon.c.err:2012-10-02 20:18:45.728044 7f45a75a5700 -1 mon/PaxosService.cc: In function 'void PaxosService::propose_pending()' thread 7f45a75a5700 time 2012-10-02 20:18:45.726714
2012-10-02T20:18:45.734 INFO:teuthology.task.ceph.mon.c.err:mon/PaxosService.cc: 110: FAILED assert(have_pending)
2012-10-02T20:18:45.734 INFO:teuthology.task.ceph.mon.c.err:
2012-10-02T20:18:45.735 INFO:teuthology.task.ceph.mon.c.err: ceph version 0.52-938-g9e8f69a (commit:9e8f69a0da243566fd400cbc17dc557a4f0a57f8)
2012-10-02T20:18:45.735 INFO:teuthology.task.ceph.mon.c.err: 1: /tmp/cephtest/binary/usr/local/bin/ceph-mon() [0x4cb1fc]
2012-10-02T20:18:45.735 INFO:teuthology.task.ceph.mon.c.err: 2: (AuthMonitor::check_rotate()+0x24d) [0x5704dd]
2012-10-02T20:18:45.735 INFO:teuthology.task.ceph.mon.c.err: 3: (AuthMonitor::tick()+0x180) [0x570770]
2012-10-02T20:18:45.735 INFO:teuthology.task.ceph.mon.c.err: 4: (Monitor::tick()+0x8d) [0x49b00d]
2012-10-02T20:18:45.735 INFO:teuthology.task.ceph.mon.c.err: 5: (C_Mon_Tick::finish(int)+0x15) [0x4b5935]
2012-10-02T20:18:45.735 INFO:teuthology.task.ceph.mon.c.err: 6: (SafeTimer::timer_thread()+0x7a5) [0x670b15]
2012-10-02T20:18:45.735 INFO:teuthology.task.ceph.mon.c.err: 7: (SafeTimerThread::entry()+0x15) [0x673f45]
2012-10-02T20:18:45.735 INFO:teuthology.task.ceph.mon.c.err: 8: (Thread::_entry_func(void*)+0x12) [0x669fe2]
2012-10-02T20:18:45.736 INFO:teuthology.task.ceph.mon.c.err: 9: (()+0x7e9a) [0x7f45ad60ce9a]
2012-10-02T20:18:45.736 INFO:teuthology.task.ceph.mon.c.err: 10: (clone()+0x6d) [0x7f45abdc54bd]
2012-10-02T20:18:45.736 INFO:teuthology.task.ceph.mon.c.err: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
2012-10-02T20:18:45.736 INFO:teuthology.task.ceph.mon.c.err:

ubuntu@teuthology:/a/teuthology-2012-10-02_19:00:06-regression-master-testing-gcov/731$ cat config.yaml 
kernel: &id001
  kdb: true
  sha1: dc77db1c6af9893b03aa869f54b5634c65d8e290
nuke-on-error: true
overrides:
  ceph:
    conf:
      global:
        ms inject socket failures: 5000
    coverage: true
    log-whitelist:
    - slow request
    sha1: 9e8f69a0da243566fd400cbc17dc557a4f0a57f8
  s3tests:
    branch: master
  workunit:
    sha1: 9e8f69a0da243566fd400cbc17dc557a4f0a57f8
roles:
- - mon.a
  - mon.d
  - mon.g
  - osd.0
- - mon.b
  - mon.e
  - mon.h
  - mds.a
- - mon.c
  - mon.f
  - mon.i
  - osd.1
targets:
  ubuntu@plana24.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQClf8RuTWfOjpMgfDd3/Km+B8HeCQA3P0vqSF5UXyq5ZZ//pVogPo40/kJR//UvErWLMYM1udR+NVdRpecFZHObFMWeMVSvGTIaaz6JSWRNjBZ5sG8qoaViW4Q2CMN1htz1sfUkrafCALTPjtIAfKIJJf6h2gcU/UHQEWXed4nyLmzA6XLHJUUg89tkCimfWqQ5A0vZz+MdlgWkjHgmv502AYaWMl838q6v6iMiqOcVEjw/p4B+v9NTGlqdByzOCIS80Ps8AKWLE6VoyF+BEgLnnvO2kARufJvokt6Q8HHsIlvCOKz1d0EiXEu5TRw6D779QpveaoRLKMnr2dsSN6gn
  ubuntu@plana34.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDc3URAyD772tnoWHzOPXosRcjoZsDTSs3bhFuCp8DZdkkphtXy9eAuhTz5+nCUCzcumDfpxcYsisl84204gw3lmwvEicW4Tf+3NRWuHu05s+VGSFe81fGKGxUviM64dEvA/A54KAXbg+hxPg/rYFhHgDSLDL61MKYYZWI8kMV+M93qShzApy73Z8tDdva3WF0DrXV37TnPwSxc4R5j08t/y+5/WDjIj7u21S+kxqfGNQ9ycx/yHT5lf//2HS60dvBnMfQda8NEsUAtS+/9lA85qFiBOI2MzrnCRdi1Z5htVvM/em7mQmT5ttDvF4R5Qn9MgeV438lVoagO0+lvKoMf
  ubuntu@plana35.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtCEUU7ZjJxMBOYq9dlPZppwlahU15OIq1a/xqI0YIpAF7ZLf1eis04s6Kzvl45ooKP2J6DW598Lr9zTtFb6j9RoAkgIbIOkWZmLQjeZesi//7B8uXBw7niNzU1iBI83K8dPL2cwTsgwXXtoaD03pW9HCv4ADhH12/CsfoqNPfql7ze4JlI07seHmUqkBuBFxLVgdZDXb+Cjdg2sQnMS5qZo0OGBp2yfBla01Pk8V1sHGg39miF0EDJZyIOL2ziZOcm1f02WVkUAD8MMOtfQbf33DT/ya4Th20YtRMQxNLkUYG6rxcjutzILFKDr7ugbvOwtBhVS7qplqLWAwHdi0p
tasks:
- internal.lock_machines: 3
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock: null
- ceph: null
- mon_recovery: null
ubuntu@teuthology:/a/teuthology-2012-10-02_19:00:06-regression-master-testing-gcov/731$ cat summary.yaml 
ceph-sha1: 9e8f69a0da243566fd400cbc17dc557a4f0a57f8
description: collection:rados-multimon clusters:9.yaml msgr-failures:few.yaml tasks:mon_recovery.yaml
duration: 671.0348539352417
failure_reason: failed to reach quorum size 9 before timeout expired
flavor: gcov
mon.a-kernel-sha1: dc77db1c6af9893b03aa869f54b5634c65d8e290
mon.b-kernel-sha1: dc77db1c6af9893b03aa869f54b5634c65d8e290
mon.c-kernel-sha1: dc77db1c6af9893b03aa869f54b5634c65d8e290
owner: scheduled_teuthology@teuthology
success: false