|
Tue 09 Apr 2024 07:09:41 AM UTC
|
|
cluster:
|
|
id: 346a7566-c0d1-11ec-8d00-e9947da2ae5b
|
|
health: HEALTH_WARN
|
|
1 MDSs report slow requests
|
|
(muted: LARGE_OMAP_OBJECTS MDS_CLIENT_LATE_RELEASE MDS_CLIENT_RECALL PG_NOT_DEEP_SCRUBBED PG_NOT_SCRUBBED)
|
|
|
|
services:
|
|
mon: 3 daemons, quorum ceph-head-03,ceph-head-02,ceph-head-01 (age 4w)
|
|
mgr: ceph-head-02(active, since 4w), standbys: ceph-head-03, ceph-head-01
|
|
mds: 1/1 daemons up, 2 standby
|
|
osd: 186 osds: 186 up (since 10d), 186 in (since 12d)
|
|
|
|
data:
|
|
volumes: 1/1 healthy
|
|
pools: 4 pools, 2337 pgs
|
|
objects: 995.62M objects, 797 TiB
|
|
usage: 1.1 PiB used, 753 TiB / 1.9 PiB avail
|
|
pgs: 2316 active+clean
|
|
13 active+clean+scrubbing+deep
|
|
8 active+clean+scrubbing
|
|
|
|
io:
|
|
client: 447 MiB/s rd, 1.9 MiB/s wr, 1.01k op/s rd, 20 op/s wr
|
|
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "client_request(client.787937105:3561542 getattr Xs #0x1006f197b80 2024-04-08T15:20:46.191504+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129211+0000",
|
|
"age": 8493.1739054369991,
|
|
"duration": 8493.1739168330005,
|
|
"type_data": {
|
|
"flag_point": "failed to rdlock, waiting",
|
|
"reqid": "client.787937105:3561542",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3561542
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129212+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129225+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337521+0000",
|
|
"event": "failed to rdlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"description": "client_request(client.787937105:3565397 setattr size=0 mtime=2024-04-08T16:15:57.636737+0000 #0x1006f197b80 2024-04-08T16:15:57.636737+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129229+0000",
|
|
"age": 8493.173887723,
|
|
"duration": 8493.1739199349995,
|
|
"type_data": {
|
|
"flag_point": "failed to xlock, waiting",
|
|
"reqid": "client.787937105:3565397",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3565397
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129230+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129235+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337524+0000",
|
|
"event": "failed to xlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"num_ops": 2
|
|
}
|
|
Tue 09 Apr 2024 07:09:47 AM UTC
|
|
cluster:
|
|
id: 346a7566-c0d1-11ec-8d00-e9947da2ae5b
|
|
health: HEALTH_WARN
|
|
1 MDSs report slow requests
|
|
(muted: LARGE_OMAP_OBJECTS MDS_CLIENT_LATE_RELEASE MDS_CLIENT_RECALL PG_NOT_DEEP_SCRUBBED PG_NOT_SCRUBBED)
|
|
|
|
services:
|
|
mon: 3 daemons, quorum ceph-head-03,ceph-head-02,ceph-head-01 (age 4w)
|
|
mgr: ceph-head-02(active, since 4w), standbys: ceph-head-03, ceph-head-01
|
|
mds: 1/1 daemons up, 2 standby
|
|
osd: 186 osds: 186 up (since 10d), 186 in (since 12d)
|
|
|
|
data:
|
|
volumes: 1/1 healthy
|
|
pools: 4 pools, 2337 pgs
|
|
objects: 995.62M objects, 797 TiB
|
|
usage: 1.1 PiB used, 753 TiB / 1.9 PiB avail
|
|
pgs: 2316 active+clean
|
|
13 active+clean+scrubbing+deep
|
|
8 active+clean+scrubbing
|
|
|
|
io:
|
|
client: 419 MiB/s rd, 2.3 MiB/s wr, 1.01k op/s rd, 41 op/s wr
|
|
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "client_request(client.787937105:3561542 getattr Xs #0x1006f197b80 2024-04-08T15:20:46.191504+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129211+0000",
|
|
"age": 8498.7845084809996,
|
|
"duration": 8498.7845205270005,
|
|
"type_data": {
|
|
"flag_point": "failed to rdlock, waiting",
|
|
"reqid": "client.787937105:3561542",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3561542
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129212+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129225+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337521+0000",
|
|
"event": "failed to rdlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"description": "client_request(client.787937105:3565397 setattr size=0 mtime=2024-04-08T16:15:57.636737+0000 #0x1006f197b80 2024-04-08T16:15:57.636737+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129229+0000",
|
|
"age": 8498.7844907670005,
|
|
"duration": 8498.7845236859994,
|
|
"type_data": {
|
|
"flag_point": "failed to xlock, waiting",
|
|
"reqid": "client.787937105:3565397",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3565397
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129230+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129235+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337524+0000",
|
|
"event": "failed to xlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"num_ops": 2
|
|
}
|
|
Tue 09 Apr 2024 07:09:52 AM UTC
|
|
cluster:
|
|
id: 346a7566-c0d1-11ec-8d00-e9947da2ae5b
|
|
health: HEALTH_WARN
|
|
1 MDSs report slow requests
|
|
(muted: LARGE_OMAP_OBJECTS MDS_CLIENT_LATE_RELEASE MDS_CLIENT_RECALL PG_NOT_DEEP_SCRUBBED PG_NOT_SCRUBBED)
|
|
|
|
services:
|
|
mon: 3 daemons, quorum ceph-head-03,ceph-head-02,ceph-head-01 (age 4w)
|
|
mgr: ceph-head-02(active, since 4w), standbys: ceph-head-03, ceph-head-01
|
|
mds: 1/1 daemons up, 2 standby
|
|
osd: 186 osds: 186 up (since 10d), 186 in (since 12d)
|
|
|
|
data:
|
|
volumes: 1/1 healthy
|
|
pools: 4 pools, 2337 pgs
|
|
objects: 995.62M objects, 797 TiB
|
|
usage: 1.1 PiB used, 753 TiB / 1.9 PiB avail
|
|
pgs: 2316 active+clean
|
|
13 active+clean+scrubbing+deep
|
|
8 active+clean+scrubbing
|
|
|
|
io:
|
|
client: 352 MiB/s rd, 1.7 MiB/s wr, 818 op/s rd, 45 op/s wr
|
|
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "client_request(client.787937105:3561542 getattr Xs #0x1006f197b80 2024-04-08T15:20:46.191504+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129211+0000",
|
|
"age": 8504.4148384010005,
|
|
"duration": 8504.4148521909992,
|
|
"type_data": {
|
|
"flag_point": "failed to rdlock, waiting",
|
|
"reqid": "client.787937105:3561542",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3561542
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129212+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129225+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337521+0000",
|
|
"event": "failed to rdlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"description": "client_request(client.787937105:3565397 setattr size=0 mtime=2024-04-08T16:15:57.636737+0000 #0x1006f197b80 2024-04-08T16:15:57.636737+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129229+0000",
|
|
"age": 8504.4148206869995,
|
|
"duration": 8504.4148569680001,
|
|
"type_data": {
|
|
"flag_point": "failed to xlock, waiting",
|
|
"reqid": "client.787937105:3565397",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3565397
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129230+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129235+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337524+0000",
|
|
"event": "failed to xlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"num_ops": 2
|
|
}
|
|
Tue 09 Apr 2024 07:09:58 AM UTC
|
|
cluster:
|
|
id: 346a7566-c0d1-11ec-8d00-e9947da2ae5b
|
|
health: HEALTH_WARN
|
|
1 MDSs report slow requests
|
|
(muted: LARGE_OMAP_OBJECTS MDS_CLIENT_LATE_RELEASE MDS_CLIENT_RECALL PG_NOT_DEEP_SCRUBBED PG_NOT_SCRUBBED)
|
|
|
|
services:
|
|
mon: 3 daemons, quorum ceph-head-03,ceph-head-02,ceph-head-01 (age 4w)
|
|
mgr: ceph-head-02(active, since 4w), standbys: ceph-head-03, ceph-head-01
|
|
mds: 1/1 daemons up, 2 standby
|
|
osd: 186 osds: 186 up (since 10d), 186 in (since 12d)
|
|
|
|
data:
|
|
volumes: 1/1 healthy
|
|
pools: 4 pools, 2337 pgs
|
|
objects: 995.62M objects, 797 TiB
|
|
usage: 1.1 PiB used, 753 TiB / 1.9 PiB avail
|
|
pgs: 2316 active+clean
|
|
13 active+clean+scrubbing+deep
|
|
8 active+clean+scrubbing
|
|
|
|
io:
|
|
client: 395 MiB/s rd, 688 KiB/s wr, 853 op/s rd, 30 op/s wr
|
|
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "client_request(client.787937105:3561542 getattr Xs #0x1006f197b80 2024-04-08T15:20:46.191504+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129211+0000",
|
|
"age": 8510.0688218060004,
|
|
"duration": 8510.0688343790007,
|
|
"type_data": {
|
|
"flag_point": "failed to rdlock, waiting",
|
|
"reqid": "client.787937105:3561542",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3561542
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129212+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129225+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337521+0000",
|
|
"event": "failed to rdlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"description": "client_request(client.787937105:3565397 setattr size=0 mtime=2024-04-08T16:15:57.636737+0000 #0x1006f197b80 2024-04-08T16:15:57.636737+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129229+0000",
|
|
"age": 8510.0688040919995,
|
|
"duration": 8510.0688377269998,
|
|
"type_data": {
|
|
"flag_point": "failed to xlock, waiting",
|
|
"reqid": "client.787937105:3565397",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3565397
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129230+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129235+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337524+0000",
|
|
"event": "failed to xlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"num_ops": 2
|
|
}
|
|
Tue 09 Apr 2024 07:10:04 AM UTC
|
|
cluster:
|
|
id: 346a7566-c0d1-11ec-8d00-e9947da2ae5b
|
|
health: HEALTH_WARN
|
|
1 MDSs report slow requests
|
|
(muted: LARGE_OMAP_OBJECTS MDS_CLIENT_LATE_RELEASE MDS_CLIENT_RECALL PG_NOT_DEEP_SCRUBBED PG_NOT_SCRUBBED)
|
|
|
|
services:
|
|
mon: 3 daemons, quorum ceph-head-03,ceph-head-02,ceph-head-01 (age 4w)
|
|
mgr: ceph-head-02(active, since 4w), standbys: ceph-head-03, ceph-head-01
|
|
mds: 1/1 daemons up, 2 standby
|
|
osd: 186 osds: 186 up (since 10d), 186 in (since 12d)
|
|
|
|
data:
|
|
volumes: 1/1 healthy
|
|
pools: 4 pools, 2337 pgs
|
|
objects: 995.62M objects, 797 TiB
|
|
usage: 1.1 PiB used, 753 TiB / 1.9 PiB avail
|
|
pgs: 2316 active+clean
|
|
13 active+clean+scrubbing+deep
|
|
8 active+clean+scrubbing
|
|
|
|
io:
|
|
client: 527 MiB/s rd, 454 KiB/s wr, 1.09k op/s rd, 24 op/s wr
|
|
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "client_request(client.787937105:3561542 getattr Xs #0x1006f197b80 2024-04-08T15:20:46.191504+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129211+0000",
|
|
"age": 8515.6961398820004,
|
|
"duration": 8515.6961506959997,
|
|
"type_data": {
|
|
"flag_point": "failed to rdlock, waiting",
|
|
"reqid": "client.787937105:3561542",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3561542
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129212+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129225+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337521+0000",
|
|
"event": "failed to rdlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"description": "client_request(client.787937105:3565397 setattr size=0 mtime=2024-04-08T16:15:57.636737+0000 #0x1006f197b80 2024-04-08T16:15:57.636737+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129229+0000",
|
|
"age": 8515.6961221679994,
|
|
"duration": 8515.696154362,
|
|
"type_data": {
|
|
"flag_point": "failed to xlock, waiting",
|
|
"reqid": "client.787937105:3565397",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3565397
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129230+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129235+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337524+0000",
|
|
"event": "failed to xlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"num_ops": 2
|
|
}
|
|
Tue 09 Apr 2024 07:10:09 AM UTC
|
|
cluster:
|
|
id: 346a7566-c0d1-11ec-8d00-e9947da2ae5b
|
|
health: HEALTH_OK
|
|
(muted: LARGE_OMAP_OBJECTS MDS_CLIENT_LATE_RELEASE MDS_CLIENT_RECALL PG_NOT_DEEP_SCRUBBED PG_NOT_SCRUBBED)
|
|
|
|
services:
|
|
mon: 3 daemons, quorum ceph-head-03,ceph-head-02,ceph-head-01 (age 4w)
|
|
mgr: ceph-head-02(active, since 4w), standbys: ceph-head-03, ceph-head-01
|
|
mds: 1/1 daemons up, 2 standby
|
|
osd: 186 osds: 186 up (since 10d), 186 in (since 12d)
|
|
|
|
data:
|
|
volumes: 1/1 healthy
|
|
pools: 4 pools, 2337 pgs
|
|
objects: 995.62M objects, 797 TiB
|
|
usage: 1.1 PiB used, 753 TiB / 1.9 PiB avail
|
|
pgs: 2316 active+clean
|
|
13 active+clean+scrubbing+deep
|
|
8 active+clean+scrubbing
|
|
|
|
io:
|
|
client: 581 MiB/s rd, 536 KiB/s wr, 1.26k op/s rd, 24 op/s wr
|
|
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "client_request(client.787937105:3561542 getattr Xs #0x1006f197b80 2024-04-08T15:20:46.191504+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129211+0000",
|
|
"age": 8521.3127998159998,
|
|
"duration": 8521.3128120189995,
|
|
"type_data": {
|
|
"flag_point": "failed to rdlock, waiting",
|
|
"reqid": "client.787937105:3561542",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3561542
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129212+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129225+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337521+0000",
|
|
"event": "failed to rdlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"description": "client_request(client.787937105:3565397 setattr size=0 mtime=2024-04-08T16:15:57.636737+0000 #0x1006f197b80 2024-04-08T16:15:57.636737+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129229+0000",
|
|
"age": 8521.3127821020007,
|
|
"duration": 8521.3128153750004,
|
|
"type_data": {
|
|
"flag_point": "failed to xlock, waiting",
|
|
"reqid": "client.787937105:3565397",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3565397
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129230+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129235+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337524+0000",
|
|
"event": "failed to xlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"num_ops": 2
|
|
}
|
|
Tue 09 Apr 2024 07:10:15 AM UTC
|
|
cluster:
|
|
id: 346a7566-c0d1-11ec-8d00-e9947da2ae5b
|
|
health: HEALTH_OK
|
|
(muted: LARGE_OMAP_OBJECTS MDS_CLIENT_LATE_RELEASE MDS_CLIENT_RECALL PG_NOT_DEEP_SCRUBBED PG_NOT_SCRUBBED)
|
|
|
|
services:
|
|
mon: 3 daemons, quorum ceph-head-03,ceph-head-02,ceph-head-01 (age 4w)
|
|
mgr: ceph-head-02(active, since 4w), standbys: ceph-head-03, ceph-head-01
|
|
mds: 1/1 daemons up, 2 standby
|
|
osd: 186 osds: 186 up (since 10d), 186 in (since 12d)
|
|
|
|
data:
|
|
volumes: 1/1 healthy
|
|
pools: 4 pools, 2337 pgs
|
|
objects: 995.62M objects, 797 TiB
|
|
usage: 1.1 PiB used, 753 TiB / 1.9 PiB avail
|
|
pgs: 2316 active+clean
|
|
13 active+clean+scrubbing+deep
|
|
8 active+clean+scrubbing
|
|
|
|
io:
|
|
client: 550 MiB/s rd, 469 KiB/s wr, 1.34k op/s rd, 20 op/s wr
|
|
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "client_request(client.787937105:3561542 getattr Xs #0x1006f197b80 2024-04-08T15:20:46.191504+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129211+0000",
|
|
"age": 8526.9493543690005,
|
|
"duration": 8526.94936925,
|
|
"type_data": {
|
|
"flag_point": "failed to rdlock, waiting",
|
|
"reqid": "client.787937105:3561542",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3561542
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129212+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129225+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337521+0000",
|
|
"event": "failed to rdlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"description": "client_request(client.787937105:3565397 setattr size=0 mtime=2024-04-08T16:15:57.636737+0000 #0x1006f197b80 2024-04-08T16:15:57.636737+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129229+0000",
|
|
"age": 8526.9493366549996,
|
|
"duration": 8526.9493728500001,
|
|
"type_data": {
|
|
"flag_point": "failed to xlock, waiting",
|
|
"reqid": "client.787937105:3565397",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3565397
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129230+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129235+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337524+0000",
|
|
"event": "failed to xlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"num_ops": 2
|
|
}
|
|
Tue 09 Apr 2024 07:10:21 AM UTC
|
|
cluster:
|
|
id: 346a7566-c0d1-11ec-8d00-e9947da2ae5b
|
|
health: HEALTH_WARN
|
|
1 MDSs report slow requests
|
|
(muted: LARGE_OMAP_OBJECTS MDS_CLIENT_LATE_RELEASE MDS_CLIENT_RECALL PG_NOT_DEEP_SCRUBBED PG_NOT_SCRUBBED)
|
|
|
|
services:
|
|
mon: 3 daemons, quorum ceph-head-03,ceph-head-02,ceph-head-01 (age 4w)
|
|
mgr: ceph-head-02(active, since 4w), standbys: ceph-head-03, ceph-head-01
|
|
mds: 1/1 daemons up, 2 standby
|
|
osd: 186 osds: 186 up (since 10d), 186 in (since 12d)
|
|
|
|
data:
|
|
volumes: 1/1 healthy
|
|
pools: 4 pools, 2337 pgs
|
|
objects: 995.62M objects, 797 TiB
|
|
usage: 1.1 PiB used, 753 TiB / 1.9 PiB avail
|
|
pgs: 2316 active+clean
|
|
13 active+clean+scrubbing+deep
|
|
8 active+clean+scrubbing
|
|
|
|
io:
|
|
client: 582 MiB/s rd, 445 KiB/s wr, 1.55k op/s rd, 18 op/s wr
|
|
|
|
{
|
|
"ops": [
|
|
{
|
|
"description": "client_request(client.787937105:3561542 getattr Xs #0x1006f197b80 2024-04-08T15:20:46.191504+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129211+0000",
|
|
"age": 8532.5836730909996,
|
|
"duration": 8532.5836851749991,
|
|
"type_data": {
|
|
"flag_point": "failed to rdlock, waiting",
|
|
"reqid": "client.787937105:3561542",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3561542
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129212+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129211+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129225+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337521+0000",
|
|
"event": "failed to rdlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"description": "client_request(client.787937105:3565397 setattr size=0 mtime=2024-04-08T16:15:57.636737+0000 #0x1006f197b80 2024-04-08T16:15:57.636737+0000 RETRY=2 caller_uid=1714422991, caller_gid=1714422273{})",
|
|
"initiated_at": "2024-04-09T04:48:09.129229+0000",
|
|
"age": 8532.5836553770005,
|
|
"duration": 8532.5836896889996,
|
|
"type_data": {
|
|
"flag_point": "failed to xlock, waiting",
|
|
"reqid": "client.787937105:3565397",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.787937105",
|
|
"tid": 3565397
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129229+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129230+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:48:09.129235+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T04:49:59.337524+0000",
|
|
"event": "failed to xlock, waiting"
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"description": "client_request(client.847764236:29918080 rmdir #0x100104c1ec7/6aae0077-521a-406d-a10b-f24c71110bf6 2024-04-09T07:10:21.455444+0000 caller_uid=0, caller_gid=0{})",
|
|
"initiated_at": "2024-04-09T07:10:21.459282+0000",
|
|
"age": 0.25360236600000002,
|
|
"duration": 0.25365628600000001,
|
|
"type_data": {
|
|
"flag_point": "submit entry: journal_and_reply",
|
|
"reqid": "client.847764236:29918080",
|
|
"op_type": "client_request",
|
|
"client_info": {
|
|
"client": "client.847764236",
|
|
"tid": 29918080
|
|
},
|
|
"events": [
|
|
{
|
|
"time": "2024-04-09T07:10:21.459282+0000",
|
|
"event": "initiated"
|
|
},
|
|
{
|
|
"time": "2024-04-09T07:10:21.459282+0000",
|
|
"event": "throttled"
|
|
},
|
|
{
|
|
"time": "2024-04-09T07:10:21.459282+0000",
|
|
"event": "header_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T07:10:21.459287+0000",
|
|
"event": "all_read"
|
|
},
|
|
{
|
|
"time": "2024-04-09T07:10:21.459292+0000",
|
|
"event": "dispatched"
|
|
},
|
|
{
|
|
"time": "2024-04-09T07:10:21.459320+0000",
|
|
"event": "acquired locks"
|
|
},
|
|
{
|
|
"time": "2024-04-09T07:10:21.459338+0000",
|
|
"event": "acquired locks"
|
|
},
|
|
{
|
|
"time": "2024-04-09T07:10:21.459422+0000",
|
|
"event": "early_replied"
|
|
},
|
|
{
|
|
"time": "2024-04-09T07:10:21.459422+0000",
|
|
"event": "submit entry: journal_and_reply"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"num_ops": 3
|
|
}
|