集群报错如下

[root@ceph-01 ~]# ceph health detail
HEALTH_WARN 3 large omap objects
[WRN] LARGE_OMAP_OBJECTS: 3 large omap objects
    3 large objects found in pool 'lingxi-qd-1.rgw.buckets.index'
    Search the cluster log for 'Large omap object found' for more details.

查看集群日志，找到出现问题的object

[root@ceph-01 ~]# grep "Large omap object found" /var/log/messages |grep Object |awk '{print $19}'|awk -F : '{print $5}' |sort -n |uniq
.dir.cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21.3
.dir.cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21.7
.dir.cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21.9

根据object定位bucket

[root@ceph-01 ~]# radosgw-admin bucket stats |grep cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21 -B 10
        "bucket": "dfcf",
        "num_shards": 11,
        "tenant": "",
        "zonegroup": "47047f95-cfff-4a24-8e56-2e3ab6691ac3",
        "placement_rule": "default-placement",
        "explicit_placement": {
            "data_pool": "",
            "data_extra_pool": "",
            "index_pool": ""
        },
        "id": "cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21",
        "marker": "cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21

根据查到的bucket名称，查询bucket基本信息

[root@ceph-01 ~]# radosgw-admin bucket stats --bucket=dfcf
{
    "bucket": "dfcf",
    "num_shards": 11,
    "tenant": "",
    "zonegroup": "47047f95-cfff-4a24-8e56-2e3ab6691ac3",
    "placement_rule": "default-placement",
    "explicit_placement": {
        "data_pool": "",
        "data_extra_pool": "",
        "index_pool": ""
    },
    "id": "cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21",
    "marker": "cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21",
    "index_type": "Normal",
    "owner": "spider",
    "ver": "0#755427,1#763210,2#770175,3#756936,4#760306,5#768361,6#761515,7#753625,8#752718,9#763940,10#761233",
    "master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0,8#0,9#0,10#0",
    "mtime": "2021-08-04T09:51:16.920493Z",
    "creation_time": "2021-08-04T09:51:16.916603Z",
    "max_marker": "0#00000755425.6478303.5,1#00000763209.6478307.5,2#00000770174.3335244.5,3#00000756935.3388437.5,4#00000760302.5776776.5,5#00000768359.6478301.5,6#00000761512.3116803.5,7#00000753624.2586554.5,8#00000752716.5776808.5,9#00000763939.3134455.5,10#00000761231.5776810.5",
    "usage": {
        "rgw.main": {
            "size": 194987847856,
            "size_actual": 195264192512,
            "size_utilized": 194987847856,
            "size_kb": 190417821,
            "size_kb_actual": 190687688,
            "size_kb_utilized": 190417821,
            "num_objects": 135095
        }
    },
    "bucket_quota": {
        "enabled": false,
        "check_on_raw": false,
        "max_size": -1,
        "max_size_kb": 0,
        "max_objects": -1
    }
}

对bucket进行reshard操作

[root@ceph-01 ~]# radosgw-admin bucket reshard --bucket dfcf --num-shards 32
tenant: 
bucket name: dfcf
old bucket instance id: cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21
new bucket instance id: cf1290d9-5caa-4346-b0c2-b0731f1d0232.371388.1
total entries: 1000 2000 3000 4000 5000 6000 7000 8000 9000 10000 11000 12000 13000 14000 15000 16000 17000 18000 19000 20000 21000 22000 23000 24000 25000 26000 27000 28000 29000 30000 31000 32000 33000 34000 35000 36000 37000 38000 39000 40000 41000 42000 43000 44000 45000 46000 47000 48000 49000 50000 51000 52000 53000 54000 55000 56000 57000 58000 59000 60000 61000 62000 63000 64000 65000 66000 67000 68000 69000 70000 71000 72000 73000 74000 75000 76000 77000 78000 79000 80000 81000 82000 83000 84000 85000 86000 87000 88000 89000 90000 91000 92000 93000 94000 95000 96000 97000 98000 99000 100000 101000 102000 103000 104000 105000 106000 107000 108000 109000 110000 111000 112000 113000 114000 115000 116000 117000 118000 119000 120000 121000 122000 123000 124000 125000 126000 127000 128000 129000 130000 131000 132000 133000 134000 135000 135093
2021-09-09T14:28:52.185+0800 7f9227821040  1 execute INFO: reshard of bucket "dfcf" from "dfcf:cf1290d9-5caa-4346-b0c2-b0731f1d0232.34558.21" to "dfcf:cf1290d9-5caa-4346-b0c2-b0731f1d0232.371388.1" completed successfully

清理旧数据（octopus版本已经不需要清理index以及meta数据了，经过测试可以自动删除）

对pg进行deep-scrub操作，可以不执行，等集群自己进行deep-scrub也可以

[root@ceph-01 ~]# python3 large_obj.py 
Large omap objects poolname = lingxi-qd-1.rgw.buckets.index
pgid=19.1 OSDs=[1, 8, 7] num_large_omap_objects=1
pgid=19.3 OSDs=[16, 7, 19] num_large_omap_objects=1
pgid=19.7 OSDs=[20, 18, 4] num_large_omap_objects=1
You have new mail in /var/spool/mail/root
[root@ceph-01 ~]# ceph pg deep-scrub 19.1
instructing pg 19.1 on osd.1 to deep-scrub
[root@ceph-01 ~]# ceph pg deep-scrub 19.3
instructing pg 19.3 on osd.16 to deep-scrub
[root@ceph-01 ~]# ceph pg deep-scrub 19.7
instructing pg 19.7 on osd.20 to deep-scrub

再次查看集群状态，已恢复正常

附large_obj.py脚本

import json
import rados
import rbd

ceph_conf_path = '/etc/ceph/ceph.conf'
rados_connect_timeout = 5

class RADOSClient(object):
    def __init__(self, driver, pool=None):
        self.driver = driver
        self.client, self.ioctx = driver._connect_to_rados(pool)

    def __enter__(self):
        return self

    def __exit__(self, type_, value, traceback):
        self.driver._disconnect_from_rados(self.client, self.ioctx)

class RBDDriver(object):
    def __init__(self, ceph_conf_path, rados_connect_timeout, pool=None):
        self.ceph_conf_path = ceph_conf_path
        self.rados_connect_timeout = rados_connect_timeout
        self.pool = pool

    def _connect_to_rados(self, pool=None):
        client = rados.Rados(conffile=self.ceph_conf_path)
        try:
            if self.rados_connect_timeout >= 0:
                client.connect(timeout=self.rados_connect_timeout)
            else:
                client.connect()
            if self.pool == None:
                ioctx = None
            else:
                ioctx = client.open_ioctx(self.pool)
            return client, ioctx
        except rados.Error:
            msg = "Error connecting to ceph cluster."
            client.shutdown()
            raise msg

    def _disconnect_from_rados(self, client, ioctx=None):
        if ioctx == None:
            client.shutdown()
        else:
            ioctx.close()
            client.shutdown()

class cmd_manager():
    def get_large_omap_obj_poolname(self):
        with RADOSClient(RBDDriver(ceph_conf_path, rados_connect_timeout)) as dr:
            result = ''
            cmd = '{"prefix": "health", "detail": "detail", "format": "json"}'
            result = dr.client.mon_command(cmd, result)
            if result[0] == 0:
                res_ = json.loads(result[1])
                if res_["checks"]['LARGE_OMAP_OBJECTS']:
                    return res_["checks"]['LARGE_OMAP_OBJECTS']['detail'][0]['message'].split("'")[1]
            else:
                return False

    def get_pg_list_by_pool(self, poolname):
        with RADOSClient(RBDDriver(ceph_conf_path, rados_connect_timeout)) as dr:
            result = ''
            cmd = '{"prefix": "pg ls-by-pool", "poolstr": "' + \
                poolname + '", "format": "json"}'
            result = dr.client.mon_command(cmd, result)
            if result[0] == 0:
                return json.loads(result[1])
            else:
                return False

cmd_ = cmd_manager()
poolname = cmd_.get_large_omap_obj_poolname()
print("Large omap objects poolname = {0}".format(poolname))
res = cmd_.get_pg_list_by_pool(poolname)
#print (res)
for i in res['pg_stats']:
    if i["stat_sum"]["num_large_omap_objects"] != 0:
        print("pgid={0} OSDs={1} num_large_omap_objects={2}".format(
            i["pgid"],
            i["acting"],
            i["stat_sum"]["num_large_omap_objects"])
        )
最后修改：2021 年 09 月 09 日
© 允许规范转载
如果觉得我的文章对你有用，请随意赞赏