path: root/ceph-19.2.3-backport-ec-corruption-fix.patch

From f716fe0dce3c66c35a0fae20d3165c4e41d6d3d9 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Mon, 20 May 2024 12:07:01 +0200
Subject: [PATCH 01/12] os/bluestore: improve ExtentMap::update readabilty

don't use "p" to name things

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
(cherry picked from commit 859f920e96408b04ea7bd6e568d3365544b1335f)
---
 src/os/bluestore/BlueStore.cc | 107 +++++++++++++++++-----------------
 1 file changed, 55 insertions(+), 52 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 01948bcd2370a..e215e6e1afbc1 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3473,62 +3473,65 @@ void BlueStore::ExtentMap::update(KeyValueDB::Transaction t,
     // doing multiple allocations - one per each dirty shard
     encoded_shards.reserve(shards.size());
 
-    auto p = shards.begin();
-    auto prev_p = p;
-    while (p != shards.end()) {
-      ceph_assert(p->shard_info->offset >= prev_p->shard_info->offset);
-      auto n = p;
-      ++n;
-      if (p->dirty) {
-	uint32_t endoff;
-	if (n == shards.end()) {
-	  endoff = OBJECT_MAX_SIZE;
-	} else {
-	  endoff = n->shard_info->offset;
-	}
-	encoded_shards.emplace_back(dirty_shard_t(&(*p)));
-        bufferlist& bl = encoded_shards.back().bl;
-	if (encode_some(p->shard_info->offset, endoff - p->shard_info->offset,
-			bl, &p->extents)) {
-	  if (force) {
-	    _dump_extent_map<-1>(cct, *this);
-	    derr << __func__ << "  encode_some needs reshard" << dendl;
-	    ceph_assert(!force);
-	  }
-	}
-        size_t len = bl.length();
+    auto shard = shards.begin();
+    auto previous_shard = shard;
+    while (shard != shards.end()) {
+      ceph_assert(shard->shard_info->offset >= previous_shard->shard_info->offset);
+      auto next_shard = shard + 1;
+      if (!shard->dirty) {
+        previous_shard = shard;
+        shard = next_shard;
+        continue;
+      }
 
-	dout(20) << __func__ << "  shard 0x" << std::hex
-		 << p->shard_info->offset << std::dec << " is " << len
-		 << " bytes (was " << p->shard_info->bytes << ") from "
-		 << p->extents << " extents" << dendl;
+      uint32_t endoff;
+      if (next_shard == shards.end()) {
+        endoff = OBJECT_MAX_SIZE;
+      } else {
+        endoff = next_shard->shard_info->offset;
+      }
+      encoded_shards.emplace_back(dirty_shard_t(&(*shard)));
+      bufferlist& bl = encoded_shards.back().bl;
+      if (encode_some(shard->shard_info->offset, endoff - shard->shard_info->offset,
+      		bl, &shard->extents)) {
+        if (force) {
+          _dump_extent_map<-1>(cct, *this);
+          derr << __func__ << "  encode_some needs reshard" << dendl;
+          ceph_assert(!force);
+        }
+      }
+      size_t len = bl.length();
 
-        if (!force) {
-	  if (len > cct->_conf->bluestore_extent_map_shard_max_size) {
-	    // we are big; reshard ourselves
-	    request_reshard(p->shard_info->offset, endoff);
-	  }
-	  // avoid resharding the trailing shard, even if it is small
-	  else if (n != shards.end() &&
-		   len < g_conf()->bluestore_extent_map_shard_min_size) {
-            ceph_assert(endoff != OBJECT_MAX_SIZE);
-	    if (p == shards.begin()) {
-	      // we are the first shard, combine with next shard
-	      request_reshard(p->shard_info->offset, endoff + 1);
-	    } else {
-	      // combine either with the previous shard or the next,
-	      // whichever is smaller
-	      if (prev_p->shard_info->bytes > n->shard_info->bytes) {
-		request_reshard(p->shard_info->offset, endoff + 1);
-	      } else {
-		request_reshard(prev_p->shard_info->offset, endoff);
-	      }
-	    }
-	  }
+      dout(20) << __func__ << "  shard 0x" << std::hex
+      	 << shard->shard_info->offset << std::dec << " is " << len
+      	 << " bytes (was " << shard->shard_info->bytes << ") from "
+      	 << shard->extents << " extents" << dendl;
+
+      if (!force) {
+        if (len > cct->_conf->bluestore_extent_map_shard_max_size) {
+          // we are big; reshard ourselves
+          request_reshard(shard->shard_info->offset, endoff);
+        }
+        // avoid resharding the trailing shard, even if it is small
+        else if (next_shard != shards.end() &&
+      	   len < g_conf()->bluestore_extent_map_shard_min_size) {
+          ceph_assert(endoff != OBJECT_MAX_SIZE);
+          if (shard == shards.begin()) {
+            // we are the first shard, combine with next shard
+            request_reshard(shard->shard_info->offset, endoff + 1);
+          } else {
+            // combine either with the previous shard or the next,
+            // whichever is smaller
+            if (previous_shard->shard_info->bytes > next_shard->shard_info->bytes) {
+           	request_reshard(shard->shard_info->offset, endoff + 1);
+            } else {
+            	request_reshard(previous_shard->shard_info->offset, endoff);
+            }
+          }
         }
       }
-      prev_p = p;
-      p = n;
+      previous_shard = shard;
+      shard = next_shard;
     }
     if (needs_reshard()) {
       return;

From 3b9827ccee295e9672d859e35ac244fbc3267d91 Mon Sep 17 00:00:00 2001
From: Pere Diaz Bou <pere-altea@hotmail.com>
Date: Mon, 20 May 2024 15:50:27 +0200
Subject: [PATCH 02/12] os/bluestore: improve ExtentMap::reshard readabilty

rename:
    * p -> shard
    * sp -> current_shard
    * esp -> end_shard
    * si_begin -> shard_index_begin
    * si_end -> shard_index_end
    * e -> extent
    * bs -> blob_start
    * be -> blob_end
    * sv -> extent_map_shards
    ...

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>
(cherry picked from commit 91b0207ed8d22684c2221c648ab43441e0e2da74)

 Conflicts:
	src/os/bluestore/BlueStore.cc
 - trivial
---
 src/os/bluestore/BlueStore.cc | 151 +++++++++++++++++-----------------
 1 file changed, 75 insertions(+), 76 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index e215e6e1afbc1..bad247e322819 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3595,23 +3595,23 @@ void BlueStore::ExtentMap::reshard(
 	     << dendl;
   }
   // determine shard index range
-  unsigned si_begin = 0, si_end = 0;
+  unsigned shard_index_begin = 0, shard_index_end = 0;
   if (!shards.empty()) {
-    while (si_begin + 1 < shards.size() &&
-	   shards[si_begin + 1].shard_info->offset <= needs_reshard_begin) {
-      ++si_begin;
-    }
-    needs_reshard_begin = shards[si_begin].shard_info->offset;
-    for (si_end = si_begin; si_end < shards.size(); ++si_end) {
-      if (shards[si_end].shard_info->offset >= needs_reshard_end) {
-	needs_reshard_end = shards[si_end].shard_info->offset;
+    while (shard_index_begin + 1 < shards.size() &&
+	   shards[shard_index_begin + 1].shard_info->offset <= needs_reshard_begin) {
+      ++shard_index_begin;
+    }
+    needs_reshard_begin = shards[shard_index_begin].shard_info->offset;
+    for (shard_index_end = shard_index_begin; shard_index_end < shards.size(); ++shard_index_end) {
+      if (shards[shard_index_end].shard_info->offset >= needs_reshard_end) {
+	needs_reshard_end = shards[shard_index_end].shard_info->offset;
 	break;
       }
     }
-    if (si_end == shards.size()) {
+    if (shard_index_end == shards.size()) {
       needs_reshard_end = OBJECT_MAX_SIZE;
     }
-    dout(20) << __func__ << "   shards [" << si_begin << "," << si_end << ")"
+    dout(20) << __func__ << "   shards [" << shard_index_begin << "," << shard_index_end << ")"
 	     << " over 0x[" << std::hex << needs_reshard_begin << ","
 	     << needs_reshard_end << ")" << std::dec << dendl;
   } else {
@@ -3632,7 +3632,7 @@ void BlueStore::ExtentMap::reshard(
 
   // remove old keys
   string key;
-  for (unsigned i = si_begin; i < si_end; ++i) {
+  for (unsigned i = shard_index_begin; i < shard_index_end; ++i) {
     generate_extent_shard_key_and_apply(
       onode->key, shards[i].shard_info->offset, &key,
       [&](const string& final_key) {
@@ -3648,7 +3648,7 @@ void BlueStore::ExtentMap::reshard(
     bytes = inline_bl.length();
     extents = extent_map.size();
   } else {
-    for (unsigned i = si_begin; i < si_end; ++i) {
+    for (unsigned i = shard_index_begin; i < shard_index_end; ++i) {
       bytes += shards[i].shard_info->bytes;
       extents += shards[i].extents;
     }
@@ -3666,16 +3666,16 @@ void BlueStore::ExtentMap::reshard(
   vector<bluestore_onode_t::shard_info> new_shard_info;
   unsigned max_blob_end = 0;
   Extent dummy(needs_reshard_begin);
-  for (auto e = extent_map.lower_bound(dummy);
-       e != extent_map.end();
-       ++e) {
-    if (e->logical_offset >= needs_reshard_end) {
+  for (auto extent = extent_map.lower_bound(dummy);
+       extent != extent_map.end();
+       ++extent) {
+    if (extent->logical_offset >= needs_reshard_end) {
       break;
     }
-    dout(30) << " extent " << *e << dendl;
+    dout(30) << " extent " << *extent << dendl;
 
     // disfavor shard boundaries that span a blob
-    bool would_span = (e->logical_offset < max_blob_end) || e->blob_offset;
+    bool would_span = (extent->logical_offset < max_blob_end) || extent->blob_offset;
     if (estimate &&
 	estimate + extent_avg > target + (would_span ? slop : 0)) {
       // new shard
@@ -3685,7 +3685,7 @@ void BlueStore::ExtentMap::reshard(
 	dout(20) << __func__ << "  new shard 0x" << std::hex << offset
                  << std::dec << dendl;
       }
-      offset = e->logical_offset;
+      offset = extent->logical_offset;
       new_shard_info.emplace_back(bluestore_onode_t::shard_info());
       new_shard_info.back().offset = offset;
       dout(20) << __func__ << "  new shard 0x" << std::hex << offset
@@ -3693,20 +3693,20 @@ void BlueStore::ExtentMap::reshard(
       estimate = 0;
     }
     estimate += extent_avg;
-    unsigned bs = e->blob_start();
-    if (bs < spanning_scan_begin) {
-      spanning_scan_begin = bs;
+    unsigned blob_start = extent->blob_start();
+    if (blob_start < spanning_scan_begin) {
+      spanning_scan_begin = blob_start;
     }
-    uint32_t be = e->blob_end();
-    if (be > max_blob_end) {
-      max_blob_end = be;
+    uint32_t blob_end = extent->blob_end();
+    if (blob_end > max_blob_end) {
+      max_blob_end = blob_end;
     }
-    if (be > spanning_scan_end) {
-      spanning_scan_end = be;
+    if (blob_end > spanning_scan_end) {
+      spanning_scan_end = blob_end;
     }
   }
-  if (new_shard_info.empty() && (si_begin > 0 ||
-				 si_end < shards.size())) {
+  if (new_shard_info.empty() && (shard_index_begin > 0 ||
+				 shard_index_end < shards.size())) {
     // we resharded a partial range; we must produce at least one output
     // shard
     new_shard_info.emplace_back(bluestore_onode_t::shard_info());
@@ -3715,48 +3715,48 @@ void BlueStore::ExtentMap::reshard(
 	     << std::dec << " (singleton degenerate case)" << dendl;
   }
 
-  auto& sv = onode->onode.extent_map_shards;
+  auto& extent_map_shards = onode->onode.extent_map_shards;
   dout(20) << __func__ << "  new " << new_shard_info << dendl;
-  dout(20) << __func__ << "  old " << sv << dendl;
-  if (sv.empty()) {
+  dout(20) << __func__ << "  old " << extent_map_shards << dendl;
+  if (extent_map_shards.empty()) {
     // no old shards to keep
-    sv.swap(new_shard_info);
+    extent_map_shards.swap(new_shard_info);
     init_shards(true, true);
   } else {
     // splice in new shards
-    sv.erase(sv.begin() + si_begin, sv.begin() + si_end);
-    shards.erase(shards.begin() + si_begin, shards.begin() + si_end);
-    sv.insert(
-      sv.begin() + si_begin,
+    extent_map_shards.erase(extent_map_shards.begin() + shard_index_begin, extent_map_shards.begin() + shard_index_end);
+    shards.erase(shards.begin() + shard_index_begin, shards.begin() + shard_index_end);
+    extent_map_shards.insert(
+      extent_map_shards.begin() + shard_index_begin,
       new_shard_info.begin(),
       new_shard_info.end());
-    shards.insert(shards.begin() + si_begin, new_shard_info.size(), Shard());
-    si_end = si_begin + new_shard_info.size();
+    shards.insert(shards.begin() + shard_index_begin, new_shard_info.size(), Shard());
+    shard_index_end = shard_index_begin + new_shard_info.size();
 
-    ceph_assert(sv.size() == shards.size());
+    ceph_assert(extent_map_shards.size() == shards.size());
 
     // note that we need to update every shard_info of shards here,
-    // as sv might have been totally re-allocated above
+    // as extent_map_shards might have been totally re-allocated above
     for (unsigned i = 0; i < shards.size(); i++) {
-      shards[i].shard_info = &sv[i];
+      shards[i].shard_info = &extent_map_shards[i];
     }
 
     // mark newly added shards as dirty
-    for (unsigned i = si_begin; i < si_end; ++i) {
+    for (unsigned i = shard_index_begin; i < shard_index_end; ++i) {
       shards[i].loaded = true;
       shards[i].dirty = true;
     }
   }
-  dout(20) << __func__ << "  fin " << sv << dendl;
+  dout(20) << __func__ << "  fin " << extent_map_shards << dendl;
   inline_bl.clear();
 
-  if (sv.empty()) {
+  if (extent_map_shards.empty()) {
     // no more shards; unspan all previously spanning blobs
-    auto p = spanning_blob_map.begin();
-    while (p != spanning_blob_map.end()) {
-      p->second->id = -1;
-      dout(30) << __func__ << " un-spanning " << *p->second << dendl;
-      p = spanning_blob_map.erase(p);
+    auto spanning_blob_it = spanning_blob_map.begin();
+    while (spanning_blob_it != spanning_blob_map.end()) {
+      spanning_blob_it->second->id = -1;
+      dout(30) << __func__ << " un-spanning " << *spanning_blob_it->second << dendl;
+      spanning_blob_it = spanning_blob_map.erase(spanning_blob_it);
     }
   } else {
     // identify new spanning blobs
@@ -3770,18 +3770,17 @@ void BlueStore::ExtentMap::reshard(
       fault_range(db, needs_reshard_end,
 		  spanning_scan_end - needs_reshard_end);
     }
-    auto sp = sv.begin() + si_begin;
-    auto esp = sv.end();
-    unsigned shard_start = sp->offset;
+    auto current_shard = extent_map_shards.begin() + shard_index_begin;
+    auto end_shard = extent_map_shards.end();
+    unsigned shard_start = current_shard->offset;
     unsigned shard_end;
-    ++sp;
-    if (sp == esp) {
+    ++current_shard;
+    if (current_shard == end_shard) {
       shard_end = OBJECT_MAX_SIZE;
     } else {
-      shard_end = sp->offset;
+      shard_end = current_shard->offset;
     }
-    Extent dummy(needs_reshard_begin);
-
+    
     bool was_too_many_blobs_check = false;
     auto too_many_blobs_threshold =
       g_conf()->bluestore_debug_too_many_blobs_threshold;
@@ -3789,36 +3788,36 @@ void BlueStore::ExtentMap::reshard(
     decltype(onode->c->onode_space.cache->dumped_onodes)::value_type* oid_slot = nullptr;
     decltype(onode->c->onode_space.cache->dumped_onodes)::value_type* oldest_slot = nullptr;
 
-    for (auto e = extent_map.lower_bound(dummy); e != extent_map.end(); ++e) {
-      if (e->logical_offset >= needs_reshard_end) {
+    for (auto extent = extent_map.lower_bound(Extent(needs_reshard_begin)); extent != extent_map.end(); ++extent) {
+      if (extent->logical_offset >= needs_reshard_end) {
 	break;
       }
-      dout(30) << " extent " << *e << dendl;
-      while (e->logical_offset >= shard_end) {
+      dout(30) << " extent " << *extent << dendl;
+      while (extent->logical_offset >= shard_end) {
 	shard_start = shard_end;
-	ceph_assert(sp != esp);
-	++sp;
-	if (sp == esp) {
+	ceph_assert(current_shard != end_shard);
+	++current_shard;
+	if (current_shard == end_shard) {
 	  shard_end = OBJECT_MAX_SIZE;
 	} else {
-	  shard_end = sp->offset;
+	  shard_end = current_shard->offset;
 	}
 	dout(30) << __func__ << "  shard 0x" << std::hex << shard_start
 		 << " to 0x" << shard_end << std::dec << dendl;
       }
 
-      if (e->blob_escapes_range(shard_start, shard_end - shard_start)) {
-	if (!e->blob->is_spanning()) {
+      if (extent->blob_escapes_range(shard_start, shard_end - shard_start)) {
+	if (!extent->blob->is_spanning()) {
 	  // We have two options: (1) split the blob into pieces at the
 	  // shard boundaries (and adjust extents accordingly), or (2)
 	  // mark it spanning.  We prefer to cut the blob if we can.  Note that
 	  // we may have to split it multiple times--potentially at every
 	  // shard boundary.
 	  bool must_span = false;
-	  BlobRef b = e->blob;
+	  BlobRef b = extent->blob;
 	  if (b->can_split()) {
-	    uint32_t bstart = e->blob_start();
-	    uint32_t bend = e->blob_end();
+	    uint32_t bstart = extent->blob_start();
+	    uint32_t bend = extent->blob_end();
 	    for (const auto& sh : shards) {
 	      if (bstart < sh.shard_info->offset &&
 		  bend > sh.shard_info->offset) {
@@ -3865,10 +3864,10 @@ void BlueStore::ExtentMap::reshard(
 	  }
 	}
       } else {
-	if (e->blob->is_spanning()) {
-	  spanning_blob_map.erase(e->blob->id);
-	  e->blob->id = -1;
-	  dout(30) << __func__ << "    un-spanning " << *e->blob << dendl;
+	if (extent->blob->is_spanning()) {
+	  spanning_blob_map.erase(extent->blob->id);
+	  extent->blob->id = -1;
+	  dout(30) << __func__ << "    un-spanning " << *extent->blob << dendl;
 	}
       }
     }

From 80d563a181fb4f26186608bbae5e9239ce19daa4 Mon Sep 17 00:00:00 2001
From: Jaya Prakash <jayaprakash@ibm.com>
Date: Wed, 16 Apr 2025 16:20:30 +0530
Subject: [PATCH 03/12] qa: Add Teuthology test for BlueStore ESB assertion
 failure

Adds a test to reproduce the !ito->is_valid() assertion in BlueStore
with bluestore_elastic_shared_blobs=true on a 2+1 EC pool using a
FIO randwrite workload (512 concurrent ops, 50G, 12,500 objects).
The test deploys a 6-OSD cluster and runs FIO for 1 hour via workunit,
failing if an OSD crashes.

Signed-off-by: Jaya Prakash <jayaprakash@ibm.com>
(cherry picked from commit 593896de02087a3669fb8caba950dfe02e96f894)
---
 qa/suites/rados/singleton/all/ec-esb-fio.yaml | 46 +++++++++
 qa/workunits/rados/ec-esb-fio.sh              | 99 +++++++++++++++++++
 2 files changed, 145 insertions(+)
 create mode 100644 qa/suites/rados/singleton/all/ec-esb-fio.yaml
 create mode 100755 qa/workunits/rados/ec-esb-fio.sh

diff --git a/qa/suites/rados/singleton/all/ec-esb-fio.yaml b/qa/suites/rados/singleton/all/ec-esb-fio.yaml
new file mode 100644
index 0000000000000..a2818d0523b94
--- /dev/null
+++ b/qa/suites/rados/singleton/all/ec-esb-fio.yaml
@@ -0,0 +1,46 @@
+meta:
+  - desc:
+      all/ec-esb-fio
+
+roles:
+- - mon.a
+  - mgr.x
+  - client.0
+- - osd.0
+  - osd.1
+- - osd.2
+  - osd.3
+- - osd.4
+  - osd.5
+openstack:
+  - volumes: # attached to each instance
+      count: 6
+      size: 20 # GB
+
+overrides:
+  ceph:
+    conf:
+      osd:
+        bluestore write v2: false
+        debug osd: 5
+        debug bluestore: 5
+        bluestore_elastic_shared_blobs: true
+        osd memory target: 939524096
+        bluestore onode segment size: 0
+
+tasks:
+- install:
+- ceph:
+    log-ignorelist:
+      - \(POOL_APP_NOT_ENABLED\)
+      - \(OSDMAP_FLAGS\)
+      - \(OSD_
+      - \(OBJECT_
+      - \(PG_
+      - \(SLOW_OPS\)
+      - overall HEALTH
+      - slow request
+- workunit:
+    clients:
+      client.0:
+        - rados/ec-esb-fio.sh
diff --git a/qa/workunits/rados/ec-esb-fio.sh b/qa/workunits/rados/ec-esb-fio.sh
new file mode 100755
index 0000000000000..32866f73166af
--- /dev/null
+++ b/qa/workunits/rados/ec-esb-fio.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+# vim: ts=8 sw=2 smarttab
+set -ex
+
+# Install FIO
+if [[ -f /etc/debian_version ]]; then
+    sudo apt-get update
+    sudo apt-get install -y git gcc make librados-dev librbd-dev zlib1g-dev libaio-dev
+    git clone -b master https://github.com/axboe/fio.git /home/ubuntu/cephtest/fio
+    cd /home/ubuntu/cephtest/fio
+    ./configure
+    make
+    sudo make install
+    cd -
+elif [[ -f /etc/redhat-release ]]; then
+    sudo yum install -y fio
+else
+    echo "Unsupported OS"
+    exit 1
+fi
+
+sleep 10
+
+ceph config set osd osd_memory_target 939524096
+ceph config set osd bluestore_onode_segment_size 0
+ceph osd erasure-code-profile set myecprofile k=2 m=1
+ceph osd pool create ecpool 16 16 erasure myecprofile
+ceph osd pool set ecpool allow_ec_overwrites true
+
+status_log() {
+    echo "Cluster status on failure:"
+    ceph -s
+    ceph health detail
+}
+
+cleanup() {
+    ceph osd pool rm ecpool ecpool --yes-i-really-really-mean-it || true
+    ceph osd erasure-code-profile rm myecprofile || true
+    rm -rf /home/ubuntu/cephtest/fio || true
+    status_log
+}
+
+trap cleanup EXIT INT TERM
+
+echo "[ec-esb-fio] Starting FIO test..."
+
+
+fio --name=test-ec-esb \
+    --ioengine=rados \
+    --pool=ecpool \
+    --clientname=admin \
+    --conf=/etc/ceph/ceph.conf \
+    --time_based=1 \
+    --runtime=1h \
+    --invalidate=0 \
+    --direct=1 \
+    --touch_objects=0 \
+    --iodepth=32 \
+    --numjobs=4 \
+    --rw=randwrite \
+    --file_service_type=pareto:0.20:0 \
+    --bssplit=4k/16:8k/10:12k/9:16k/8:20k/7:24k/7 \
+    --size=15G \
+    --nrfiles=12500 \
+    --filename_format=stress_obj.\$jobnum.\$filenum \
+    &
+
+FIO_PID=$!
+
+ceph config dump | grep bluestore_elastic_shared_blobs || true
+ceph config dump | grep bluestore_onode_segment_size || true
+ceph osd dump | grep -A 10 ecpool || true
+
+
+TIMEOUT=3600
+START_TIME=$(date +%s)
+while true; do
+    CURRENT_TIME=$(date +%s)
+    ELAPSED=$((CURRENT_TIME - START_TIME))
+    if [ $ELAPSED -ge $TIMEOUT ]; then
+        echo "Reached 1-hour timeout, stopping FIO"
+        break
+    fi
+    if ceph health detail | grep -i "osd.*down"; then
+        echo "Detected OSD down state:"
+	ceph health detail | grep -i "osd.*down"
+        echo "Cleaning up..."
+	if [[ -n "$FIO_PID" ]]; then
+	  kill -9 $FIO_PID || true
+	fi
+          exit 1
+    fi
+    ceph -s
+    ceph tell osd.0 perf dump bluestore | grep -A 2 onode || true
+    sleep 60
+done
+
+echo "[ec-esb-fio] FIO test completed, log checks to follow"
+exit 0

From b967ecd4e01d3e7ac617dda95303c710da4c07a9 Mon Sep 17 00:00:00 2001
From: Adam Kupczyk <akupczyk@ibm.com>
Date: Tue, 15 Apr 2025 08:31:49 +0000
Subject: [PATCH 04/12] os/bluestore: Debug code to make reshard fail faster

Catch reshard on producing invalid result on encode.
This makes it much easier to catch error.
Intented for teuthology testing.

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
(cherry picked from commit d27c19bc54ad2c60299f1f40cf7ebe55f3c461c8)

 Conflicts:
	src/os/bluestore/BlueStore.cc
	src/os/bluestore/BlueStore.h
 - no v2 write path
---
 src/common/options/global.yaml.in |  8 ++++++
 src/os/bluestore/BlueStore.cc     | 42 +++++++++++++++++++++++--------
 src/os/bluestore/BlueStore.h      | 14 ++++++++---
 3 files changed, 51 insertions(+), 13 deletions(-)

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index ddbd12d1f5c90..18ad377cad678 100644
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -4781,6 +4781,14 @@ options:
   desc: Preallocated buffer for inline shards
   default: 256
   with_legacy: true
+- name: bluestore_debug_extent_map_encode_check
+  type: bool
+  level: dev
+  desc: Check correctness of extents in encode_some
+  default: false
+  with_legacy: false
+  flags:
+  - startup
 - name: bluestore_cache_trim_interval
   type: float
   level: advanced
diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index bad247e322819..75889c6c04f60 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3441,21 +3441,23 @@ void BlueStore::ExtentMap::dup_esb(BlueStore* b, TransContext* txc,
 }
 
 void BlueStore::ExtentMap::update(KeyValueDB::Transaction t,
-                                  bool force)
+                                  bool just_after_reshard)
 {
   auto cct = onode->c->store->cct; //used by dout
-  dout(20) << __func__ << " " << onode->oid << (force ? " force" : "") << dendl;
+  bool do_check = onode->c->store->debug_extent_map_encode_check;
+  dout(20) << __func__ << " " << onode->oid << (just_after_reshard ? " force" : "") << dendl;
   if (onode->onode.extent_map_shards.empty()) {
     if (inline_bl.length() == 0) {
       unsigned n;
       // we need to encode inline_bl to measure encoded length
-      bool never_happen = encode_some(0, OBJECT_MAX_SIZE, inline_bl, &n);
+      bool never_happen = encode_some(0, OBJECT_MAX_SIZE, inline_bl, &n,
+        do_check, do_check && just_after_reshard);
       inline_bl.reassign_to_mempool(mempool::mempool_bluestore_inline_bl);
       ceph_assert(!never_happen);
       size_t len = inline_bl.length();
       dout(20) << __func__ << "  inline shard " << len << " bytes from " << n
 	       << " extents" << dendl;
-      if (!force && len > cct->_conf->bluestore_extent_map_shard_max_size) {
+      if (!just_after_reshard && len > cct->_conf->bluestore_extent_map_shard_max_size) {
 	request_reshard(0, OBJECT_MAX_SIZE);
 	return;
       }
@@ -3493,11 +3495,11 @@ void BlueStore::ExtentMap::update(KeyValueDB::Transaction t,
       encoded_shards.emplace_back(dirty_shard_t(&(*shard)));
       bufferlist& bl = encoded_shards.back().bl;
       if (encode_some(shard->shard_info->offset, endoff - shard->shard_info->offset,
-      		bl, &shard->extents)) {
-        if (force) {
+          bl, &shard->extents, do_check, do_check && just_after_reshard)) {
+        if (just_after_reshard) {
           _dump_extent_map<-1>(cct, *this);
           derr << __func__ << "  encode_some needs reshard" << dendl;
-          ceph_assert(!force);
+          ceph_assert(!just_after_reshard);
         }
       }
       size_t len = bl.length();
@@ -3507,7 +3509,7 @@ void BlueStore::ExtentMap::update(KeyValueDB::Transaction t,
       	 << " bytes (was " << shard->shard_info->bytes << ") from "
       	 << shard->extents << " extents" << dendl;
 
-      if (!force) {
+      if (!just_after_reshard) {
         if (len > cct->_conf->bluestore_extent_map_shard_max_size) {
           // we are big; reshard ourselves
           request_reshard(shard->shard_info->offset, endoff);
@@ -3897,7 +3899,9 @@ bool BlueStore::ExtentMap::encode_some(
   uint32_t offset,
   uint32_t length,
   bufferlist& bl,
-  unsigned *pn)
+  unsigned *pn,
+  bool complain_extent_overlap,
+  bool complain_shard_spanning)
 {
   Extent dummy(offset);
   auto start = extent_map.lower_bound(dummy);
@@ -3910,10 +3914,20 @@ bool BlueStore::ExtentMap::encode_some(
   unsigned n = 0;
   size_t bound = 0;
   bool must_reshard = false;
+  uint32_t prev_offset_end = 0;
   for (auto p = start;
        p != extent_map.end() && p->logical_offset < end;
        ++p, ++n) {
     ceph_assert(p->logical_offset >= offset);
+    if (complain_extent_overlap) {
+      if (p->logical_offset < prev_offset_end) {
+        using P = BlueStore::printer;
+        dout(-1) << __func__ << " extents overlap: " << std::endl
+                 << onode->print(P::NICK + P::SDISK + P::SUSE + P::SBUF) << dendl;
+        ceph_abort();
+      }
+      prev_offset_end = p->logical_end();
+    }
     p->blob->last_encoded_id = -1;
     if (!p->blob->is_spanning() && p->blob_escapes_range(offset, length)) {
       dout(30) << __func__ << " 0x" << std::hex << offset << "~" << length
@@ -3956,6 +3970,14 @@ bool BlueStore::ExtentMap::encode_some(
 	 p != extent_map.end() && p->logical_offset < end;
 	 ++p, ++n) {
       unsigned blobid;
+      if (complain_shard_spanning) {
+        if (p->logical_end() > end) {
+          using P = BlueStore::printer;
+          dout(-1) << __func__ << " extent spans shard after reshard " << ": " << std::endl
+            << onode->print(P::NICK + P::SDISK + P::SUSE + P::SBUF) << dendl;
+          ceph_abort();
+        }
+      }
       bool include_blob = false;
       if (p->blob->is_spanning()) {
 	blobid = p->blob->id << BLOBID_SHIFT_BITS;
@@ -9256,7 +9278,7 @@ int BlueStore::_mount()
       return r;
     }
   }
-
+  debug_extent_map_encode_check = cct->_conf.get_val<bool>("bluestore_debug_extent_map_encode_check");
   _kv_only = false;
   if (cct->_conf->bluestore_fsck_on_mount) {
     int rc = fsck(cct->_conf->bluestore_fsck_on_mount_deep);
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 4b2ef759c90d3..8eaf77b8a36bf 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1025,8 +1025,12 @@ class BlueStore : public ObjectStore,
 
     void dump(ceph::Formatter* f) const;
 
-    bool encode_some(uint32_t offset, uint32_t length, ceph::buffer::list& bl,
-		     unsigned *pn);
+    bool encode_some(
+      uint32_t offset, uint32_t length, ceph::buffer::list& bl, unsigned *pn,
+      bool complain_extent_overlap, //verification; in debug mode assert if extents overlap
+      bool complain_shard_spanning  //verification; in debug mode assert if extent spans shards;
+                                    //must be used only on encode after reshard
+    );
 
     class ExtentDecoder {
       uint64_t pos = 0;
@@ -1083,7 +1087,10 @@ class BlueStore : public ObjectStore,
       return p->second;
     }
 
-    void update(KeyValueDB::Transaction t, bool force);
+    void update(
+      KeyValueDB::Transaction t,
+      bool just_after_reshard //true to indicate that update should now respect shard boundaries
+    );                        //as no further resharding will be done
     decltype(BlueStore::Blob::id) allocate_spanning_blob_id();
     void reshard(
       KeyValueDB *db,
@@ -2400,6 +2407,7 @@ class BlueStore : public ObjectStore,
 		std::numeric_limits<decltype(min_alloc_size)>::digits,
 		"not enough bits for min_alloc_size");
   bool elastic_shared_blobs = false; ///< use smart ExtentMap::dup to reduce shared blob count
+  bool debug_extent_map_encode_check = false;
 
   enum {
     // Please preserve the order since it's DB persistent

From 9ebb98e50ca84293d5131432ff00ca0a623a10cf Mon Sep 17 00:00:00 2001
From: Adam Kupczyk <akupczyk@ibm.com>
Date: Thu, 5 Jun 2025 08:09:48 +0000
Subject: [PATCH 05/12] qa, ec-esb-fio: Add flag for easier replication

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
(cherry picked from commit 3ce9d033cb674be42271c79a33315fb0559e94c3)
---
 qa/suites/rados/singleton/all/ec-esb-fio.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/suites/rados/singleton/all/ec-esb-fio.yaml b/qa/suites/rados/singleton/all/ec-esb-fio.yaml
index a2818d0523b94..049e9cfc76aad 100644
--- a/qa/suites/rados/singleton/all/ec-esb-fio.yaml
+++ b/qa/suites/rados/singleton/all/ec-esb-fio.yaml
@@ -27,6 +27,7 @@ overrides:
         bluestore_elastic_shared_blobs: true
         osd memory target: 939524096
         bluestore onode segment size: 0
+        bluestore debug extent map encode check: true
 
 tasks:
 - install:

From cd1c4ebe5c65206dcb676dbcf7ffb5dba009b298 Mon Sep 17 00:00:00 2001
From: Jaya Prakash <jayaprakash@ibm.com>
Date: Wed, 12 Mar 2025 19:01:52 +0000
Subject: [PATCH 06/12] os/bluestore: Refactor ExtentMap::reshard() into
 separate decision and action phases

Refactored ExtentMap::reshard() by splitting it into two distinct functions:
- reshard_decision(): Determines the resharding plan by analyzing shard distribution, extent sizes, and spanning blob ranges
- reshard_action(): Applies the resharding plan, handling faulting, key removals, and shard updates

Signed-off-by: Jaya Prakash <jayaprakash@ibm.com>
(cherry picked from commit 44f93a610222338f7e763b08fc6c52ac3e82473e)
(cherry picked from commit f8a9e229deb434b493887ac3c6c387fe599a24c0)

 Conflicts:
	src/os/bluestore/BlueStore.cc
---
 src/os/bluestore/BlueStore.cc | 77 +++++++++++++++++++++++++----------
 src/os/bluestore/BlueStore.h  | 17 ++++++++
 2 files changed, 72 insertions(+), 22 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 75889c6c04f60..6e151b3350dc6 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3582,10 +3582,9 @@ bid_t BlueStore::ExtentMap::allocate_spanning_blob_id()
   ceph_abort_msg("no available blob id");
 }
 
-void BlueStore::ExtentMap::reshard(
-  KeyValueDB *db,
-  KeyValueDB::Transaction t)
-{
+BlueStore::ExtentMap::ReshardPlan
+BlueStore::ExtentMap::reshard_decision() {
+  ReshardPlan plan;
   auto cct = onode->c->store->cct; // used by dout
 
   dout(10) << __func__ << " 0x[" << std::hex << needs_reshard_begin << ","
@@ -3624,7 +3623,6 @@ void BlueStore::ExtentMap::reshard(
     needs_reshard_end = OBJECT_MAX_SIZE;
   }
 
-  fault_range(db, needs_reshard_begin, (needs_reshard_end - needs_reshard_begin));
 
   // we may need to fault in a larger interval later must have all
   // referring extents for spanning blobs loaded in order to have
@@ -3632,17 +3630,6 @@ void BlueStore::ExtentMap::reshard(
   uint32_t spanning_scan_begin = needs_reshard_begin;
   uint32_t spanning_scan_end = needs_reshard_end;
 
-  // remove old keys
-  string key;
-  for (unsigned i = shard_index_begin; i < shard_index_end; ++i) {
-    generate_extent_shard_key_and_apply(
-      onode->key, shards[i].shard_info->offset, &key,
-      [&](const string& final_key) {
-	t->rmkey(PREFIX_OBJ, final_key);
-      }
-      );
-  }
-
   // calculate average extent size
   unsigned bytes = 0;
   unsigned extents = 0;
@@ -3720,6 +3707,47 @@ void BlueStore::ExtentMap::reshard(
   auto& extent_map_shards = onode->onode.extent_map_shards;
   dout(20) << __func__ << "  new " << new_shard_info << dendl;
   dout(20) << __func__ << "  old " << extent_map_shards << dendl;
+
+  plan.shard_index_begin = shard_index_begin;
+  plan.shard_index_end = shard_index_end;
+  plan.spanning_scan_begin = spanning_scan_begin;
+  plan.spanning_scan_end = spanning_scan_end;
+  plan.new_shard_info = std::move(new_shard_info);
+  return plan;
+}
+
+
+void BlueStore::ExtentMap::reshard_action(
+  ReshardPlan& plan,
+  KeyValueDB *db,
+  KeyValueDB::Transaction t) {
+  auto cct = onode->c->store->cct; // For configuration and logging
+
+  std::vector<bluestore_onode_t::shard_info> new_shard_info = plan.new_shard_info;
+  unsigned shard_index_begin = plan.shard_index_begin;
+  unsigned shard_index_end = plan.shard_index_end;
+  uint32_t spanning_scan_begin = plan.spanning_scan_begin;
+  uint32_t spanning_scan_end = plan.spanning_scan_end;
+
+  dout(20) << __func__ << " applying plan with shards [" << shard_index_begin << ","
+           << shard_index_end << ")" << dendl;
+
+  // Fault the range
+  fault_range(db, needs_reshard_begin, (needs_reshard_end - needs_reshard_begin));
+
+  // Remove old shard keys
+  string key;
+  for (unsigned i = shard_index_begin; i < shard_index_end; ++i) {
+    generate_extent_shard_key_and_apply(
+      onode->key, shards[i].shard_info->offset, &key,
+      [&](const string& final_key) {
+	t->rmkey(PREFIX_OBJ, final_key);
+      }
+      );
+  }
+
+  // Update extent_map_shards and shards
+  auto& extent_map_shards = onode->onode.extent_map_shards;
   if (extent_map_shards.empty()) {
     // no old shards to keep
     extent_map_shards.swap(new_shard_info);
@@ -3895,6 +3923,13 @@ void BlueStore::ExtentMap::reshard(
   clear_needs_reshard();
 }
 
+void BlueStore::ExtentMap::reshard(
+  KeyValueDB *db,
+  KeyValueDB::Transaction t) {
+  auto plan = reshard_decision();
+  reshard_action(plan, db, t);
+}
+
 bool BlueStore::ExtentMap::encode_some(
   uint32_t offset,
   uint32_t length,
@@ -3921,9 +3956,8 @@ bool BlueStore::ExtentMap::encode_some(
     ceph_assert(p->logical_offset >= offset);
     if (complain_extent_overlap) {
       if (p->logical_offset < prev_offset_end) {
-        using P = BlueStore::printer;
-        dout(-1) << __func__ << " extents overlap: " << std::endl
-                 << onode->print(P::NICK + P::SDISK + P::SUSE + P::SBUF) << dendl;
+        dout(-1) << __func__ << " extents overlap: " << dendl;
+        _dump_onode<-1>(onode->c->store->cct, *onode);
         ceph_abort();
       }
       prev_offset_end = p->logical_end();
@@ -3972,9 +4006,8 @@ bool BlueStore::ExtentMap::encode_some(
       unsigned blobid;
       if (complain_shard_spanning) {
         if (p->logical_end() > end) {
-          using P = BlueStore::printer;
-          dout(-1) << __func__ << " extent spans shard after reshard " << ": " << std::endl
-            << onode->print(P::NICK + P::SDISK + P::SUSE + P::SBUF) << dendl;
+          dout(-1) << __func__ << " extent spans shard after reshard " << ": " << dendl;
+          _dump_onode<-1>(onode->c->store->cct, *onode);
           ceph_abort();
         }
       }
diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h
index 8eaf77b8a36bf..f97ac21ae4392 100644
--- a/src/os/bluestore/BlueStore.h
+++ b/src/os/bluestore/BlueStore.h
@@ -1092,6 +1092,23 @@ class BlueStore : public ObjectStore,
       bool just_after_reshard //true to indicate that update should now respect shard boundaries
     );                        //as no further resharding will be done
     decltype(BlueStore::Blob::id) allocate_spanning_blob_id();
+
+    struct ReshardPlan {
+      std::vector<bluestore_onode_t::shard_info> new_shard_info;
+      unsigned shard_index_begin;
+      unsigned shard_index_end;
+      uint32_t spanning_scan_begin;
+      uint32_t spanning_scan_end;
+    };
+
+    ReshardPlan reshard_decision();
+
+    void reshard_action(
+      ReshardPlan& plan,
+      KeyValueDB *db,
+      KeyValueDB::Transaction t);
+
+
     void reshard(
       KeyValueDB *db,
       KeyValueDB::Transaction t);

From 22a085dcd73f1f54e5b73ba7c35d2a68099056de Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Wed, 13 Aug 2025 19:29:52 +0300
Subject: [PATCH 07/12] test/bluestore_types: reproduce extent spanning over
 shard boundary.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
(cherry picked from commit a7f082963827ed9754c6bc6efeca4657af430132)
(cherry picked from commit 718b103d872fcc4a3b8f7c6d50b52090e4fc0b72)

 Conflicts:
	src/test/objectstore/test_bluestore_types.cc
---
 src/test/objectstore/test_bluestore_types.cc | 96 ++++++++++++++++++++
 1 file changed, 96 insertions(+)

diff --git a/src/test/objectstore/test_bluestore_types.cc b/src/test/objectstore/test_bluestore_types.cc
index db0ee38089119..da3e5860b94aa 100644
--- a/src/test/objectstore/test_bluestore_types.cc
+++ b/src/test/objectstore/test_bluestore_types.cc
@@ -1263,6 +1263,102 @@ TEST(ExtentMap, compress_extent_map)
   ASSERT_EQ(6u, em.extent_map.size());
 }
 
+TEST(ExtentMap, reshard_failure) {
+  BlueStore store(g_ceph_context, "", 4096);
+  BlueStore::OnodeCacheShard* oc =
+    BlueStore::OnodeCacheShard::create(g_ceph_context, "lru", NULL);
+  BlueStore::BufferCacheShard* bc =
+    BlueStore::BufferCacheShard::create(g_ceph_context, "lru", NULL);
+
+  auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
+  BlueStore::Onode onode(coll.get(), ghobject_t(), "");
+  // csum block size = 1K, full blob length covered with csum
+  size_t csum_order = 12;
+  int csum_type = Checksummer::CSUM_CRC32C;
+
+  auto make_blob = [&](uint64_t o1,
+    uint64_t l1,
+    uint64_t o2,
+    uint64_t l2,
+    uint64_t o3,
+    uint64_t l3,
+    uint64_t o4,
+    uint64_t l4) {
+      BlueStore::BlobRef b1(coll->new_blob());
+      b1->dirty_blob().allocated_test(bluestore_pextent_t(o1, l1));
+      b1->dirty_blob().allocated_test(bluestore_pextent_t(o2, l2));
+      b1->dirty_blob().allocated_test(bluestore_pextent_t(o3, l3));
+      b1->dirty_blob().allocated_test(bluestore_pextent_t(o4, l4));
+      b1->dirty_blob().init_csum(csum_type, csum_order, l1 + l2 + l3 + l4);
+      return b1;
+    };
+  {
+    // [0(0x2000)~0x1000, 0x1000(0x4000)~0x2000, 0x3000(0x7000)~0x3000, 0xa000(0xb000)~0x4000]
+    // (note: offsets above are in the following format: blob_offset(lba))
+    uint64_t o1 = 0x2000;
+    uint64_t l1 = 0x1000;
+    uint64_t o2 = 0x4000;
+    uint64_t l2 = 0x2000;
+    uint64_t o3 = 0x7000;
+    uint64_t l3 = 0x3000;
+    uint64_t o4 = 0xb000;
+    uint64_t l4 = 0x4000;
+
+    size_t blob_len = l1 + l2 + l3 + l4;
+
+    BlueStore::ExtentMap& em = onode.extent_map;
+    BlueStore::BlobRef lb = make_blob(o1, l1, o2, l2, o3, l3, o4, l4);
+    BlueStore::BlobRef rb = make_blob(o1 + blob_len, l1,
+                                      o2 + blob_len, l2,
+                                      o3 + blob_len, l3,
+                                      o4 + blob_len, l4);
+
+    size_t expected_spanning_blobs = 0;
+    size_t expected_extents = 4;
+    em.set_lextent(coll, 0, 0, blob_len, lb, nullptr);
+    em.set_lextent(coll, blob_len, 0, blob_len, rb, nullptr);
+
+    // make blob unsplittable which causes
+    // reshard() to make it spanning.
+    // Relevant extent must be splitted anyway but this didn't happen
+    // in the original reshard() implementaion.
+    if(1) {
+      lb->dirty_blob().set_flag(bluestore_blob_t::FLAG_HAS_UNUSED);
+      ASSERT_FALSE(lb->can_split());
+      expected_spanning_blobs = 1;
+    }
+    em.request_reshard(0, blob_len * 2);
+
+    BlueStore::ExtentMap::ReshardPlan rp;
+    rp.new_shard_info.emplace_back(0, 0);
+    // shard before 3rd blob in the first extent
+    rp.new_shard_info.emplace_back(l1 + l2, 0);
+    // and shard before 3rd blob in the second extent
+    rp.new_shard_info.emplace_back(blob_len + l1 + l2, 0);
+    rp.shard_index_begin = 0;
+    rp.shard_index_end = 2;
+    rp.spanning_scan_begin = 0; // doesn't matter
+    rp.spanning_scan_end =  0; // doesn't matter
+
+    em.reshard_action(rp, nullptr, nullptr);
+    
+    ASSERT_EQ(em.shards.size(), 3);
+    ASSERT_EQ(em.spanning_blob_map.size(), expected_spanning_blobs);
+    EXPECT_EQ(em.extent_map.size(), expected_extents);
+    if (false) {
+      // make sure hat encode_some detects that extent spans over shard boundary
+      bufferlist bl;
+      size_t prev_shard = 0;
+      for (auto& s : em.shards) {
+        if (s.shard_info->offset > 0) {
+          em.encode_some(prev_shard, s.shard_info->offset, bl, nullptr, true, true);
+          prev_shard = s.shard_info->offset;
+        }
+      }
+    }
+  }
+}
+
 class ExtentMapFixture : virtual public ::testing::Test {
 
 public:

From dafd164a33eb51d730452e8c40e06817a986c1ce Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Tue, 2 Sep 2025 16:09:54 +0300
Subject: [PATCH 08/12] os/bluestore: enable dummy db/txc in
 ExtentMap::reshard_action.

Intended for UT simplication only.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
(cherry picked from commit 9f84eb851d512515e57dc8f7ed093bf1673cee24)
---
 src/os/bluestore/BlueStore.cc | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 6e151b3350dc6..45da3e4052305 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3733,11 +3733,13 @@ void BlueStore::ExtentMap::reshard_action(
            << shard_index_end << ")" << dendl;
 
   // Fault the range
-  fault_range(db, needs_reshard_begin, (needs_reshard_end - needs_reshard_begin));
+  if (db) {
+    fault_range(db, needs_reshard_begin, (needs_reshard_end - needs_reshard_begin));
+  }
 
   // Remove old shard keys
   string key;
-  for (unsigned i = shard_index_begin; i < shard_index_end; ++i) {
+  for (unsigned i = shard_index_begin; t && i < shard_index_end; ++i) {
     generate_extent_shard_key_and_apply(
       onode->key, shards[i].shard_info->offset, &key,
       [&](const string& final_key) {
@@ -3792,13 +3794,15 @@ void BlueStore::ExtentMap::reshard_action(
     // identify new spanning blobs
     dout(20) << __func__ << " checking spanning blobs 0x[" << std::hex
 	     << spanning_scan_begin << "," << spanning_scan_end << ")" << dendl;
-    if (spanning_scan_begin < needs_reshard_begin) {
-      fault_range(db, spanning_scan_begin,
-		  needs_reshard_begin - spanning_scan_begin);
-    }
-    if (spanning_scan_end > needs_reshard_end) {
-      fault_range(db, needs_reshard_end,
-		  spanning_scan_end - needs_reshard_end);
+    if (db) {
+      if (spanning_scan_begin < needs_reshard_begin) {
+        fault_range(db, spanning_scan_begin,
+		    needs_reshard_begin - spanning_scan_begin);
+      }
+      if (spanning_scan_end > needs_reshard_end) {
+        fault_range(db, needs_reshard_end,
+		       spanning_scan_end - needs_reshard_end);
+      }
     }
     auto current_shard = extent_map_shards.begin() + shard_index_begin;
     auto end_shard = extent_map_shards.end();

From c081b9eae7a7082dbc86b8d50a7044bc085729c5 Mon Sep 17 00:00:00 2001
From: Adam Kupczyk <akupczyk@ibm.com>
Date: Tue, 15 Apr 2025 08:34:38 +0000
Subject: [PATCH 09/12] os/bluestore: Fix reshard on spanning blobs

Make sure that spanning blobs are not allowed to have extents crossing
shard boundary.

Partially fixes: https://tracker.ceph.com/issues/70390

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
(cherry picked from commit ce05ade7980397cad48e8fc78bebc839c76ba327)
(cherry picked from commit 0f5e240e49a3a16b611fc80cf6ca06cfd8b1b303)
---
 src/os/bluestore/BlueStore.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 45da3e4052305..95ee6fcc00cb9 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3972,6 +3972,11 @@ bool BlueStore::ExtentMap::encode_some(
 	       << std::dec << " hit new spanning blob " << *p << dendl;
       request_reshard(p->blob_start(), p->blob_end());
       must_reshard = true;
+    } else if (p->blob->is_spanning() && p->logical_end() > end) {
+      dout(30) << __func__ << std::hex << offset << "~" << length
+               << std::dec << " extent stands out " << *p << dendl;
+      request_reshard(p->blob_start(), p->blob_end());
+      must_reshard = true;
     }
     if (!must_reshard) {
       denc_varint(0, bound); // blobid

From 37248077f4550c85258b98f184193101a02dae0e Mon Sep 17 00:00:00 2001
From: Adam Kupczyk <akupczyk@ibm.com>
Date: Tue, 15 Apr 2025 08:37:25 +0000
Subject: [PATCH 10/12] os/bluestore: Fix dirty_range in BlueStore::_do_remove

dirty_range used to have length = 1 byte.
This is good if whole extent is inside shard.
But this has proven not to be the case.
dirty_range(offset, length) is slower only when it crosses shard.

Partially fixes: https://tracker.ceph.com/issues/70390

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>
(cherry picked from commit 4f566eaf6c4646e513ea6747c7df17383d8716e2)
(cherry picked from commit d6c61326a125f8bd278ec1c656d673e53edf47cd)
---
 src/os/bluestore/BlueStore.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 95ee6fcc00cb9..b115accf49d0a 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -17743,7 +17743,7 @@ int BlueStore::_do_remove(
       bluestore_blob_t& blob = e.blob->dirty_blob();
       blob.clear_flag(bluestore_blob_t::FLAG_SHARED);
       e.blob->get_dirty_shared_blob() = nullptr;
-      h->extent_map.dirty_range(e.logical_offset, 1);
+      h->extent_map.dirty_range(e.logical_offset, e.length);
     }
   }
   txc->write_onode(h);

From 0611ed6d8980b8b8839bb0d6c7af07b598fcc089 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Fri, 15 Aug 2025 13:15:07 +0300
Subject: [PATCH 11/12] os/bluestore: enforce extent split on shard boundary

Partially fixes: https://tracker.ceph.com/issues/70390

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
(cherry picked from commit 5beee2ad46cfeb8ffc70d106c1180f531e455e3e)
---
 src/os/bluestore/BlueStore.cc | 106 ++++++++++++++++++++--------------
 1 file changed, 62 insertions(+), 44 deletions(-)

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index b115accf49d0a..67127b048fd50 100644
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -3826,7 +3826,7 @@ void BlueStore::ExtentMap::reshard_action(
       if (extent->logical_offset >= needs_reshard_end) {
 	break;
       }
-      dout(30) << " extent " << *extent << dendl;
+      dout(30) << __func__ << " extent " << *extent << dendl;
       while (extent->logical_offset >= shard_end) {
 	shard_start = shard_end;
 	ceph_assert(current_shard != end_shard);
@@ -3841,58 +3841,78 @@ void BlueStore::ExtentMap::reshard_action(
       }
 
       if (extent->blob_escapes_range(shard_start, shard_end - shard_start)) {
-	if (!extent->blob->is_spanning()) {
+	BlobRef b = extent->blob;
+	uint32_t bstart = extent->blob_start();
+	uint32_t bend = extent->blob_end();
+	if (!b->is_spanning()) {
 	  // We have two options: (1) split the blob into pieces at the
 	  // shard boundaries (and adjust extents accordingly), or (2)
 	  // mark it spanning.  We prefer to cut the blob if we can.  Note that
 	  // we may have to split it multiple times--potentially at every
 	  // shard boundary.
-	  bool must_span = false;
-	  BlobRef b = extent->blob;
+	  auto _make_spanning = [&](BlobRef& b) {
+	    auto bid = allocate_spanning_blob_id();
+	    b->id = bid;
+	    spanning_blob_map[b->id] = b;
+	    dout(20) << __func__ << "    adding spanning " << *b << dendl;
+	    if (!was_too_many_blobs_check &&
+	      too_many_blobs_threshold &&
+	      spanning_blob_map.size() >= size_t(too_many_blobs_threshold)) {
+
+	      was_too_many_blobs_check = true;
+	      for (size_t i = 0; i < dumped_onodes.size(); ++i) {
+		if (dumped_onodes[i].first == onode->oid) {
+		  oid_slot = &dumped_onodes[i];
+		  break;
+		}
+		if (!oldest_slot || (oldest_slot &&
+		  dumped_onodes[i].second < oldest_slot->second)) {
+		  oldest_slot = &dumped_onodes[i];
+		}
+	      }
+	    }
+	  };
 	  if (b->can_split()) {
-	    uint32_t bstart = extent->blob_start();
-	    uint32_t bend = extent->blob_end();
+	    auto bstart1 = bstart;
 	    for (const auto& sh : shards) {
-	      if (bstart < sh.shard_info->offset &&
+	      if (bstart1 < sh.shard_info->offset &&
 		  bend > sh.shard_info->offset) {
-		uint32_t blob_offset = sh.shard_info->offset - bstart;
+		uint32_t blob_offset = sh.shard_info->offset - bstart1;
 		if (b->can_split_at(blob_offset)) {
 		  dout(20) << __func__ << "    splitting blob, bstart 0x"
-			   << std::hex << bstart << " blob_offset 0x"
+			   << std::hex << bstart1 << " blob_offset 0x"
 			   << blob_offset << std::dec << " " << *b << dendl;
 		  b = split_blob(b, blob_offset, sh.shard_info->offset);
 		  // switch b to the new right-hand side, in case it
 		  // *also* has to get split.
-		  bstart += blob_offset;
+		  bstart1 = sh.shard_info->offset;
 		  onode->c->store->logger->inc(l_bluestore_blob_split);
 		} else {
-		  must_span = true;
+		  _make_spanning(b);
 		  break;
 		}
 	      }
 	    }
 	  } else {
-	    must_span = true;
+	    _make_spanning(b);
 	  }
-	  if (must_span) {
-            auto bid = allocate_spanning_blob_id();
-            b->id = bid;
-	    spanning_blob_map[b->id] = b;
-	    dout(20) << __func__ << "    adding spanning " << *b << dendl;
-	    if (!was_too_many_blobs_check &&
-	      too_many_blobs_threshold &&
-	      spanning_blob_map.size() >= size_t(too_many_blobs_threshold)) {
-
-	      was_too_many_blobs_check = true;
-	      for (size_t i = 0; i < dumped_onodes.size(); ++i) {
-		if (dumped_onodes[i].first == onode->oid) {
-		  oid_slot = &dumped_onodes[i];
-		  break;
-		}
-		if (!oldest_slot || (oldest_slot &&
-		    dumped_onodes[i].second < oldest_slot->second)) {
-		  oldest_slot = &dumped_onodes[i];
-		}
+	} // if (!extent->blob->is_spanning())
+	// Make sure extent with a spanning blob doesn't span over shard boundary
+	if (extent->blob->is_spanning()) {
+	  BlobRef b = extent->blob;
+	  uint32_t bstart = extent->blob_start();
+	  for (const auto& sh : shards) {
+	    if (bstart < sh.shard_info->offset && bend > sh.shard_info->offset) {
+	      uint32_t blob_offset = sh.shard_info->offset - bstart;
+	      auto pos = sh.shard_info->offset;
+	      if (extent->logical_offset < pos && extent->logical_end() > pos) {
+		// split extent
+		size_t left = pos - extent->logical_offset;
+		Extent* ne = new Extent(pos, blob_offset, extent->length - left, b);
+		extent_map.insert(*ne);
+		extent->length = left;
+		dout(20) << __func__ << "  split " << *extent << dendl;
+		dout(20) << __func__ << "     to " << *ne << dendl;
 	      }
 	    }
 	  }
@@ -3901,7 +3921,7 @@ void BlueStore::ExtentMap::reshard_action(
 	if (extent->blob->is_spanning()) {
 	  spanning_blob_map.erase(extent->blob->id);
 	  extent->blob->id = -1;
-	  dout(30) << __func__ << "    un-spanning " << *extent->blob << dendl;
+	  dout(20) << __func__ << "    un-spanning " << *extent->blob << dendl;
 	}
       }
     }
@@ -3952,7 +3972,6 @@ bool BlueStore::ExtentMap::encode_some(
 
   unsigned n = 0;
   size_t bound = 0;
-  bool must_reshard = false;
   uint32_t prev_offset_end = 0;
   for (auto p = start;
        p != extent_map.end() && p->logical_offset < end;
@@ -3960,25 +3979,27 @@ bool BlueStore::ExtentMap::encode_some(
     ceph_assert(p->logical_offset >= offset);
     if (complain_extent_overlap) {
       if (p->logical_offset < prev_offset_end) {
-        dout(-1) << __func__ << " extents overlap: " << dendl;
+        dout(-1) << __func__ << " extents overlap: "
+                 << std::hex << offset <<"~" << length
+                 << " " << p->logical_offset <<"~" << p->length
+                 << std::dec << dendl;
         _dump_onode<-1>(onode->c->store->cct, *onode);
-        ceph_abort();
+	ceph_abort_msg("extents overlaps");
       }
       prev_offset_end = p->logical_end();
     }
     p->blob->last_encoded_id = -1;
     if (!p->blob->is_spanning() && p->blob_escapes_range(offset, length)) {
-      dout(30) << __func__ << " 0x" << std::hex << offset << "~" << length
+      dout(20) << __func__ << " 0x" << std::hex << offset << "~" << length
 	       << std::dec << " hit new spanning blob " << *p << dendl;
       request_reshard(p->blob_start(), p->blob_end());
-      must_reshard = true;
+      return true;
     } else if (p->blob->is_spanning() && p->logical_end() > end) {
-      dout(30) << __func__ << std::hex << offset << "~" << length
+      dout(20) << __func__ << std::hex << offset << "~" << length
                << std::dec << " extent stands out " << *p << dendl;
       request_reshard(p->blob_start(), p->blob_end());
-      must_reshard = true;
-    }
-    if (!must_reshard) {
+      return true;
+    } else {
       denc_varint(0, bound); // blobid
       denc_varint(0, bound); // logical_offset
       denc_varint(0, bound); // len
@@ -3991,9 +4012,6 @@ bool BlueStore::ExtentMap::encode_some(
         false);
     }
   }
-  if (must_reshard) {
-    return true;
-  }
 
   denc(struct_v, bound);
   denc_varint(0, bound); // number of extents

From c5b5d51a20bbec3ea293fbb1f3412d8f77f50e35 Mon Sep 17 00:00:00 2001
From: Igor Fedotov <igor.fedotov@croit.io>
Date: Mon, 10 Nov 2025 16:57:20 +0300
Subject: [PATCH 12/12] os/bluestore: introduce explicit ctor for shard_info

This fixes building at github build bot.

Signed-off-by: Igor Fedotov <igor.fedotov@croit.io>
---
 src/os/bluestore/bluestore_types.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/os/bluestore/bluestore_types.h b/src/os/bluestore/bluestore_types.h
index 3fa9e4e86b487..75cf2ab1b9c7f 100644
--- a/src/os/bluestore/bluestore_types.h
+++ b/src/os/bluestore/bluestore_types.h
@@ -1015,6 +1015,8 @@ struct bluestore_onode_t {
   struct shard_info {
     uint32_t offset = 0;  ///< logical offset for start of shard
     uint32_t bytes = 0;   ///< encoded bytes
+    shard_info() {}
+    shard_info(uint32_t o, uint32_t b) : offset(o), bytes(b) {}
     DENC(shard_info, v, p) {
       denc_varint(v.offset, p);
       denc_varint(v.bytes, p);