diff --git a/src/test/objectstore/CMakeLists.txt b/src/test/objectstore/CMakeLists.txt index 3294616b9b2..4d1cac3b4fe 100644 --- a/src/test/objectstore/CMakeLists.txt +++ b/src/test/objectstore/CMakeLists.txt @@ -133,6 +133,78 @@ if(WITH_BLUESTORE) add_ceph_unittest(unittest_bluefs) target_link_libraries(unittest_bluefs os global) + ### ==PKGBUILD PATCH BOUNDARY== ### + # unittest_bluefs_large_write_1 + add_executable(unittest_bluefs_large_write_1 + test_bluefs_large_write_1.cc + ) + add_ceph_unittest(unittest_bluefs_large_write_1) + target_link_libraries(unittest_bluefs_large_write_1 os global) + + # unittest_bluefs_large_write_2 + add_executable(unittest_bluefs_large_write_2 + test_bluefs_large_write_2.cc + ) + add_ceph_unittest(unittest_bluefs_large_write_2) + target_link_libraries(unittest_bluefs_large_write_2 os global) + + # unittest_bluefs_flush_1 + add_executable(unittest_bluefs_flush_1 + test_bluefs_flush_1.cc + ) + add_ceph_unittest(unittest_bluefs_flush_1) + target_link_libraries(unittest_bluefs_flush_1 os global) + + # unittest_bluefs_flush_2 + add_executable(unittest_bluefs_flush_2 + test_bluefs_flush_2.cc + ) + add_ceph_unittest(unittest_bluefs_flush_2) + target_link_libraries(unittest_bluefs_flush_2 os global) + + # unittest_bluefs_flush_3 + add_executable(unittest_bluefs_flush_3 + test_bluefs_flush_3.cc + ) + add_ceph_unittest(unittest_bluefs_flush_3) + target_link_libraries(unittest_bluefs_flush_3 os global) + + # unittest_bluefs_compact_sync + add_executable(unittest_bluefs_compact_sync + test_bluefs_compact_sync.cc + ) + add_ceph_unittest(unittest_bluefs_compact_sync) + target_link_libraries(unittest_bluefs_compact_sync os global) + + # unittest_bluefs_compact_async + add_executable(unittest_bluefs_compact_async + test_bluefs_compact_async.cc + ) + add_ceph_unittest(unittest_bluefs_compact_async) + target_link_libraries(unittest_bluefs_compact_async os global) + + # unittest_bluefs_replay + add_executable(unittest_bluefs_replay + test_bluefs_replay.cc + ) + add_ceph_unittest(unittest_bluefs_replay) + target_link_libraries(unittest_bluefs_replay os global) + + # unittest_bluefs_replay_growth + add_executable(unittest_bluefs_replay_growth + test_bluefs_replay_growth.cc + ) + add_ceph_unittest(unittest_bluefs_replay_growth) + target_link_libraries(unittest_bluefs_replay_growth os global) + + # unittest_bluefs_regression + add_executable(unittest_bluefs_regression + test_bluefs_regression.cc + ) + add_ceph_unittest(unittest_bluefs_regression) + target_link_libraries(unittest_bluefs_regression os global) + ### ==PKGBUILD PATCH BOUNDARY== ### + # unittest_bluestore_types add_executable(unittest_bluestore_types test_bluestore_types.cc diff --git a/src/test/objectstore/test_bluefs.cc b/src/test/objectstore/test_bluefs.cc index 4f77d8597ae..04b25842bcd 100644 --- a/src/test/objectstore/test_bluefs.cc +++ b/src/test/objectstore/test_bluefs.cc @@ -172,6 +172,7 @@ TEST(BlueFS, small_appends) { fs.umount(); } +/* TEST(BlueFS, very_large_write) { // we'll write a ~5G file, so allocate more than that for the whole fs uint64_t size = 1048576 * 1024 * 6ull; @@ -245,7 +246,9 @@ TEST(BlueFS, very_large_write) { g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old)); } +*/ +/* TEST(BlueFS, very_large_write2) { // we'll write a ~5G file, so allocate more than that for the whole fs uint64_t size_full = 1048576 * 1024 * 6ull; @@ -298,6 +301,7 @@ TEST(BlueFS, very_large_write2) { g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old)); } +*/ #define ALLOC_SIZE 4096 @@ -411,6 +415,7 @@ void join_all(std::vector& v) #define NUM_SINGLE_FILE_WRITERS 1 #define NUM_MULTIPLE_FILE_WRITERS 2 +/* TEST(BlueFS, test_flush_1) { uint64_t size = 1048576 * 128; TempBdev bdev{size}; @@ -444,7 +449,9 @@ TEST(BlueFS, test_flush_1) { } fs.umount(); } +*/ +/* TEST(BlueFS, test_flush_2) { uint64_t size = 1048576 * 256; TempBdev bdev{size}; @@ -471,7 +478,9 @@ TEST(BlueFS, test_flush_2) { } fs.umount(); } +*/ +/* TEST(BlueFS, test_flush_3) { uint64_t size = 1048576 * 256; TempBdev bdev{size}; @@ -505,7 +514,9 @@ TEST(BlueFS, test_flush_3) { } fs.umount(); } +*/ +/* TEST(BlueFS, test_simple_compaction_sync) { g_ceph_context->_conf.set_val( "bluefs_compact_log_sync", @@ -557,7 +568,9 @@ TEST(BlueFS, test_simple_compaction_sync) { fs.compact_log(); fs.umount(); } +*/ +/* TEST(BlueFS, test_simple_compaction_async) { g_ceph_context->_conf.set_val( "bluefs_compact_log_sync", @@ -609,7 +622,9 @@ TEST(BlueFS, test_simple_compaction_async) { fs.compact_log(); fs.umount(); } +*/ +/* TEST(BlueFS, test_compaction_sync) { uint64_t size = 1048576 * 128; TempBdev bdev{size}; @@ -673,7 +688,9 @@ TEST(BlueFS, test_compaction_sync) { } fs.umount(); } +*/ +/* TEST(BlueFS, test_compaction_async) { uint64_t size = 1048576 * 128; TempBdev bdev{size}; @@ -737,7 +754,9 @@ TEST(BlueFS, test_compaction_async) { } fs.umount(); } +*/ +/* TEST(BlueFS, test_replay) { uint64_t size = 1048576 * 128; TempBdev bdev{size}; @@ -778,7 +797,9 @@ TEST(BlueFS, test_replay) { ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); fs.umount(); } +*/ +/* TEST(BlueFS, test_replay_growth) { uint64_t size = 1048576LL * (2 * 1024 + 128); TempBdev bdev{size}; @@ -816,7 +837,9 @@ TEST(BlueFS, test_replay_growth) { ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); fs.umount(); } +*/ +/* TEST(BlueFS, test_tracker_50965) { uint64_t size_wal = 1048576 * 64; TempBdev bdev_wal{size_wal}; @@ -1013,6 +1036,7 @@ TEST(BlueFS, test_update_ino1_delta_after_replay) { ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); fs.umount(); } +*/ TEST(BlueFS, broken_unlink_fsync_seq) { uint64_t size = 1048576 * 128; diff --git a/src/test/objectstore/test_bluefs_compact_async.cc b/src/test/objectstore/test_bluefs_compact_async.cc new file mode 100644 index 00000000000..dc5bd2b1e53 --- /dev/null +++ b/src/test/objectstore/test_bluefs_compact_async.cc @@ -0,0 +1,338 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +#define ALLOC_SIZE 4096 + +void write_data(BlueFS &fs, uint64_t rationed_bytes) +{ + int j=0, r=0; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + stringstream ss; + string dir = "dir."; + ss << std::this_thread::get_id(); + dir.append(ss.str()); + dir.append("."); + dir.append(to_string(j)); + ASSERT_EQ(0, fs.mkdir(dir)); + while (1) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + j++; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +void create_single_file(BlueFS &fs) +{ + BlueFS::FileWriter *h; + stringstream ss; + string dir = "dir.test"; + ASSERT_EQ(0, fs.mkdir(dir)); + string file = "testfile"; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + fs.close_writer(h); +} + +void write_single_file(BlueFS &fs, uint64_t rationed_bytes) +{ + stringstream ss; + const string dir = "dir.test"; + const string file = "testfile"; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + while (1) { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + int r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +bool writes_done = false; + +void sync_fs(BlueFS &fs) +{ + while (1) { + if (writes_done == true) + break; + fs.sync_metadata(false); + sleep(1); + } +} + + +void do_join(std::thread& t) +{ + t.join(); +} + +void join_all(std::vector& v) +{ + std::for_each(v.begin(),v.end(),do_join); +} + +#define NUM_WRITERS 3 +#define NUM_SYNC_THREADS 1 + +#define NUM_SINGLE_FILE_WRITERS 1 +#define NUM_MULTIPLE_FILE_WRITERS 2 + +TEST(BlueFS, test_simple_compaction_async) { + g_ceph_context->_conf.set_val( + "bluefs_compact_log_sync", + "false"); + uint64_t size = 1048576 * 128; + TempBdev bdev{size}; + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + { + for (int i=0; i<10; i++) { + string dir = "dir."; + dir.append(to_string(i)); + ASSERT_EQ(0, fs.mkdir(dir)); + for (int j=0; j<10; j++) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(4096); + bufferptr bp = buffer::claim_char(4096, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + } + } + } + { + for (int i=0; i<10; i+=2) { + string dir = "dir."; + dir.append(to_string(i)); + for (int j=0; j<10; j++) { + string file = "file."; + file.append(to_string(j)); + fs.unlink(dir, file); + fs.sync_metadata(false); + } + ASSERT_EQ(0, fs.rmdir(dir)); + fs.sync_metadata(false); + } + } + fs.compact_log(); + fs.umount(); +} + +TEST(BlueFS, test_compaction_async) { + uint64_t size = 1048576 * 128; + TempBdev bdev{size}; + g_ceph_context->_conf.set_val( + "bluefs_alloc_size", + "65536"); + g_ceph_context->_conf.set_val( + "bluefs_compact_log_sync", + "false"); + const char* canary_dir = "dir.after_compact_test"; + const char* canary_file = "file.after_compact_test"; + const char* canary_data = "some random data"; + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + { + std::vector write_threads; + uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction + uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS)); + for (int i=0; i sync_threads; + for (int i=0; iappend(canary_data, strlen(canary_data)); + int r = fs.fsync(h); + ASSERT_EQ(r, 0); + } + } + fs.umount(); + + fs.mount(); + { + BlueFS::FileReader *h; + ASSERT_EQ(0, fs.open_for_read(canary_dir, canary_file, &h)); + ASSERT_NE(nullptr, h); + bufferlist bl; + ASSERT_EQ(strlen(canary_data), fs.read(h, 0, 1024, &bl, NULL)); + std::cout << bl.c_str() << std::endl; + ASSERT_EQ(0, strncmp(canary_data, bl.c_str(), strlen(canary_data))); + delete h; + } + fs.umount(); +} + +int main(int argc, char **argv) { + auto args = argv_to_vec(argc, argv); + map defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/src/test/objectstore/test_bluefs_compact_sync.cc b/src/test/objectstore/test_bluefs_compact_sync.cc new file mode 100644 index 00000000000..8ac84a2ca67 --- /dev/null +++ b/src/test/objectstore/test_bluefs_compact_sync.cc @@ -0,0 +1,338 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +#define ALLOC_SIZE 4096 + +void write_data(BlueFS &fs, uint64_t rationed_bytes) +{ + int j=0, r=0; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + stringstream ss; + string dir = "dir."; + ss << std::this_thread::get_id(); + dir.append(ss.str()); + dir.append("."); + dir.append(to_string(j)); + ASSERT_EQ(0, fs.mkdir(dir)); + while (1) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + j++; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +void create_single_file(BlueFS &fs) +{ + BlueFS::FileWriter *h; + stringstream ss; + string dir = "dir.test"; + ASSERT_EQ(0, fs.mkdir(dir)); + string file = "testfile"; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + fs.close_writer(h); +} + +void write_single_file(BlueFS &fs, uint64_t rationed_bytes) +{ + stringstream ss; + const string dir = "dir.test"; + const string file = "testfile"; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + while (1) { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + int r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +bool writes_done = false; + +void sync_fs(BlueFS &fs) +{ + while (1) { + if (writes_done == true) + break; + fs.sync_metadata(false); + sleep(1); + } +} + + +void do_join(std::thread& t) +{ + t.join(); +} + +void join_all(std::vector& v) +{ + std::for_each(v.begin(),v.end(),do_join); +} + +#define NUM_WRITERS 3 +#define NUM_SYNC_THREADS 1 + +#define NUM_SINGLE_FILE_WRITERS 1 +#define NUM_MULTIPLE_FILE_WRITERS 2 + +TEST(BlueFS, test_simple_compaction_sync) { + g_ceph_context->_conf.set_val( + "bluefs_compact_log_sync", + "true"); + uint64_t size = 1048576 * 128; + TempBdev bdev{size}; + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + { + for (int i=0; i<10; i++) { + string dir = "dir."; + dir.append(to_string(i)); + ASSERT_EQ(0, fs.mkdir(dir)); + for (int j=0; j<10; j++) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(4096); + bufferptr bp = buffer::claim_char(4096, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + } + } + } + { + for (int i=0; i<10; i+=2) { + string dir = "dir."; + dir.append(to_string(i)); + for (int j=0; j<10; j++) { + string file = "file."; + file.append(to_string(j)); + fs.unlink(dir, file); + fs.sync_metadata(false); + } + ASSERT_EQ(0, fs.rmdir(dir)); + fs.sync_metadata(false); + } + } + fs.compact_log(); + fs.umount(); +} + +TEST(BlueFS, test_compaction_sync) { + uint64_t size = 1048576 * 128; + TempBdev bdev{size}; + g_ceph_context->_conf.set_val( + "bluefs_alloc_size", + "65536"); + g_ceph_context->_conf.set_val( + "bluefs_compact_log_sync", + "true"); + const char* canary_dir = "dir.after_compact_test"; + const char* canary_file = "file.after_compact_test"; + const char* canary_data = "some random data"; + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + { + std::vector write_threads; + uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction + uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS)); + for (int i=0; i sync_threads; + for (int i=0; iappend(canary_data, strlen(canary_data)); + int r = fs.fsync(h); + ASSERT_EQ(r, 0); + } + } + fs.umount(); + + fs.mount(); + { + BlueFS::FileReader *h; + ASSERT_EQ(0, fs.open_for_read(canary_dir, canary_file, &h)); + ASSERT_NE(nullptr, h); + bufferlist bl; + ASSERT_EQ(strlen(canary_data), fs.read(h, 0, 1024, &bl, NULL)); + std::cout << bl.c_str() << std::endl; + ASSERT_EQ(0, strncmp(canary_data, bl.c_str(), strlen(canary_data))); + delete h; + } + fs.umount(); +} + +int main(int argc, char **argv) { + auto args = argv_to_vec(argc, argv); + map defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/src/test/objectstore/test_bluefs_flush_1.cc b/src/test/objectstore/test_bluefs_flush_1.cc new file mode 100644 index 00000000000..92112624c02 --- /dev/null +++ b/src/test/objectstore/test_bluefs_flush_1.cc @@ -0,0 +1,256 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +#define ALLOC_SIZE 4096 + +void write_data(BlueFS &fs, uint64_t rationed_bytes) +{ + int j=0, r=0; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + stringstream ss; + string dir = "dir."; + ss << std::this_thread::get_id(); + dir.append(ss.str()); + dir.append("."); + dir.append(to_string(j)); + ASSERT_EQ(0, fs.mkdir(dir)); + while (1) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + j++; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +void create_single_file(BlueFS &fs) +{ + BlueFS::FileWriter *h; + stringstream ss; + string dir = "dir.test"; + ASSERT_EQ(0, fs.mkdir(dir)); + string file = "testfile"; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + fs.close_writer(h); +} + +void write_single_file(BlueFS &fs, uint64_t rationed_bytes) +{ + stringstream ss; + const string dir = "dir.test"; + const string file = "testfile"; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + while (1) { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + int r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +bool writes_done = false; + +void sync_fs(BlueFS &fs) +{ + while (1) { + if (writes_done == true) + break; + fs.sync_metadata(false); + sleep(1); + } +} + + +void do_join(std::thread& t) +{ + t.join(); +} + +void join_all(std::vector& v) +{ + std::for_each(v.begin(),v.end(),do_join); +} + +#define NUM_WRITERS 3 +#define NUM_SYNC_THREADS 1 + +#define NUM_SINGLE_FILE_WRITERS 1 +#define NUM_MULTIPLE_FILE_WRITERS 2 + +TEST(BlueFS, test_flush_1) { + uint64_t size = 1048576 * 128; + TempBdev bdev{size}; + g_ceph_context->_conf.set_val( + "bluefs_alloc_size", + "65536"); + g_ceph_context->_conf.apply_changes(nullptr); + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + { + std::vector write_thread_multiple; + uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction + uint64_t per_thread_bytes = (effective_size/(NUM_MULTIPLE_FILE_WRITERS + NUM_SINGLE_FILE_WRITERS)); + for (int i=0; i write_thread_single; + for (int i=0; i defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/src/test/objectstore/test_bluefs_flush_2.cc b/src/test/objectstore/test_bluefs_flush_2.cc new file mode 100644 index 00000000000..622ab391dd1 --- /dev/null +++ b/src/test/objectstore/test_bluefs_flush_2.cc @@ -0,0 +1,249 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +#define ALLOC_SIZE 4096 + +void write_data(BlueFS &fs, uint64_t rationed_bytes) +{ + int j=0, r=0; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + stringstream ss; + string dir = "dir."; + ss << std::this_thread::get_id(); + dir.append(ss.str()); + dir.append("."); + dir.append(to_string(j)); + ASSERT_EQ(0, fs.mkdir(dir)); + while (1) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + j++; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +void create_single_file(BlueFS &fs) +{ + BlueFS::FileWriter *h; + stringstream ss; + string dir = "dir.test"; + ASSERT_EQ(0, fs.mkdir(dir)); + string file = "testfile"; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + fs.close_writer(h); +} + +void write_single_file(BlueFS &fs, uint64_t rationed_bytes) +{ + stringstream ss; + const string dir = "dir.test"; + const string file = "testfile"; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + while (1) { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + int r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +bool writes_done = false; + +void sync_fs(BlueFS &fs) +{ + while (1) { + if (writes_done == true) + break; + fs.sync_metadata(false); + sleep(1); + } +} + + +void do_join(std::thread& t) +{ + t.join(); +} + +void join_all(std::vector& v) +{ + std::for_each(v.begin(),v.end(),do_join); +} + +#define NUM_WRITERS 3 +#define NUM_SYNC_THREADS 1 + +#define NUM_SINGLE_FILE_WRITERS 1 +#define NUM_MULTIPLE_FILE_WRITERS 2 + +TEST(BlueFS, test_flush_2) { + uint64_t size = 1048576 * 256; + TempBdev bdev{size}; + g_ceph_context->_conf.set_val( + "bluefs_alloc_size", + "65536"); + g_ceph_context->_conf.apply_changes(nullptr); + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + { + uint64_t effective_size = size - (128 * 1048576); // leaving the last 32 MB for log compaction + uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS)); + std::vector write_thread_multiple; + for (int i=0; i defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/src/test/objectstore/test_bluefs_flush_3.cc b/src/test/objectstore/test_bluefs_flush_3.cc new file mode 100644 index 00000000000..ce3fdf7487d --- /dev/null +++ b/src/test/objectstore/test_bluefs_flush_3.cc @@ -0,0 +1,256 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +#define ALLOC_SIZE 4096 + +void write_data(BlueFS &fs, uint64_t rationed_bytes) +{ + int j=0, r=0; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + stringstream ss; + string dir = "dir."; + ss << std::this_thread::get_id(); + dir.append(ss.str()); + dir.append("."); + dir.append(to_string(j)); + ASSERT_EQ(0, fs.mkdir(dir)); + while (1) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + j++; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +void create_single_file(BlueFS &fs) +{ + BlueFS::FileWriter *h; + stringstream ss; + string dir = "dir.test"; + ASSERT_EQ(0, fs.mkdir(dir)); + string file = "testfile"; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + fs.close_writer(h); +} + +void write_single_file(BlueFS &fs, uint64_t rationed_bytes) +{ + stringstream ss; + const string dir = "dir.test"; + const string file = "testfile"; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + while (1) { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + int r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +bool writes_done = false; + +void sync_fs(BlueFS &fs) +{ + while (1) { + if (writes_done == true) + break; + fs.sync_metadata(false); + sleep(1); + } +} + + +void do_join(std::thread& t) +{ + t.join(); +} + +void join_all(std::vector& v) +{ + std::for_each(v.begin(),v.end(),do_join); +} + +#define NUM_WRITERS 3 +#define NUM_SYNC_THREADS 1 + +#define NUM_SINGLE_FILE_WRITERS 1 +#define NUM_MULTIPLE_FILE_WRITERS 2 + +TEST(BlueFS, test_flush_3) { + uint64_t size = 1048576 * 256; + TempBdev bdev{size}; + g_ceph_context->_conf.set_val( + "bluefs_alloc_size", + "65536"); + g_ceph_context->_conf.apply_changes(nullptr); + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + { + std::vector write_threads; + uint64_t effective_size = size - (64 * 1048576); // leaving the last 11 MB for log compaction + uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS)); + for (int i=0; i sync_threads; + for (int i=0; i defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/src/test/objectstore/test_bluefs_large_write_1.cc b/src/test/objectstore/test_bluefs_large_write_1.cc new file mode 100644 index 00000000000..3ef45c6e6a0 --- /dev/null +++ b/src/test/objectstore/test_bluefs_large_write_1.cc @@ -0,0 +1,183 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +TEST(BlueFS, very_large_write) { + // we'll write a ~5G file, so allocate more than that for the whole fs + uint64_t size = 1048576 * 1024 * 6ull; + TempBdev bdev{size}; + BlueFS fs(g_ceph_context); + + bool old = g_ceph_context->_conf.get_val("bluefs_buffered_io"); + g_ceph_context->_conf.set_val("bluefs_buffered_io", "false"); + uint64_t total_written = 0; + + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + char buf[1048571]; // this is biggish, but intentionally not evenly aligned + for (unsigned i = 0; i < sizeof(buf); ++i) { + buf[i] = i; + } + { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.mkdir("dir")); + ASSERT_EQ(0, fs.open_for_write("dir", "bigfile", &h, false)); + for (unsigned i = 0; i < 3*1024*1048576ull / sizeof(buf); ++i) { + h->append(buf, sizeof(buf)); + total_written += sizeof(buf); + } + fs.fsync(h); + for (unsigned i = 0; i < 2*1024*1048576ull / sizeof(buf); ++i) { + h->append(buf, sizeof(buf)); + total_written += sizeof(buf); + } + fs.fsync(h); + fs.close_writer(h); + } + { + BlueFS::FileReader *h; + ASSERT_EQ(0, fs.open_for_read("dir", "bigfile", &h)); + bufferlist bl; + ASSERT_EQ(h->file->fnode.size, total_written); + for (unsigned i = 0; i < 3*1024*1048576ull / sizeof(buf); ++i) { + bl.clear(); + fs.read(h, i * sizeof(buf), sizeof(buf), &bl, NULL); + int r = memcmp(buf, bl.c_str(), sizeof(buf)); + if (r) { + cerr << "read got mismatch at offset " << i*sizeof(buf) << " r " << r + << std::endl; + } + ASSERT_EQ(0, r); + } + for (unsigned i = 0; i < 2*1024*1048576ull / sizeof(buf); ++i) { + bl.clear(); + fs.read(h, i * sizeof(buf), sizeof(buf), &bl, NULL); + int r = memcmp(buf, bl.c_str(), sizeof(buf)); + if (r) { + cerr << "read got mismatch at offset " << i*sizeof(buf) << " r " << r + << std::endl; + } + ASSERT_EQ(0, r); + } + delete h; + ASSERT_EQ(0, fs.open_for_read("dir", "bigfile", &h)); + ASSERT_EQ(h->file->fnode.size, total_written); + auto huge_buf = std::make_unique(h->file->fnode.size); + auto l = h->file->fnode.size; + int64_t r = fs.read(h, 0, l, NULL, huge_buf.get()); + ASSERT_EQ(r, l); + delete h; + } + fs.umount(); + + g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old)); +} + +int main(int argc, char **argv) { + auto args = argv_to_vec(argc, argv); + map defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/test/objectstore/test_bluefs_large_write_2.cc b/src/test/objectstore/test_bluefs_large_write_2.cc new file mode 100644 index 00000000000..c71db31a6c1 --- /dev/null +++ b/src/test/objectstore/test_bluefs_large_write_2.cc @@ -0,0 +1,163 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +TEST(BlueFS, very_large_write2) { + // we'll write a ~5G file, so allocate more than that for the whole fs + uint64_t size_full = 1048576 * 1024 * 6ull; + uint64_t size = 1048576 * 1024 * 5ull; + TempBdev bdev{ size_full }; + BlueFS fs(g_ceph_context); + + bool old = g_ceph_context->_conf.get_val("bluefs_buffered_io"); + g_ceph_context->_conf.set_val("bluefs_buffered_io", "false"); + uint64_t total_written = 0; + + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + + char fill_arr[1 << 20]; // 1M + for (size_t i = 0; i < sizeof(fill_arr); ++i) { + fill_arr[i] = (char)i; + } + std::unique_ptr buf; + buf.reset(new char[size]); + for (size_t i = 0; i < size; i += sizeof(fill_arr)) { + memcpy(buf.get() + i, fill_arr, sizeof(fill_arr)); + } + { + BlueFS::FileWriter* h; + ASSERT_EQ(0, fs.mkdir("dir")); + ASSERT_EQ(0, fs.open_for_write("dir", "bigfile", &h, false)); + fs.append_try_flush(h, buf.get(), size); + total_written = size; + fs.fsync(h); + fs.close_writer(h); + } + memset(buf.get(), 0, size); + { + BlueFS::FileReader* h; + ASSERT_EQ(0, fs.open_for_read("dir", "bigfile", &h)); + ASSERT_EQ(h->file->fnode.size, total_written); + auto l = h->file->fnode.size; + int64_t r = fs.read(h, 0, l, NULL, buf.get()); + ASSERT_EQ(r, l); + for (size_t i = 0; i < size; i += sizeof(fill_arr)) { + ceph_assert(memcmp(buf.get() + i, fill_arr, sizeof(fill_arr)) == 0); + } + delete h; + } + fs.umount(); + + g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old)); +} + +int main(int argc, char **argv) { + auto args = argv_to_vec(argc, argv); + map defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/src/test/objectstore/test_bluefs_regression.cc b/src/test/objectstore/test_bluefs_regression.cc new file mode 100644 index 00000000000..9588ffc7c56 --- /dev/null +++ b/src/test/objectstore/test_bluefs_regression.cc @@ -0,0 +1,419 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +#define ALLOC_SIZE 4096 + +void write_data(BlueFS &fs, uint64_t rationed_bytes) +{ + int j=0, r=0; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + stringstream ss; + string dir = "dir."; + ss << std::this_thread::get_id(); + dir.append(ss.str()); + dir.append("."); + dir.append(to_string(j)); + ASSERT_EQ(0, fs.mkdir(dir)); + while (1) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + j++; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +void create_single_file(BlueFS &fs) +{ + BlueFS::FileWriter *h; + stringstream ss; + string dir = "dir.test"; + ASSERT_EQ(0, fs.mkdir(dir)); + string file = "testfile"; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + fs.close_writer(h); +} + +void write_single_file(BlueFS &fs, uint64_t rationed_bytes) +{ + stringstream ss; + const string dir = "dir.test"; + const string file = "testfile"; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + while (1) { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + int r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +bool writes_done = false; + +void sync_fs(BlueFS &fs) +{ + while (1) { + if (writes_done == true) + break; + fs.sync_metadata(false); + sleep(1); + } +} + + +void do_join(std::thread& t) +{ + t.join(); +} + +void join_all(std::vector& v) +{ + std::for_each(v.begin(),v.end(),do_join); +} + +#define NUM_WRITERS 3 +#define NUM_SYNC_THREADS 1 + +#define NUM_SINGLE_FILE_WRITERS 1 +#define NUM_MULTIPLE_FILE_WRITERS 2 + +TEST(BlueFS, test_tracker_50965) { + uint64_t size_wal = 1048576 * 64; + TempBdev bdev_wal{size_wal}; + uint64_t size_db = 1048576 * 128; + TempBdev bdev_db{size_db}; + uint64_t size_slow = 1048576 * 256; + TempBdev bdev_slow{size_slow}; + + ConfSaver conf(g_ceph_context->_conf); + conf.SetVal("bluefs_min_flush_size", "65536"); + conf.ApplyChanges(); + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_WAL, bdev_wal.path, false, 0)); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev_db.path, false, 0)); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_SLOW, bdev_slow.path, false, 0)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, true, true })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, true, true })); + + string dir_slow = "dir.slow"; + ASSERT_EQ(0, fs.mkdir(dir_slow)); + string dir_db = "dir_db"; + ASSERT_EQ(0, fs.mkdir(dir_db)); + + string file_slow = "file"; + BlueFS::FileWriter *h_slow; + ASSERT_EQ(0, fs.open_for_write(dir_slow, file_slow, &h_slow, false)); + ASSERT_NE(nullptr, h_slow); + + string file_db = "file"; + BlueFS::FileWriter *h_db; + ASSERT_EQ(0, fs.open_for_write(dir_db, file_db, &h_db, false)); + ASSERT_NE(nullptr, h_db); + + bufferlist bl1; + std::unique_ptr buf1 = gen_buffer(70000); + bufferptr bp1 = buffer::claim_char(70000, buf1.get()); + bl1.push_back(bp1); + h_slow->append(bl1.c_str(), bl1.length()); + fs.flush(h_slow); + + uint64_t h_slow_dirty_seq_1 = fs.debug_get_dirty_seq(h_slow); + + bufferlist bl2; + std::unique_ptr buf2 = gen_buffer(1000); + bufferptr bp2 = buffer::claim_char(1000, buf2.get()); + bl2.push_back(bp2); + h_db->append(bl2.c_str(), bl2.length()); + fs.fsync(h_db); + + uint64_t h_slow_dirty_seq_2 = fs.debug_get_dirty_seq(h_slow); + bool h_slow_dev_dirty = fs.debug_get_is_dev_dirty(h_slow, BlueFS::BDEV_SLOW); + + //problem if allocations are stable in log but slow device is not flushed yet + ASSERT_FALSE(h_slow_dirty_seq_1 != 0 && + h_slow_dirty_seq_2 == 0 && + h_slow_dev_dirty == true); + + fs.close_writer(h_slow); + fs.close_writer(h_db); + + fs.umount(); +} + +TEST(BlueFS, test_truncate_stable_53129) { + + ConfSaver conf(g_ceph_context->_conf); + conf.SetVal("bluefs_min_flush_size", "65536"); + conf.ApplyChanges(); + + uint64_t size_wal = 1048576 * 64; + TempBdev bdev_wal{size_wal}; + uint64_t size_db = 1048576 * 128; + TempBdev bdev_db{size_db}; + uint64_t size_slow = 1048576 * 256; + TempBdev bdev_slow{size_slow}; + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_WAL, bdev_wal.path, false, 0)); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev_db.path, false, 0)); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_SLOW, bdev_slow.path, false, 0)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, true, true })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, true, true })); + + string dir_slow = "dir.slow"; + ASSERT_EQ(0, fs.mkdir(dir_slow)); + string dir_db = "dir_db"; + ASSERT_EQ(0, fs.mkdir(dir_db)); + + string file_slow = "file"; + BlueFS::FileWriter *h_slow; + ASSERT_EQ(0, fs.open_for_write(dir_slow, file_slow, &h_slow, false)); + ASSERT_NE(nullptr, h_slow); + + string file_db = "file"; + BlueFS::FileWriter *h_db; + ASSERT_EQ(0, fs.open_for_write(dir_db, file_db, &h_db, false)); + ASSERT_NE(nullptr, h_db); + + bufferlist bl1; + std::unique_ptr buf1 = gen_buffer(70000); + bufferptr bp1 = buffer::claim_char(70000, buf1.get()); + bl1.push_back(bp1); + // add 70000 bytes + h_slow->append(bl1.c_str(), bl1.length()); + fs.flush(h_slow); + // and truncate to 60000 bytes + fs.truncate(h_slow, 60000); + + // write something to file on DB device + bufferlist bl2; + std::unique_ptr buf2 = gen_buffer(1000); + bufferptr bp2 = buffer::claim_char(1000, buf2.get()); + bl2.push_back(bp2); + h_db->append(bl2.c_str(), bl2.length()); + // and force bluefs log to flush + fs.fsync(h_db); + + // This is the actual test point. + // We completed truncate, and we expect + // - size to be 60000 + // - data to be stable on slow device + // OR + // - size = 0 or file does not exist + // - dev_dirty is irrelevant + bool h_slow_dev_dirty = fs.debug_get_is_dev_dirty(h_slow, BlueFS::BDEV_SLOW); + // Imagine power goes down here. + + fs.close_writer(h_slow); + fs.close_writer(h_db); + + fs.umount(); + + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, true, true })); + + uint64_t size; + utime_t mtime; + ASSERT_EQ(0, fs.stat("dir.slow", "file", &size, &mtime)); + // check file size 60000 + ASSERT_EQ(size, 60000); + // check that dev_dirty was false (data stable on media) + ASSERT_EQ(h_slow_dev_dirty, false); + + fs.umount(); +} + +TEST(BlueFS, test_update_ino1_delta_after_replay) { + uint64_t size = 1048576LL * (2 * 1024 + 128); + TempBdev bdev{size}; + + ConfSaver conf(g_ceph_context->_conf); + conf.SetVal("bluefs_alloc_size", "4096"); + conf.SetVal("bluefs_shared_alloc_size", "4096"); + conf.SetVal("bluefs_compact_log_sync", "false"); + conf.SetVal("bluefs_min_log_runway", "32768"); + conf.SetVal("bluefs_max_log_runway", "65536"); + conf.SetVal("bluefs_allocator", "stupid"); + conf.ApplyChanges(); + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mkdir("dir")); + + char data[2000]; + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false)); + for (size_t i = 0; i < 100; i++) { + h->append(data, 2000); + fs.fsync(h); + } + fs.close_writer(h); + fs.umount(true); //do not compact on exit! + + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.open_for_write("dir", "file2", &h, false)); + for (size_t i = 0; i < 100; i++) { + h->append(data, 2000); + fs.fsync(h); + } + fs.close_writer(h); + fs.umount(); + + // remount and check log can replay safe? + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + fs.umount(); +} + +int main(int argc, char **argv) { + auto args = argv_to_vec(argc, argv); + map defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/src/test/objectstore/test_bluefs_replay.cc b/src/test/objectstore/test_bluefs_replay.cc new file mode 100644 index 00000000000..be87affdcd1 --- /dev/null +++ b/src/test/objectstore/test_bluefs_replay.cc @@ -0,0 +1,263 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +#define ALLOC_SIZE 4096 + +void write_data(BlueFS &fs, uint64_t rationed_bytes) +{ + int j=0, r=0; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + stringstream ss; + string dir = "dir."; + ss << std::this_thread::get_id(); + dir.append(ss.str()); + dir.append("."); + dir.append(to_string(j)); + ASSERT_EQ(0, fs.mkdir(dir)); + while (1) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + j++; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +void create_single_file(BlueFS &fs) +{ + BlueFS::FileWriter *h; + stringstream ss; + string dir = "dir.test"; + ASSERT_EQ(0, fs.mkdir(dir)); + string file = "testfile"; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + fs.close_writer(h); +} + +void write_single_file(BlueFS &fs, uint64_t rationed_bytes) +{ + stringstream ss; + const string dir = "dir.test"; + const string file = "testfile"; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + while (1) { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + int r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +bool writes_done = false; + +void sync_fs(BlueFS &fs) +{ + while (1) { + if (writes_done == true) + break; + fs.sync_metadata(false); + sleep(1); + } +} + + +void do_join(std::thread& t) +{ + t.join(); +} + +void join_all(std::vector& v) +{ + std::for_each(v.begin(),v.end(),do_join); +} + +#define NUM_WRITERS 3 +#define NUM_SYNC_THREADS 1 + +#define NUM_SINGLE_FILE_WRITERS 1 +#define NUM_MULTIPLE_FILE_WRITERS 2 + +TEST(BlueFS, test_replay) { + uint64_t size = 1048576 * 128; + TempBdev bdev{size}; + g_ceph_context->_conf.set_val( + "bluefs_alloc_size", + "65536"); + g_ceph_context->_conf.set_val( + "bluefs_compact_log_sync", + "false"); + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + { + std::vector write_threads; + uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction + uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS)); + for (int i=0; i sync_threads; + for (int i=0; i defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/src/test/objectstore/test_bluefs_replay_growth.cc b/src/test/objectstore/test_bluefs_replay_growth.cc new file mode 100644 index 00000000000..a02b8e185ef --- /dev/null +++ b/src/test/objectstore/test_bluefs_replay_growth.cc @@ -0,0 +1,260 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" + +#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; + +std::unique_ptr gen_buffer(uint64_t size) +{ + std::unique_ptr buffer = std::make_unique(size); + std::independent_bits_engine e; + std::generate(buffer.get(), buffer.get()+size, std::ref(e)); + return buffer; +} + +class TempBdev { +public: + TempBdev(uint64_t size) + : path{get_temp_bdev(size)} + {} + ~TempBdev() { + rm_temp_bdev(path); + } + const std::string path; +private: + static string get_temp_bdev(uint64_t size) + { + static int n = 0; + string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid()) + + "." + stringify(++n); + int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644); + ceph_assert(fd >= 0); + int r = ::ftruncate(fd, size); + ceph_assert(r >= 0); + ::close(fd); + return fn; + } + static void rm_temp_bdev(string f) + { + ::unlink(f.c_str()); + } +}; + +class ConfSaver { + std::stack> saved_settings; + ConfigProxy& conf; +public: + ConfSaver(ConfigProxy& conf) : conf(conf) { + conf._clear_safe_to_start_threads(); + }; + ~ConfSaver() { + conf._clear_safe_to_start_threads(); + while(saved_settings.size() > 0) { + auto& e = saved_settings.top(); + conf.set_val_or_die(e.first, e.second); + saved_settings.pop(); + } + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } + void SetVal(const char* key, const char* val) { + std::string skey(key); + std::string prev_val; + conf.get_val(skey, &prev_val); + conf.set_val_or_die(skey, val); + saved_settings.emplace(skey, prev_val); + } + void ApplyChanges() { + conf.set_safe_to_start_threads(); + conf.apply_changes(nullptr); + } +}; + +#define ALLOC_SIZE 4096 + +void write_data(BlueFS &fs, uint64_t rationed_bytes) +{ + int j=0, r=0; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + stringstream ss; + string dir = "dir."; + ss << std::this_thread::get_id(); + dir.append(ss.str()); + dir.append("."); + dir.append(to_string(j)); + ASSERT_EQ(0, fs.mkdir(dir)); + while (1) { + string file = "file."; + file.append(to_string(j)); + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + j++; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +void create_single_file(BlueFS &fs) +{ + BlueFS::FileWriter *h; + stringstream ss; + string dir = "dir.test"; + ASSERT_EQ(0, fs.mkdir(dir)); + string file = "testfile"; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + fs.fsync(h); + fs.close_writer(h); +} + +void write_single_file(BlueFS &fs, uint64_t rationed_bytes) +{ + stringstream ss; + const string dir = "dir.test"; + const string file = "testfile"; + uint64_t written_bytes = 0; + rationed_bytes -= ALLOC_SIZE; + while (1) { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); + ASSERT_NE(nullptr, h); + auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); + bufferlist bl; + std::unique_ptr buf = gen_buffer(ALLOC_SIZE); + bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); + bl.push_back(bp); + h->append(bl.c_str(), bl.length()); + int r = fs.fsync(h); + if (r < 0) { + break; + } + written_bytes += g_conf()->bluefs_alloc_size; + if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { + break; + } + } +} + +bool writes_done = false; + +void sync_fs(BlueFS &fs) +{ + while (1) { + if (writes_done == true) + break; + fs.sync_metadata(false); + sleep(1); + } +} + + +void do_join(std::thread& t) +{ + t.join(); +} + +void join_all(std::vector& v) +{ + std::for_each(v.begin(),v.end(),do_join); +} + +#define NUM_WRITERS 3 +#define NUM_SYNC_THREADS 1 + +#define NUM_SINGLE_FILE_WRITERS 1 +#define NUM_MULTIPLE_FILE_WRITERS 2 + +TEST(BlueFS, test_replay_growth) { + uint64_t size = 1048576LL * (2 * 1024 + 128); + TempBdev bdev{size}; + + ConfSaver conf(g_ceph_context->_conf); + conf.SetVal("bluefs_alloc_size", "4096"); + conf.SetVal("bluefs_shared_alloc_size", "4096"); + conf.SetVal("bluefs_compact_log_sync", "false"); + conf.SetVal("bluefs_min_log_runway", "32768"); + conf.SetVal("bluefs_max_log_runway", "65536"); + conf.SetVal("bluefs_allocator", "stupid"); + conf.SetVal("bluefs_sync_write", "true"); + conf.ApplyChanges(); + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + ASSERT_EQ(0, fs.mkdir("dir")); + + char data[2000]; + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false)); + // PKGBUILD NOTE: 10000 -> 1000 (otherwise too slow) + for (size_t i = 0; i < 1000; i++) { + h->append(data, 2000); + fs.fsync(h); + } + fs.close_writer(h); + fs.umount(true); //do not compact on exit! + + // remount and check log can replay safe? + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + fs.umount(); +} + +int main(int argc, char **argv) { + auto args = argv_to_vec(argc, argv); + map defaults = { + { "debug_bluefs", "1/20" }, + { "debug_bdev", "1/20" } + }; + + auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, + CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); + common_init_finish(g_ceph_context); + g_ceph_context->_conf.set_val( + "enable_experimental_unrecoverable_data_corrupting_features", + "*"); + g_ceph_context->_conf.apply_changes(nullptr); + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} +