diff options
| author | Bazaah | 2023-09-09 17:48:54 +0000 |
|---|---|---|
| committer | Bazaah | 2023-09-09 17:48:54 +0000 |
| commit | 526abeb5c9b62beb4803954ea9bc36b3fb1375d4 (patch) | |
| tree | e78bf20a0f001225757d1e4dc2d759d130ed656d | |
| parent | 764375e8303a475c871af78be5278717ab828f5a (diff) | |
| parent | 321091d61d0a9e75dd39a1f3eac662e47f60c2a4 (diff) | |
| download | aur-526abeb5c9b62beb4803954ea9bc36b3fb1375d4.tar.gz | |
merge: master <--- feature/v18.2.0-1
* HEAD
| \
| * 321091d <github@luxolus.com> pkgbuild: pkgver=v18.2.0 pkgrel=1
| * fe66554 <github@luxolus.com> justfile: add cexec, ctest, repackage targets
| * 343f78c <github@luxolus.com> justfile: improvements to upload to handle new packages
| * 3ddb703 <github@luxolus.com> justfile: remove watch, logs
| * 9ca7efd <github@luxolus.com> gh-upload-artifact: try pretty print happy path of upload
| * 4cc7eb5 <github@luxolus.com> pkgbuild: updates to install files for v18
| * b019b81 <github@luxolus.com> pkgbuild: enable optional rbd components
| * 1ce90d2 <github@luxolus.com> pkgbuild: disable proprietary rgw backends
| * 76a50ba <github@luxolus.com> pkgbuild: disable tracing / opentelemetry
| * 030000e <github@luxolus.com> pkgbuild: reenable system utf8proc
| * ffbeef6 <github@luxolus.com> pkgbuild: use backport-log-runway-expansion.patch
| * 216795b <github@luxolus.com> repo: add backport-log-runway-expansion.patch
| * 9e5e0ad <github@luxolus.com> pkgbuild: use fix-ecode-shec-test.patch
| * 2f5f113 <github@luxolus.com> repo: add ceph-18.2.0-fix-ecode-shec-test.patch
| * 1f5029b <github@luxolus.com> pkgbuild: use ceph-18.2.0-fmt10-fixes.patch
| * 279af4c <github@luxolus.com> repo: add ceph-18.2.0-fmt10-fixes.patch
| * c1d084d <github@luxolus.com> repo: update ceph-17.2.4-test-bluefs-split.patch
| * 5dd05c5 <github@luxolus.com> repo: rm upstreamed iterator depreciations
| * ef24ee7 <github@luxolus.com> pkgbuild: update tox-mypy-false-postive.patch
| * 5b923a5 <github@luxolus.com> pkgbuild: rm upstreamed gcc13 patches
| * 4095603 <github@luxolus.com> pkgbuild: rm delete-test-librados-asio.patch
| * a60c26a <github@luxolus.com> pkgbuild: rm -DFMT_DEPRECATED_OSTREAM
| * 1531a6a <github@luxolus.com> pkgbuild: rm ceph-17.2.4.-tox-cephadm-rm.patch
| * 187fed6 <github@luxolus.com> repo: revert backport-with-fmt-version.patch
| * 4df4495 <github@luxolus.com> pkgbuild: add inetutils,xmlstarlet to checkdepends
| * f36dcde <github@luxolus.com> pkgbuild: add makedepend 'thrift'
| * da1d129 <github@luxolus.com> pkgbuild: cleanup pkgname, reorganize arch,pkgdesc
| * bfd09af <github@luxolus.com> pkgbuild: reorganize makedepends, checkdepends
| * f649488 <github@luxolus.com> pkgbuild: package virtual targets
| * bf7fef7 <github@luxolus.com> pkgbuild: package misc. ceph tooling / libs
| * eb35c47 <github@luxolus.com> pkgbuild: package more client / utils
| * 0c50604 <github@luxolus.com> pkgbuild: package ceph cluster components
| * f115c61 <github@luxolus.com> pkgbuild: package ceph python libs
| * 7f61cf4 <github@luxolus.com> pkgbuild: package core clients
| * d2936ef <github@luxolus.com> pkgbuild: package core libs
| * 38b0f6b <github@luxolus.com> pkgbuild: add func _make_ceph_packages
| * 7666ef7 <github@luxolus.com> pkgbuild: add _package, _print funcs
| * a2346ce <github@luxolus.com> pkgbuild: rm old package() functions
| /
| master
Closes: #16
Signed-off-by: Bazaah <github@luxolus.com>
| -rw-r--r-- | .SRCINFO | 495 | ||||
| -rw-r--r-- | PKGBUILD | 1100 | ||||
| -rw-r--r-- | ceph-17.2.4-test-bluefs-split.patch | 359 | ||||
| -rw-r--r-- | ceph-17.2.4-tox-cephadm-rm.patch | 11 | ||||
| -rw-r--r-- | ceph-17.2.4-tox-mypy-false-postive.patch | 2 | ||||
| -rw-r--r-- | ceph-17.2.5-fix-iterator-depreciations.patch | 79 | ||||
| -rw-r--r-- | ceph-17.2.6-backport-with-fmt-version.patch | 15 | ||||
| -rw-r--r-- | ceph-17.2.6-ceph-gcc13-includes.patch | 49 | ||||
| -rw-r--r-- | ceph-17.2.6-rocksdb-gcc13-includes.patch | 77 | ||||
| -rw-r--r-- | ceph-18.2.0-backport-log-runway-expansion.patch | 577 | ||||
| -rw-r--r-- | ceph-18.2.0-fix-ecode-shec-test.patch | 19 | ||||
| -rw-r--r-- | ceph-18.2.0-fmt10-fixes.patch | 206 | ||||
| -rw-r--r-- | justfile | 45 | ||||
| -rwxr-xr-x | scripts.gh-upload-artifact.sh | 5 |
14 files changed, 2138 insertions, 901 deletions
@@ -1,285 +1,446 @@ pkgbase = ceph pkgdesc = Distributed, fault-tolerant storage platform delivering object, block, and file system - pkgver = 17.2.6 - pkgrel = 4 + pkgver = 18.2.0 + pkgrel = 1 url = https://ceph.com/ arch = x86_64 license = GPL + checkdepends = inetutils + checkdepends = xmlstarlet checkdepends = python-mock checkdepends = python-nose checkdepends = python-pycodestyle checkdepends = python-pylint checkdepends = python-pytest checkdepends = python-pytest-cov - makedepends = zstd makedepends = bash - makedepends = bc makedepends = boost makedepends = boost-libs - makedepends = bzip2 - makedepends = c-ares makedepends = cmake makedepends = coreutils - makedepends = coffeescript - makedepends = cpio - makedepends = crypto++ makedepends = cryptsetup - makedepends = cunit makedepends = curl makedepends = cython makedepends = expat - makedepends = fcgi - makedepends = fontconfig - makedepends = fuse2 - makedepends = fuse3 makedepends = fmt - makedepends = gcc + makedepends = fuse3 + makedepends = gawk makedepends = gcc-libs makedepends = git makedepends = glibc - makedepends = gmock - makedepends = gnutls makedepends = gperf makedepends = gperftools - makedepends = gptfdisk - makedepends = gtest - makedepends = hwloc - makedepends = inetutils makedepends = java-runtime makedepends = jq makedepends = jre11-openjdk-headless makedepends = junit makedepends = keyutils - makedepends = leveldb makedepends = libaio makedepends = libatomic_ops makedepends = libcap makedepends = libcap-ng - makedepends = libcroco makedepends = libcurl-compat makedepends = libedit makedepends = libgudev makedepends = libnl makedepends = librabbitmq-c - makedepends = libtool - makedepends = util-linux - makedepends = libuv - makedepends = libxml2 makedepends = librdkafka - makedepends = libpciaccess - makedepends = lsb-release + makedepends = libutil-linux + makedepends = libuv + makedepends = libxcrypt makedepends = lua makedepends = lz4 - makedepends = ncurses makedepends = nss - makedepends = numactl makedepends = oath-toolkit makedepends = openssl - makedepends = parted - makedepends = pcre - makedepends = pcre2 makedepends = pkgconf - makedepends = protobuf - makedepends = procps-ng - makedepends = python-astroid - makedepends = python-attrs + makedepends = python + makedepends = snappy + makedepends = sqlite + makedepends = systemd-libs + makedepends = thrift + makedepends = util-linux + makedepends = xfsprogs + makedepends = zlib + makedepends = zstd makedepends = python-bcrypt - makedepends = python-cheroot makedepends = python-cherrypy makedepends = python-coverage makedepends = python-dateutil - makedepends = python-elasticsearch - makedepends = python-flask - makedepends = python-flask-restful - makedepends = python-google-api-python-client - makedepends = python-google-auth - makedepends = python-google-auth-httplib2 - makedepends = python-grpcio - makedepends = python-isort makedepends = python-jinja - makedepends = python-lazy-object-proxy - makedepends = python-mccabe - makedepends = python-isodate - makedepends = python-defusedxml - makedepends = python-pkgconfig - makedepends = python-protobuf - makedepends = python-lxml - makedepends = python-xmlsec - makedepends = python-yaml - makedepends = python-more-itertools - makedepends = python-numpy - makedepends = python-pbr makedepends = python-pecan - makedepends = python-pip - makedepends = python-pluggy - makedepends = python-portend makedepends = python-prettytable - makedepends = python-prometheus_client - makedepends = python-py - makedepends = python-pycparser makedepends = python-pyjwt makedepends = python-pyopenssl - makedepends = python-pytz makedepends = python-requests - makedepends = python-routes - makedepends = python-scikit-learn makedepends = python-scipy makedepends = python-setuptools - makedepends = python-six makedepends = python-sphinx - makedepends = python-tempora - makedepends = python-virtualenv + makedepends = python-typing_extensions makedepends = python-werkzeug - makedepends = python-wrapt - makedepends = rabbitmq - makedepends = sed - makedepends = snappy - makedepends = socat - makedepends = systemd - makedepends = systemd-libs - makedepends = valgrind - makedepends = xfsprogs - makedepends = xmlstarlet - makedepends = xmlsec - makedepends = xxhash - makedepends = yaml-cpp - makedepends = yasm - makedepends = zlib + makedepends = python-yaml options = emptydirs options = !lto - source = https://download.ceph.com/tarballs/ceph-17.2.6.tar.gz + source = https://download.ceph.com/tarballs/ceph-18.2.0.tar.gz source = ceph.sysusers source = ceph.sudoers source = ceph-13.2.2-dont-install-sysvinit-script.patch source = disable-empty-readable.sh-test.patch source = ceph-17.2.5-logrotate-ignore-dups.patch - source = ceph-16.2.7-delete-test-librados-asio.patch source = ceph-17.2.4-tox-flake8-git-ls-files.patch source = ceph-17.2.4-tox-mypy-false-postive.patch - source = ceph-17.2.4-tox-cephadm-rm.patch source = ceph-17.2.4-test-bluefs-split.patch source = ceph-17.2.4-pybind-unmock-cythonize.patch source = ceph-17.2.5-fix-iterator-depreciations.patch source = ceph-17.2.5-rgw-client-boost-string-view.patch - source = ceph-17.2.6-rocksdb-gcc13-includes.patch - source = ceph-17.2.6-ceph-gcc13-includes.patch source = ceph-17.2.6-mgr-dashboard-node-version.patch source = ceph-17.2.6-mgr-dashboard-cherrypy-18.patch source = ceph-17.2.6-mgr-dashboard-pylint-217.patch - source = ceph-17.2.6-backport-with-fmt-version.patch source = ceph-17.2.6-cython-fixes.patch - sha512sums = dca9aea2ce210c15fcc34cb06a5dc5b4488ffa36d684166d47ebd87e48b54b6fee0882e1c67007a780e1c25754e9bc6e760cc10f60ea1183263f8504ef2dbd9b + source = ceph-18.2.0-fmt10-fixes.patch + source = ceph-18.2.0-fix-ecode-shec-test.patch + source = ceph-18.2.0-backport-log-runway-expansion.patch + sha512sums = fe6a3ca4227cc05b5effc33a02670c466583ec9e4c7c025241248223992254e6e6551d7db486d5ead36bf0ba922d54e0aabc2aaf456d4593679376ec88bd971a sha512sums = 4354001c1abd9a0c385ba7bd529e3638fb6660b6a88d4e49706d4ac21c81b8e829303a20fb5445730bdac18c4865efb10bc809c1cd56d743c12aa9a52e160049 sha512sums = 41dbc1c395cdf9b3edf5c5d91bbc90f416b4338ad964fa3471f26a4312d3ec2a5dcebbc351a1640dc4b047b4f71aa134ac7486747e5f62980092b0176e7567f5 sha512sums = ea069b75b786c22166c609b127b512802cc5c6e9512d792d7b7b34d276f5b86d57c8c35cfc7b5c855a59c0ba87ba1aabe2ca26da72b26bff46b6ba8410ddb27e sha512sums = 2234d005df71b3b6013e6b76ad07a5791e3af7efec5f41c78eb1a9c92a22a67f0be9560be59b52534e90bfe251bcf32c33d5d40163f3f8f7e7420691f0f4a222 sha512sums = b12cabda7184721c494edd22250fd05019694d2bc445722d100cdefab5385bd25c2267a029d2f6053932fa6717e38c4314385afd986969ee2744d745b53c8b58 - sha512sums = 66770a80ba4e05ea72d4809cb5819cce7499ea7523b85b1a57370df68de1d7f6f94b1c10d0f9f9a3c8e6a86d0419434c70778c568cd06a0dd2e6126631a3355c sha512sums = 31e578b240ceaaf1216b56cdce654661eed6529ef642ecad164a02669e850100a49a85dc70f3d744671e2c5dad10aee64be7d091fa33007cb8fc6788a4336799 - sha512sums = 85c0a00419c292204be03e8962b0d4bc67c5dc21eb19e74897ec88168f19ac93fc2e38c405b212421c419a74923ec8c5c10844e23269225c52937fd2ad40d817 - sha512sums = 6b53e3a2b1941f8dfdb4b83cafc4c500e818dd3a0736d7061f0ebcdfb514094b217e4688384c5b428288181ef6fff0ca73895c65a046e2d6ae2b834375164216 - sha512sums = 81f540c8312972887a7cb43b8a4e29bfc6f24d5774787a4a8edfe65cca7d3b08faa08ecd09066d7ea67111769a5aec7385fe9a969546626f58874dd8aff5b664 + sha512sums = f1f549c5da829db787ee6ee3cf47912d5b48efb2533de05fd9735736e9218a78cf79c5958e1409a2c8adcca51460eb3ae7e73b5c1ec6cf73d4f07670c9934c3b + sha512sums = 5b43ba48dde8b7371d4b3976aa3a27ab3ad3a10e110479a101b8f222590bbd6aca318f3ac117f851596fc23daafb53aaa566e055dc9694ab16ab1170e8f77763 sha512sums = 781a01e622a70d56bf1948bdc0b427ffa95a86cec7dd9d26c6007a9ec024a942a8ca55f2acc3d37344862f1d6bf11cae998d8071754cd841a66bfba4ec9c58bf - sha512sums = 2a6f33791760e14543c90077bfc6bf1b6b82ee2996e80b4762eadb887a0d9a67c221b6f10832ddf780dc6abaed246a1e2ee7680c9c861c4ff70e61b752a37b36 + sha512sums = 612faebfb5eec3651832f349ea3c23b50d2386889ff77592b0acff653049efdc5c2254f63c30d88b9a730813bf1f1945dda0d0beab0db7db3e0708ba8d057a40 sha512sums = 02ca1a9bf15f9cd6f474f264ba2bf66ae725bac990a7cac315dabe377c66935a7afd8117f18a1f82c78bdf9ab2c3e5a2a227f2ffc166974dd7bb99b290f1f233 - sha512sums = 1897ddfcd21fbbd1f18f304db6fddb5ba9336bbe2be365ece57a3ae8195f075ceb66f40fc37a4b5bf6481e8a87561383108493aa88cb01d78a904c9da28bb36c - sha512sums = 2971475ed43480802d6053e259e8540475b3da0d2a84edc9df05e822d51f31b25de450f387fb33e1f7034d3139e502e8da1b0b00ccbb066d01091055e880804b sha512sums = d9729c33f2b8c03ab918919e3b62cbc0825314ad09596dc12984ac5964f8b421db3376b84fa26e0952e0781deb218f8aff6ba32f9c8e5a22bd27afac8dcaf6d3 sha512sums = 79be1630ae4a599509e5d789d4aefe412ce47e67ad482f853664fa4b01e063c20593e3da668e6a776ad038fb07606ae948eea41bab20776c33c87f9ab49505e0 sha512sums = c767a8e6fd02ea2ab88e99b50b206d0f825acdf177136ded38d93594fc7663b7c9612af7195b85e0b2b501d8ee482af5e088e9abb5ebee7b8a69e0153ce89782 - sha512sums = 6b4d829c9ecacdc8fbbc6d36eb59bcd844544b728121f75101d5078fd87f6531aeafff5de636c325ebcb88eb0da49f9f62f947bf7d7db4be426ab8ddb02996f6 sha512sums = 0c5124693bd317a73707dfd34b17664cc05233aec08e07739fe08fc9a73be7a1f4446052b1addde832cba141a382c35f45e60c89a00bb7dab81cee7ed6be07e1 + sha512sums = 4613232e5a0003c08d233e40fe3ac1cd00e1195d29bdd9892188587b4a782d6979004232927c0a1bff554eabf2fb9b18eb751682b7ad90762292b63891f3b301 + sha512sums = 9a1183c08f8799b14235c9271519203cbf93e48ca3a8607d3a0500910efca5379c8a08421c377227f93d8436a850f5ca99784f28aaa920e55f0457c657511f17 + sha512sums = e238b326609636bc7dd10cec59290e22898948ef105c49643c38d2621abf16c2efcf9581b0b6bad65066607510c9827d00a7abdb14f2054701cc33b7101ea054 -pkgname = ceph - depends = ceph-libs=17.2.6-4 +pkgname = ceph-common + pkgdesc = Ceph Storage common libraries and dependencies + depends = ceph-compressor=18.2.0-1 + depends = ceph-crypto=18.2.0-1 + depends = ceph-erasure=18.2.0-1 depends = boost-libs depends = curl - depends = fuse3 - depends = fmt depends = glibc - depends = gperftools - depends = java-runtime depends = keyutils - depends = libaio depends = libutil-linux - depends = librdkafka - depends = cryptsetup - depends = libnl - depends = ncurses depends = nss - depends = oath-toolkit - depends = python - depends = snappy - depends = sudo depends = systemd-libs - depends = lua - depends = gawk - optdepends = smartmontools: disk monitoring via S.M.A.R.T - optdepends = nvme-cli: disk monitoring for NVMe drives - provides = ceph=17.2.6-4 - conflicts = ceph-bin + depends = bash + depends = fmt + depends = cryptsetup + depends = libxcrypt + depends = libaio + depends = libcap + depends = gperftools + provides = libceph-common.so + provides = libceph_ebd_vdo.so + +pkgname = ceph-compressor + pkgdesc = Ceph Storage compressor libs + depends = gcc-libs + depends = lz4 + depends = snappy + depends = zlib + depends = zstd + +pkgname = ceph-crypto + pkgdesc = Ceph Storage crypto libs + depends = gcc-libs + depends = openssl + +pkgname = ceph-erasure + pkgdesc = Ceph Storage erasure coding libs + depends = gcc-libs + +pkgname = ceph-tools + pkgdesc = Ceph Storage miscellaneous tooling and utilities + depends = ceph-base=18.2.0-1 + depends = bash + depends = boost-libs + depends = gperftools + depends = libaio + depends = libcap + depends = snappy + +pkgname = ceph-test + pkgdesc = Ceph Storage tools for benchmarking and testing live clusters + depends = ceph-base=18.2.0-1 + depends = libcap + depends = libaio + depends = boost-libs + depends = fmt + depends = gperftools + depends = snappy + +pkgname = ceph-volume + pkgdesc = Ceph Storage utility for preparing block devices for use as OSDs + depends = python-ceph-common=18.2.0-1 + depends = ceph-osd=18.2.0-1 + depends = python + depends = lvm2 + depends = python-setuptools + depends = python-importlib-metadata + +pkgname = ceph-cephadm + pkgdesc = Ceph Storage adminstration and configuration utility + depends = python + depends = podman + provides = cephadm + conflicts = cephadm + +pkgname = ceph-rados + pkgdesc = Ceph Storage utilities and tools for librados + depends = librados=18.2.0-1 + depends = gcc-libs + +pkgname = ceph-base + pkgdesc = Ceph Storage cluster base utilities and configuration + depends = ceph-common=18.2.0-1 + depends = librados=18.2.0-1 + depends = python-ceph-common=18.2.0-1 + depends = python-rados=18.2.0-1 + depends = python backup = etc/logrotate.d/ceph - backup = etc/udev/rules.d/50-ceph-rbd.rules - backup = etc/sysctl.d/90-ceph-osd.conf backup = etc/sudoers.d/90-ceph -pkgname = ceph-libs +pkgname = ceph-mon + pkgdesc = Ceph Storage cluster monitor daemon, for distributed state storage via PAXOS + depends = ceph-base=18.2.0-1 + depends = bash depends = boost-libs - depends = curl - depends = glibc - depends = keyutils - depends = libutil-linux - depends = bzip2 - depends = lz4 - depends = nss - depends = oath-toolkit + depends = fmt + depends = gperftools + depends = snappy + +pkgname = ceph-mgr + pkgdesc = Ceph Storage cluster manager daemon, the API gateway for cluster management + depends = ceph-base=18.2.0-1 + depends = python-cephfs=18.2.0-1 + depends = python-rbd=18.2.0-1 + depends = libcephsqlite=18.2.0-1 + depends = sqlite depends = python + depends = boost-libs + depends = fmt + depends = gperftools + depends = python-requests + depends = python-typing_extensions + depends = python-pyjwt + depends = python-coverage + depends = python-jinja + depends = python-pyopenssl + depends = python-cherrypy + depends = python-werkzeug + depends = python-prettytable + depends = python-pecan + depends = python-scipy + depends = python-yaml + depends = python-setuptools + depends = python-bcrypt + depends = python-dateutil + depends = python-cheroot + depends = python-urllib3 + depends = python-jsonpatch + depends = python-cryptography + optdepends = cephadm: Required if cluster is managed via cephadm + optdepends = python-kubernetes: For mgr/module:rook,k8sevents + optdepends = python-numpy: For mgr/module:diskprediction_local + optdepends = python-influxdb: For mgr/module:influx + +pkgname = ceph-osd + pkgdesc = Ceph Storage cluster object storage daemon, for managing block devices + depends = ceph-base=18.2.0-1 + depends = fuse3 + depends = bash + depends = boost-libs + depends = fmt + depends = gperftools + depends = libaio depends = snappy depends = systemd-libs + depends = python + optdepends = ceph-volume: For preparing block devices for OSD daemons + optdepends = smartmontools: disk monitoring via S.M.A.R.T + optdepends = nvme-cli: disk monitoring for NVMe drives + backup = etc/sysctl.d/90-ceph-osd.conf + +pkgname = ceph-mds + pkgdesc = Ceph Storage cluster metadata server, the API gateway for CephFS + depends = ceph-base=18.2.0-1 + depends = lua + depends = fmt + depends = gperftools + +pkgname = ceph-rbd + pkgdesc = Ceph Storage utilities and tooling for librbd + depends = librbd=18.2.0-1 + depends = libnl depends = fmt + depends = fuse3 + depends = gperftools + depends = boost-libs + depends = bash depends = cryptsetup + backup = etc/udev/rules.d/50-ceph-rbd.rules + +pkgname = ceph-cephfs + pkgdesc = Ceph Storage utilities and tooling for libcephfs + depends = libcephfs=18.2.0-1 + depends = fuse3 + depends = fmt + depends = gperftools + depends = libcap-ng depends = lua - depends = librdkafka - depends = libaio + depends = python + optdepends = cephfs-shell: Shell access to a CephFS filesystem + optdepends = cephfs-top: Usage and metrics for CephFS, inspired by top(1) + +pkgname = ceph-rgw + pkgdesc = Ceph Storage cluster RADOS Object Gateway daemon, for serving RESTful traffic + depends = librgw=18.2.0-1 + depends = gawk + depends = oath-toolkit + depends = boost-libs + depends = expat depends = gperftools depends = librabbitmq-c - depends = python-prettytable - depends = python-yaml - depends = python-setuptools - provides = ceph-libs=17.2.6-4 - conflicts = ceph-libs-bin + depends = librdkafka + depends = lua -pkgname = ceph-mgr - depends = ceph=17.2.6-4 - depends = ceph-libs=17.2.6-4 +pkgname = librados + pkgdesc = Ceph Storage client library to the RADOS distributed object store + depends = ceph-common=18.2.0-1 depends = bash depends = boost-libs - depends = curl + depends = fmt + depends = lua + depends = oath-toolkit + provides = libradosstriper.so + provides = librados.so + +pkgname = libcephfs + pkgdesc = Ceph Storage client library for CephFS, a distributed POSIX filesystem + depends = librados=18.2.0-1 + depends = fmt + provides = libcephfs.so + +pkgname = librbd + pkgdesc = Ceph Storage client library for RADOS block devices + depends = librados=18.2.0-1 + depends = cryptsetup + depends = fmt + provides = librbd.so + provides = libceph_librbd_parent_cache.so + +pkgname = librgw + pkgdesc = Ceph Storage client library to RADOS Object Gateway, a S3 and Swift compatible REST API + depends = librados=18.2.0-1 + depends = librabbitmq-c + depends = lua + depends = librdkafka + depends = expat + depends = boost-libs depends = gperftools - depends = nss + provides = librgw.so + +pkgname = libcephsqlite + pkgdesc = Ceph Storage client library for a RADOS backed sqlite3 VFS extension + depends = librados=18.2.0-1 depends = fmt + provides = libcephsqlite.so + +pkgname = python-ceph-common + pkgdesc = Ceph Storage python module for common classes, objects and types + depends = ceph-common=18.2.0-1 depends = python - depends = python-cherrypy - depends = python-pecan - depends = python-pyjwt - depends = python-more-itertools - depends = python-numpy - depends = python-scipy - depends = python-six - depends = python-coverage - depends = python-pytest - depends = python-dateutil + depends = python-setuptools depends = python-prettytable - depends = python-requests - depends = python-pyopenssl - depends = python-bcrypt depends = python-yaml - depends = python-werkzeug - depends = python-jinja - optdepends = python-influxdb: influx module - optdepends = python-kubernetes: rook module - optdepends = python-prometheus_client: prometheus module - optdepends = python-remoto: ssh module - provides = ceph-mgr=17.2.6-4 - conflicts = ceph<16.2.1-1 - conflicts = ceph-mgr-bin + +pkgname = python-rados + pkgdesc = Ceph Storage python library for librados + depends = python-ceph-common=18.2.0-1 + depends = librados=18.2.0-1 + depends = libxcrypt + +pkgname = python-rbd + pkgdesc = Ceph Storage python library for librbd + depends = python-ceph-common=18.2.0-1 + depends = python-rados=18.2.0-1 + depends = librbd=18.2.0-1 + depends = libxcrypt + +pkgname = python-cephfs + pkgdesc = Ceph Storage python library for libcephfs + depends = python-ceph-common=18.2.0-1 + depends = python-rados=18.2.0-1 + depends = libcephfs=18.2.0-1 + depends = libxcrypt + +pkgname = python-rgw + pkgdesc = Ceph Storage python library for librgw + depends = python-ceph-common=18.2.0-1 + depends = python-rados=18.2.0-1 + depends = librgw=18.2.0-1 + depends = libxcrypt + +pkgname = cephfs-top + pkgdesc = Ceph Storage utility for a top(1) inspired curses TUI for CephFS metrics + depends = python-ceph-common=18.2.0-1 + depends = python-cephfs=18.2.0-1 + depends = python + +pkgname = cephfs-shell + pkgdesc = Ceph Storage utility for accessing a CephFS filesystem shell + depends = python-ceph-common=18.2.0-1 + depends = python-cephfs=18.2.0-1 + depends = python + depends = python-cmd2 + depends = python-colorama + +pkgname = java-cephfs + pkgdesc = Ceph Storage JNI bindings for CephFS + depends = libcephfs=18.2.0-1 + depends = java-runtime + +pkgname = ceph + pkgdesc = Ceph Storage full install [VIRTUAL] + depends = ceph-libs=18.2.0-1 + depends = ceph-cluster=18.2.0-1 + depends = ceph-rados=18.2.0-1 + depends = ceph-rbd=18.2.0-1 + depends = ceph-cephfs=18.2.0-1 + depends = ceph-tools=18.2.0-1 + depends = ceph-test=18.2.0-1 + depends = python-rados=18.2.0-1 + depends = python-rbd=18.2.0-1 + depends = python-cephfs=18.2.0-1 + depends = python-rgw=18.2.0-1 + +pkgname = ceph-libs + pkgdesc = Ceph Storage client libraries [VIRTUAL] + depends = librados=18.2.0-1 + depends = librbd=18.2.0-1 + depends = libcephfs=18.2.0-1 + depends = librgw=18.2.0-1 + depends = libcephsqlite=18.2.0-1 + +pkgname = ceph-cluster + pkgdesc = Ceph Storage cluster daemons and components [VIRTUAL] + depends = ceph-mon=18.2.0-1 + depends = ceph-mgr=18.2.0-1 + depends = ceph-osd=18.2.0-1 + depends = ceph-mds=18.2.0-1 + depends = ceph-rgw=18.2.0-1 + depends = ceph-volume=18.2.0-1 + +pkgname = ceph-cli + pkgdesc = Ceph Storage CLI utility [VIRTUAL] + depends = ceph-base=18.2.0-1 @@ -3,41 +3,49 @@ # Contributor: Sébastien "Seblu" Luttringer <seblu@archlinux.org> pkgbase='ceph' -pkgname=('ceph' 'ceph-libs' 'ceph-mgr') -pkgver=17.2.6 -pkgrel=4 pkgdesc='Distributed, fault-tolerant storage platform delivering object, block, and file system' -arch=('x86_64') +pkgver=18.2.0 +pkgrel=1 url='https://ceph.com/' +arch=('x86_64') license=('GPL') -makedepends=('zstd' 'bash' 'bc' 'boost' 'boost-libs' 'bzip2' 'c-ares' 'cmake' 'coreutils' 'coffeescript' - 'cpio' 'crypto++' 'cryptsetup' 'cunit' 'curl' 'cython' 'expat' - 'fcgi' 'fontconfig' 'fuse2' 'fuse3' 'fmt' 'gcc' 'gcc-libs' 'git' 'glibc' 'gmock' 'gnutls' - 'gperf' 'gperftools' 'gptfdisk' 'gtest' 'hwloc' 'inetutils' 'java-runtime' - 'jq' 'jre11-openjdk-headless' 'junit' 'keyutils' 'leveldb' 'libaio' - 'libatomic_ops' 'libcap' 'libcap-ng' 'libcroco' 'libcurl-compat' - 'libedit' 'libgudev' 'libnl' 'librabbitmq-c' 'libtool' 'util-linux' - 'libuv' 'libxml2' 'librdkafka' 'libpciaccess' 'lsb-release' 'lua' 'lz4' 'ncurses' - 'nss' 'numactl' 'oath-toolkit' 'openssl' 'parted' 'pcre' 'pcre2' 'pkgconf' 'protobuf' - 'procps-ng' 'python-astroid' 'python-attrs' 'python-bcrypt' - 'python-cheroot' 'python-cherrypy' 'python-coverage' 'python-dateutil' - 'python-elasticsearch' 'python-flask' 'python-flask-restful' - 'python-google-api-python-client' 'python-google-auth' - 'python-google-auth-httplib2' 'python-grpcio' 'python-isort' - 'python-jinja' 'python-lazy-object-proxy' 'python-mccabe' - 'python-isodate' 'python-defusedxml' 'python-pkgconfig' 'python-protobuf' - 'python-lxml' 'python-xmlsec' 'python-yaml' - 'python-more-itertools' 'python-numpy' 'python-pbr' 'python-pecan' - 'python-pip' 'python-pluggy' 'python-portend' 'python-prettytable' - 'python-prometheus_client' 'python-py' 'python-pycparser' - 'python-pyjwt' 'python-pyopenssl' 'python-pytz' 'python-requests' - 'python-routes' 'python-scikit-learn' 'python-scipy' - 'python-setuptools' 'python-six' 'python-sphinx' 'python-tempora' - 'python-virtualenv' 'python-werkzeug' 'python-wrapt' 'rabbitmq' - 'sed' 'snappy' 'socat' 'systemd' 'systemd-libs' 'valgrind' - 'xfsprogs' 'xmlstarlet' 'xmlsec' 'xxhash' 'yaml-cpp' 'yasm' 'zlib' ) -checkdepends=('python-mock' 'python-nose' 'python-pycodestyle' 'python-pylint' - 'python-pytest' 'python-pytest-cov') +pkgname=( + ceph-{common,compressor,crypto,erasure,tools,test,volume,cephadm} + ceph-{rados,base,mon,mgr,osd,mds,rbd,cephfs,rgw} + lib{rados,cephfs,rbd,rgw,cephsqlite} + python-{ceph-common,rados,rbd,cephfs,rgw} + cephfs-{top,shell} + java-cephfs + + ceph + ceph-libs + ceph-cluster + ceph-cli +) +makedepends=( + 'bash' 'boost' 'boost-libs' 'cmake' 'coreutils' + 'cryptsetup' 'curl' 'cython' 'expat' 'fmt' + 'fuse3' 'gawk' 'gcc-libs' 'git' 'glibc' + 'gperf' 'gperftools' 'java-runtime' 'jq' 'jre11-openjdk-headless' + 'junit' 'keyutils' 'libaio' 'libatomic_ops' 'libcap' + 'libcap-ng' 'libcurl-compat' 'libedit' 'libgudev' 'libnl' + 'librabbitmq-c' 'librdkafka' 'libutil-linux' 'libuv' 'libxcrypt' + 'lua' 'lz4' 'nss' 'oath-toolkit' 'openssl' + 'pkgconf' 'python' 'snappy' 'sqlite' 'systemd-libs' + 'thrift' 'util-linux' 'xfsprogs' 'zlib' 'zstd' + + 'python-bcrypt' 'python-cherrypy' 'python-coverage' 'python-dateutil' 'python-jinja' + 'python-pecan' 'python-prettytable' 'python-pyjwt' 'python-pyopenssl' 'python-requests' + 'python-scipy' 'python-setuptools' 'python-sphinx' 'python-typing_extensions' 'python-werkzeug' + 'python-yaml' +) +checkdepends=( + 'inetutils' 'xmlstarlet' + + 'python-mock' 'python-nose' 'python-pycodestyle' 'python-pylint' 'python-pytest' + 'python-pytest-cov' +) + # Despite the upstream suggesting that LTO is now possible, I still am unable # to set this. I get SEGVs in tests, and repeated mentions of C++ One Definition Rule # violations in builds -- probably causing the segfaults. Need to look into this some @@ -55,10 +63,6 @@ source=( # typically around cephadm auto-generated rotate rules 'ceph-17.2.5-logrotate-ignore-dups.patch' - # Test improperly creates librados::async_write templates, in boost 1.80 - # not sure why yet, need to ask upstream for help - 'ceph-16.2.7-delete-test-librados-asio.patch' - # Test wants to use `git ls-files`, and is sad when it finds itself not running in a # git repo 'ceph-17.2.4-tox-flake8-git-ls-files.patch' @@ -67,10 +71,6 @@ source=( # See: https://github.com/python/mypy/issues/13627 'ceph-17.2.4-tox-mypy-false-postive.patch' - # Cephadm tests rely on some mock fs abstraction which dies in the build chroot, - # so disable them - 'ceph-17.2.4-tox-cephadm-rm.patch' - # Split up a very IO heavy test suite, as otherwise test is liable to timeout # NOTE: this is a very large patchset and will guarrented break if/when the upstream # touches anything in src/test/objectstore @@ -88,10 +88,6 @@ source=( # https://github.com/boostorg/beast/issues/2594 'ceph-17.2.5-rgw-client-boost-string-view.patch' - # Fix missing includes from gcc 13 changes - 'ceph-17.2.6-rocksdb-gcc13-includes.patch' - 'ceph-17.2.6-ceph-gcc13-includes.patch' - # Fixes node-gyp errors due to py3.11 incompatibilities 'ceph-17.2.6-mgr-dashboard-node-version.patch' @@ -101,34 +97,40 @@ source=( # Fixes inspect.formatargspec errors in pylint2.6->wrapt due to removal in py3.11 'ceph-17.2.6-mgr-dashboard-pylint-217.patch' - # Backports a cmake opt to allow us to use the bundled fmtlib rather than system - # Should be removed before v18 - 'ceph-17.2.6-backport-with-fmt-version.patch' - # Fixes a couple breaking changes from cython v3.0.0 'ceph-17.2.6-cython-fixes.patch' + + # Fixes fmtlib v10 issues found in v18.2.x + 'ceph-18.2.0-fmt10-fixes.patch' + + # Fix undefined behavior in unit test for erasure coding (SHEC) + 'ceph-18.2.0-fix-ecode-shec-test.patch' + + # Backport https://github.com/ceph/ceph/pull/50216 to fix an assertion failure in + # test_concurrent_dir_link_and_compact_log_56210 due to the strange mount/umount/mount + # pattern + 'ceph-18.2.0-backport-log-runway-expansion.patch' ) -sha512sums=('dca9aea2ce210c15fcc34cb06a5dc5b4488ffa36d684166d47ebd87e48b54b6fee0882e1c67007a780e1c25754e9bc6e760cc10f60ea1183263f8504ef2dbd9b' +sha512sums=('fe6a3ca4227cc05b5effc33a02670c466583ec9e4c7c025241248223992254e6e6551d7db486d5ead36bf0ba922d54e0aabc2aaf456d4593679376ec88bd971a' '4354001c1abd9a0c385ba7bd529e3638fb6660b6a88d4e49706d4ac21c81b8e829303a20fb5445730bdac18c4865efb10bc809c1cd56d743c12aa9a52e160049' '41dbc1c395cdf9b3edf5c5d91bbc90f416b4338ad964fa3471f26a4312d3ec2a5dcebbc351a1640dc4b047b4f71aa134ac7486747e5f62980092b0176e7567f5' 'ea069b75b786c22166c609b127b512802cc5c6e9512d792d7b7b34d276f5b86d57c8c35cfc7b5c855a59c0ba87ba1aabe2ca26da72b26bff46b6ba8410ddb27e' '2234d005df71b3b6013e6b76ad07a5791e3af7efec5f41c78eb1a9c92a22a67f0be9560be59b52534e90bfe251bcf32c33d5d40163f3f8f7e7420691f0f4a222' 'b12cabda7184721c494edd22250fd05019694d2bc445722d100cdefab5385bd25c2267a029d2f6053932fa6717e38c4314385afd986969ee2744d745b53c8b58' - '66770a80ba4e05ea72d4809cb5819cce7499ea7523b85b1a57370df68de1d7f6f94b1c10d0f9f9a3c8e6a86d0419434c70778c568cd06a0dd2e6126631a3355c' '31e578b240ceaaf1216b56cdce654661eed6529ef642ecad164a02669e850100a49a85dc70f3d744671e2c5dad10aee64be7d091fa33007cb8fc6788a4336799' - '85c0a00419c292204be03e8962b0d4bc67c5dc21eb19e74897ec88168f19ac93fc2e38c405b212421c419a74923ec8c5c10844e23269225c52937fd2ad40d817' - '6b53e3a2b1941f8dfdb4b83cafc4c500e818dd3a0736d7061f0ebcdfb514094b217e4688384c5b428288181ef6fff0ca73895c65a046e2d6ae2b834375164216' - '81f540c8312972887a7cb43b8a4e29bfc6f24d5774787a4a8edfe65cca7d3b08faa08ecd09066d7ea67111769a5aec7385fe9a969546626f58874dd8aff5b664' + 'f1f549c5da829db787ee6ee3cf47912d5b48efb2533de05fd9735736e9218a78cf79c5958e1409a2c8adcca51460eb3ae7e73b5c1ec6cf73d4f07670c9934c3b' + '5b43ba48dde8b7371d4b3976aa3a27ab3ad3a10e110479a101b8f222590bbd6aca318f3ac117f851596fc23daafb53aaa566e055dc9694ab16ab1170e8f77763' '781a01e622a70d56bf1948bdc0b427ffa95a86cec7dd9d26c6007a9ec024a942a8ca55f2acc3d37344862f1d6bf11cae998d8071754cd841a66bfba4ec9c58bf' - '2a6f33791760e14543c90077bfc6bf1b6b82ee2996e80b4762eadb887a0d9a67c221b6f10832ddf780dc6abaed246a1e2ee7680c9c861c4ff70e61b752a37b36' + '612faebfb5eec3651832f349ea3c23b50d2386889ff77592b0acff653049efdc5c2254f63c30d88b9a730813bf1f1945dda0d0beab0db7db3e0708ba8d057a40' '02ca1a9bf15f9cd6f474f264ba2bf66ae725bac990a7cac315dabe377c66935a7afd8117f18a1f82c78bdf9ab2c3e5a2a227f2ffc166974dd7bb99b290f1f233' - '1897ddfcd21fbbd1f18f304db6fddb5ba9336bbe2be365ece57a3ae8195f075ceb66f40fc37a4b5bf6481e8a87561383108493aa88cb01d78a904c9da28bb36c' - '2971475ed43480802d6053e259e8540475b3da0d2a84edc9df05e822d51f31b25de450f387fb33e1f7034d3139e502e8da1b0b00ccbb066d01091055e880804b' 'd9729c33f2b8c03ab918919e3b62cbc0825314ad09596dc12984ac5964f8b421db3376b84fa26e0952e0781deb218f8aff6ba32f9c8e5a22bd27afac8dcaf6d3' '79be1630ae4a599509e5d789d4aefe412ce47e67ad482f853664fa4b01e063c20593e3da668e6a776ad038fb07606ae948eea41bab20776c33c87f9ab49505e0' 'c767a8e6fd02ea2ab88e99b50b206d0f825acdf177136ded38d93594fc7663b7c9612af7195b85e0b2b501d8ee482af5e088e9abb5ebee7b8a69e0153ce89782' - '6b4d829c9ecacdc8fbbc6d36eb59bcd844544b728121f75101d5078fd87f6531aeafff5de636c325ebcb88eb0da49f9f62f947bf7d7db4be426ab8ddb02996f6' - '0c5124693bd317a73707dfd34b17664cc05233aec08e07739fe08fc9a73be7a1f4446052b1addde832cba141a382c35f45e60c89a00bb7dab81cee7ed6be07e1') + '0c5124693bd317a73707dfd34b17664cc05233aec08e07739fe08fc9a73be7a1f4446052b1addde832cba141a382c35f45e60c89a00bb7dab81cee7ed6be07e1' + '4613232e5a0003c08d233e40fe3ac1cd00e1195d29bdd9892188587b4a782d6979004232927c0a1bff554eabf2fb9b18eb751682b7ad90762292b63891f3b301' + '9a1183c08f8799b14235c9271519203cbf93e48ca3a8607d3a0500910efca5379c8a08421c377227f93d8436a850f5ca99784f28aaa920e55f0457c657511f17' + 'e238b326609636bc7dd10cec59290e22898948ef105c49643c38d2621abf16c2efcf9581b0b6bad65066607510c9827d00a7abdb14f2054701cc33b7101ea054') +__version="${pkgver}-${pkgrel}" # -fno-plt causes linker errors (undefined reference to internal methods) # similar issue: https://bugs.archlinux.org/task/54845 @@ -170,12 +172,6 @@ prepare() { build() { cd "${srcdir}/${pkgbase}-${pkgver}" - # 2022-09-27 fmt>9 has deprecated an API that is used extensively throughout - # the code base. See: - # Upstream: https://tracker.ceph.com/issues/56610 - # Debian: https://salsa.debian.org/ceph-team/ceph/-/merge_requests/9 - CPPFLAGS+=' -DFMT_DEPRECATED_OSTREAM' - export CFLAGS+=" ${CPPFLAGS}" export CXXFLAGS+=" ${CPPFLAGS}" export CMAKE_BUILD_TYPE='RelWithDebInfo' @@ -193,12 +189,20 @@ build() { -DENABLE_GIT_VERSION=ON \ -DWITH_BABELTRACE=OFF \ -DWITH_LTTNG=OFF \ + -DWITH_BLKIN=OFF \ + -DWITH_JAEGER=OFF \ -DWITH_FIO=OFF \ -DWITH_OPENLDAP=OFF \ -DWITH_RDMA=OFF \ -DWITH_OCF=OFF \ -DWITH_DPDK=OFF \ -DWITH_SPDK=OFF \ + -DWITH_QAT=OFF \ + -DWITH_QATZIP=OFF \ + -DWITH_RBD=ON \ + -DWITH_RBD_RWL=ON \ + -DWITH_RBD_SSD_CACHE=ON \ + -DWITH_RBD_MIRROR=ON \ -DWITH_CEPHFS=ON \ -DWITH_CEPHFS_JAVA=ON \ -DWITH_CEPHFS_SHELL=ON \ @@ -215,12 +219,14 @@ build() { -DWITH_RADOSGW_LUA_PACKAGES=ON \ -DWITH_RADOSGW_DBSTORE=OFF \ -DWITH_RADOSGW_SELECT_PARQUET=OFF \ + -DWITH_RADOSGW_MOTR=OFF \ + -DWITH_RADOSGW_DAOS=OFF \ -DWITH_SYSTEMD=ON \ -DWITH_SYSTEM_BOOST=ON \ -DWITH_SYSTEM_ZSTD=ON \ + -DWITH_SYSTEM_UTF8PROC=ON \ -DWITH_SYSTEM_GTEST=OFF \ -DWITH_SYSTEM_NPM=OFF \ - -DWITH_FMT_VERSION="9.0.0" \ -DENABLE_SHARED=ON \ -DWITH_TESTS=ON \ -Wno-dev @@ -242,167 +248,885 @@ check() { done } -package_ceph-libs() { - depends=('boost-libs' 'curl' 'glibc' 'keyutils' 'libutil-linux' 'bzip2' 'lz4' 'nss' - 'oath-toolkit' 'python' 'snappy' 'systemd-libs' 'fmt' 'cryptsetup' - 'lua' 'librdkafka' 'libaio' 'gperftools' 'librabbitmq-c' - 'python-prettytable' 'python-yaml' 'python-setuptools') - provides=("ceph-libs=${pkgver}-${pkgrel}") - conflicts=('ceph-libs-bin') +_package() { + local p="$1" f d; shift + for f in "$@"; do + d="$srcdir/__pkg__/$p/${f#${_staging}/}" + mkdir -p "$(dirname "$d")" + mv -v "$f" "$d" + rmdir -p --ignore-fail-on-non-empty "$(dirname "$f")" + done +} - cd "${srcdir}/${pkgbase}-${pkgver}" +_print() { + ( + cd "$pkgdir" && \ + find . -type f,l -print0 \ + | xargs --null -I@ echo ' > @' + ) +} + +# Real packaging function, make sure to call this in the *first* +# package_() function makepkg calls +# +# Keep in mind that adding _package lines here does nothing unless +# that package is: +# 1. Referenced in pkgname=() +# 2. Has a package_<name>() function defined for it +_make_ceph_packages() { + + # Main install + local install="${pkgdir}/staging" ; mkdir "${install}" + make DESTDIR="${install}" -C "${srcdir}/${pkgbase}-${pkgver}/build" install + + # Clear _package cache + rm -rf "${srcdir}/__pkg__" || : + + ( + cd "${install}" + + pyv=$(python -c 'import sys ; v = sys.version_info ; print("%d.%d" %(v[0], v[1]))') + + lib=usr/lib + inc=usr/include + bin=usr/bin + sbin=usr/sbin + share=usr/share + etc=etc + python=$lib/python${pyv}/site-packages + systemd=$lib/systemd/system + man=$share/man + + ############################################### + # Print install files # + ############################################### + + printf '##### CEPH INSTALL ' ; printf '%0.s#' {1..29} ; printf '\n' + find . | sed -e 's|^\./||' + printf '##### CEPH INSTALL ' ; printf '%0.s#' {1..29} ; printf '\n' + + ############################################### + # Install pruning # + ############################################### + + # Remove 'tests' directories + find . -depth -type d -name 'tests' -exec rm -vr '{}' \+ + + # Remove most common test bins/scripts + find . -depth -type f,l -name '*test*' -exec rm -v '{}' \+ + + # $share/ceph/mgr/test_orchestrator + find . -depth -type d -name 'test_orchestrator' -exec rm -vr '{}' \+ + + # Live test tools + rm -v $bin/ceph_scratchtool{,pp} + rm -rv $bin/ceph_psim + rm -rv $bin/ceph-run $man/man8/ceph-run.8 + rm -rv $bin/ceph_multi_stress_watch + rm -rv $bin/ceph-coverage + + # Old ssh key stuff + rm -vf $share/ceph/*drop.ceph* + + # TODO: Move this into a patch + # Fix EnvironmentFile location in systemd service files + sed -i -e 's|/etc/sysconfig/|/etc/conf.d/|g' $systemd/*.service + + # Fix bash completions path + install -d -m 755 "$share/bash-completion" + mv -v $etc/bash_completion.d $share/bash-completion/completions + + # TODO: Move this into a patch + # Fix sbin dir (cmake opt seems to have no effect) + mv -v $sbin/* $bin/ + rm -vrf $sbin + + ############################################### + # Ceph core libraries # + ############################################### + + _package ceph-compressor \ + $lib/ceph/compressor/* + + _package ceph-crypto \ + $lib/ceph/crypto/* + + _package ceph-erasure \ + $lib/ceph/erasure-code/* + + _package ceph-common \ + $bin/crushtool \ + $bin/ceph-authtool \ + $bin/ceph-conf \ + $lib/ceph/ceph_common.sh \ + $lib/ceph/libceph-common.so.2 \ + $lib/ceph/extblkdev/libceph_ebd_vdo.so \ + $lib/ceph/denc/* \ + $share/doc/ceph/sample.ceph.conf \ + $man/man8/crushtool.8 \ + $man/man8/ceph-{conf,authtool}.8 + + _package librados \ + $inc/rados/* \ + $inc/radosstriper/* \ + $lib/librados.so{,.2,.2.0.0} \ + $lib/libradosstriper.so{,.1,.1.0.0} \ + $lib/rados-classes/* \ + $bin/librados-config \ + $bin/ceph-clsinfo \ + $man/man8/librados-config.8 \ + $man/man8/ceph-clsinfo.8 + + _package ceph-rados \ + $bin/rados \ + $bin/ceph_radosacl \ + $share/bash-completion/completions/rados \ + $man/man8/rados.8 + + _package libcephsqlite \ + $inc/libcephsqlite.h \ + $lib/libcephsqlite.so + + ############################################### + # Ceph cluster components # + ############################################### + + _package ceph-base \ + $bin/ceph \ + $bin/ceph-crash \ + $share/bash-completion/completions/ceph \ + $systemd/ceph-crash.service \ + $systemd/ceph.target \ + $man/man8/ceph.8 + + _package ceph-mon \ + $bin/ceph-mon \ + $bin/ceph-monstore-tool \ + $bin/monmaptool \ + $bin/ceph-create-keys \ + $lib/ceph/ceph-monstore-update-crush.sh \ + $systemd/ceph-mon{.target,@.service} \ + $man/man8/{ceph-mon,monmaptool}.8 \ + $man/man8/ceph-create-keys.8 + + _package ceph-mgr \ + $bin/ceph-mgr \ + $bin/ceph-exporter \ + $systemd/ceph-mgr{.target,@.service} \ + $share/ceph/mgr/* + + _package ceph-osd \ + $bin/ceph-osd \ + $bin/ceph-{osdomap,bluestore}-tool \ + $bin/osdmaptool \ + $bin/crushdiff \ + $bin/ceph-objectstore-tool \ + $lib/ceph/ceph-osd-prestart.sh \ + $systemd/ceph-osd{.target,@.service} \ + $systemd/ceph-volume@.service \ + $man/man8/{ceph-{osd,bluestore-tool},osdmaptool,crushdiff}.8 + + _package ceph-mds \ + $bin/ceph-mds \ + $systemd/ceph-mds{.target,@.service} \ + $man/man8/ceph-mds.8 + + _package ceph-rgw \ + $bin/ceph_rgw_{jsonparser,multiparser} \ + $bin/rgw-* \ + $bin/radosgw* \ + $bin/ceph-diff-sorted \ + $systemd/ceph-radosgw{.target,@.service} \ + $share/bash-completion/completions/radosgw-admin \ + $man/man8/radosgw{,-admin}.8 \ + $man/man8/rgw-orphan-list.8 \ + $man/man8/ceph-diff-sorted.8 \ + $man/man8/rgw-policy-check.8 + + ############################################### + # Ceph clients / applications # + ############################################### + + _package librbd \ + $inc/rbd/* \ + $lib/librbd.so{,.1,.1.18.0} \ + $lib/ceph/librbd/* + + _package ceph-rbd \ + $bin/ceph-rbdnamer \ + $bin/rbd{,map,-{fuse,mirror,nbd,replay,replay-many}} \ + $bin/ceph-immutable-object-cache \ + $systemd/ceph-rbd-mirror{.target,@.service} \ + $systemd/ceph-immutable-object-cache{.target,@.service} \ + $systemd/rbdmap.service \ + $share/bash-completion/completions/rbd \ + $man/man8/ceph-rbdnamer.8 \ + $man/man8/ceph-immutable-object-cache.8 \ + $man/man8/rbd* + + _package libcephfs \ + $inc/cephfs/* \ + $lib/libcephfs.so{,.2,.2.0.0} + + _package ceph-cephfs \ + $bin/cephfs-{data-scan,{journal,table}-tool,mirror} \ + $bin/ceph-fuse \ + $bin/mount.{ceph,fuse.ceph} \ + $bin/ceph-client-debug \ + $systemd/cephfs-mirror{.target,@.service} \ + $systemd/ceph-fuse{.target,@.service} \ + $man/man8/ceph-fuse.8 \ + $man/man8/mount.{ceph,fuse.ceph}.8 \ + $man/man8/cephfs-* - # main install - VERBOSE=1 make DESTDIR="${pkgdir}" -C build install + _package librgw \ + $lib/librgw.so{,.2,.2.0.0} - # remove stuff that goes into the ceph package - rm -rf "${pkgdir}"/usr/lib/{ceph/mgr,systemd,sysusers.d,tmpfiles.d} - rm -rf "${pkgdir}/usr/share" - rm -rf "${pkgdir}/usr/sbin" - rm -rf "${pkgdir}/usr/bin" - rm -rf "${pkgdir}/etc" - rm -rf "${pkgdir}/var" + _package ceph-cephadm \ + $bin/cephadm \ + $man/man8/cephadm.8 - # Remove misc. test files - find "${pkgdir}" -depth -type d -name 'tests' -exec rm -vr '{}' \+ + _package ceph-volume \ + $bin/ceph-volume{,-systemd} \ + $python/ceph_volume \ + $python/ceph_volume-*egg-info* \ + $man/man8/ceph-volume{,-systemd}.8 + + _package cephfs-shell \ + $bin/cephfs-shell \ + $python/cephfs_shell-* + + _package cephfs-top \ + $bin/cephfs-top \ + $python/cephfs_top-* + + ############################################### + # Ceph misc. utils # + ############################################### + + _package ceph-tools \ + $bin/ceph-post-file \ + $bin/ceph-dedup-tool \ + $bin/ceph-erasure-code-tool \ + $bin/ceph-kvstore-tool \ + $bin/ceph-debugpack \ + $bin/ceph-dencoder \ + $man/man8/ceph-{post-file,dencoder}.8 \ + $man/man8/ceph-{debugpack,kvstore-tool}.8 + + _package ceph-test \ + $bin/ceph_perf_local \ + $bin/ceph_perf_msgr_client \ + $bin/ceph_perf_msgr_server \ + $bin/ceph_erasure_code_benchmark \ + $bin/ceph_objectstore_bench \ + $bin/ceph_bench_log \ + $bin/ceph_perf_objectstore \ + $bin/ceph_omapbench \ + $bin/ceph-syn \ + $man/man8/ceph-syn.8 + + _package java-cephfs \ + $lib/libcephfs_jni.so{,.1,.1.0.0} \ + $share/java/libcephfs.jar + + ############################################### + # Ceph python packages # + ############################################### + + _package python-ceph-common \ + $python/ceph \ + $python/ceph-*egg-info* \ + $python/ceph_argparse.py \ + $python/ceph_daemon.py + + _package python-rados \ + $python/rados-* \ + $python/rados.cpython-* + + _package python-rbd \ + $python/rbd-* \ + $python/rbd.cpython-* + + _package python-cephfs \ + $python/cephfs-* \ + $python/cephfs.cpython-* + + _package python-rgw \ + $python/rgw-* \ + $python/rgw.cpython-* + + ) + + local -i _ret=$(find "${install}" -type f,l | wc -l) + if (( _ret > 0 )) ; then + echo "[ERROR] Files were found after packaging! Bailing out!" + echo " Please ensure all files are either moved or" + echo " deleted before package_ceph-common() completes." + ( cd "${install}" && find . -type f,l | sed -e 's|^\./||' ) + return 1 + else + rm -rf "${install}" + fi + + return 0 } -package_ceph() { - depends=("ceph-libs=${pkgver}-${pkgrel}" - 'boost-libs' 'curl' 'fuse3' 'fmt' 'glibc' 'gperftools' 'java-runtime' - 'keyutils' 'libaio' 'libutil-linux' 'librdkafka' 'cryptsetup' 'libnl' - 'ncurses' - 'nss' 'oath-toolkit' 'python' - 'snappy' 'sudo' 'systemd-libs' 'lua' 'gawk') - backup=('etc/logrotate.d/ceph' - 'etc/udev/rules.d/50-ceph-rbd.rules' - 'etc/sysctl.d/90-ceph-osd.conf' - 'etc/sudoers.d/90-ceph') - optdepends=('smartmontools: disk monitoring via S.M.A.R.T' - 'nvme-cli: disk monitoring for NVMe drives') - provides=("ceph=${pkgver}-${pkgrel}") - conflicts=('ceph-bin') +############################################### +# Ceph core libraries # +############################################### - cd "${srcdir}/${pkgbase}-${pkgver}" +package_ceph-common() { + pkgdesc='Ceph Storage common libraries and dependencies' + depends=( + "ceph-compressor=${__version}" "ceph-crypto=${__version}" "ceph-erasure=${__version}" + + 'boost-libs' 'curl' 'glibc' 'keyutils' 'libutil-linux' + 'nss' 'systemd-libs' 'bash' 'fmt' 'cryptsetup' + 'libxcrypt' 'libaio' 'libcap' 'gperftools' + ) + provides=( + 'libceph-common.so' 'libceph_ebd_vdo.so' + ) + + #============================================# + # This section needs to be run in the first # + # package function makepkg calls to populate # + # the contents of each package function. # + _make_ceph_packages + #============================================# - # main install - VERBOSE=1 make DESTDIR="${pkgdir}" -C build install + mv __pkg__/$pkgname/* "$pkgdir" - # fix sbin dir (cmake opt seems to have no effect) - mv "${pkgdir}"/usr/sbin/* "${pkgdir}/usr/bin/" - rm -rf "${pkgdir}/usr/sbin" + # sysusers + install -Dm644 "${srcdir}/ceph.sysusers" \ + "${pkgdir}/usr/lib/sysusers.d/${pkgbase}.conf" - # remove stuff that is in the ceph-libs package - find "${pkgdir}/usr/lib" -maxdepth 1 -type f -delete - find "${pkgdir}/usr/lib" -maxdepth 1 -type l -delete - find "${pkgdir}/usr/lib/ceph" -maxdepth 1 -type f -delete - find "${pkgdir}/usr/lib/ceph" -maxdepth 1 -type l -delete - rm -rf "${pkgdir}"/usr/lib/{ceph/{compressor,crypto,erasure-code,librbd,denc},rados-classes} - rm -rf "${pkgdir}"/usr/lib/python* - rm -rf "${pkgdir}/usr/include" + # Prepare conf dir + install -D -d -m755 -o 0 -g 340 "${pkgdir}/etc/ceph" - # remove stuff that is in the ceph-mgr package - rm -rf "${pkgdir}"/usr/{bin/ceph-mgr,share/ceph/mgr,lib/systemd/system/ceph-mgr*} + _print +} + +package_ceph-compressor() { + pkgdesc='Ceph Storage compressor libs' + depends=( + 'gcc-libs' 'lz4' 'snappy' 'zlib' 'zstd' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_ceph-crypto() { + pkgdesc='Ceph Storage crypto libs' + depends=( + 'gcc-libs' 'openssl' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} - # remove _test_ binaries from the package, not needed - find "${pkgdir}/usr/bin" -maxdepth 1 -type f -iname 'ceph_test_*' -delete +package_ceph-erasure() { + pkgdesc='Ceph Storage erasure coding libs' + depends=( + 'gcc-libs' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_librados() { + pkgdesc='Ceph Storage client library to the RADOS distributed object store' + depends=( + "ceph-common=${__version}" + + 'bash' 'boost-libs' 'fmt' 'lua' 'oath-toolkit' + ) + provides=( + 'libradosstriper.so' 'librados.so' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_ceph-rados() { + pkgdesc='Ceph Storage utilities and tools for librados' + depends=( + "librados=${__version}" + + 'gcc-libs' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_libcephsqlite() { + pkgdesc='Ceph Storage client library for a RADOS backed sqlite3 VFS extension' + depends=( + "librados=${__version}" + + 'fmt' + ) + provides=( + 'libcephsqlite.so' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +############################################### +# Ceph cluster components # +############################################### + +package_ceph-base() { + pkgdesc='Ceph Storage cluster base utilities and configuration' + depends=( + "ceph-common=${__version}" "librados=${__version}" "python-ceph-common=${__version}" "python-rados=${__version}" + + 'python' + ) + backup=( + 'etc/logrotate.d/ceph' + 'etc/sudoers.d/90-ceph' + ) + + mv __pkg__/$pkgname/* "$pkgdir" - # install system config files # tmpfiles install -Dm644 "${srcdir}/${pkgbase}-${pkgver}/systemd/ceph.tmpfiles.d" \ "${pkgdir}/usr/lib/tmpfiles.d/${pkgbase}.conf" - # sysusers - install -Dm644 "${srcdir}/ceph.sysusers" \ - "${pkgdir}/usr/lib/sysusers.d/${pkgbase}.conf" # logrotate install -Dm644 "${srcdir}/${pkgbase}-${pkgver}/src/logrotate.conf" \ "${pkgdir}/etc/logrotate.d/ceph" - # rbd udev rules - install -Dm644 "${srcdir}/${pkgbase}-${pkgver}/udev/50-rbd.rules" \ - "${pkgdir}/etc/udev/rules.d/50-ceph-rbd.rules" - # sysctls - sed -i -e '/kernel.pid_max/d' "${srcdir}/${pkgbase}-${pkgver}/etc/sysctl/90-ceph-osd.conf" - install -Dm644 "${srcdir}/${pkgbase}-${pkgver}/etc/sysctl/90-ceph-osd.conf" \ - "${pkgdir}/etc/sysctl.d/90-ceph-osd.conf" # sudoers (for disk monitoring) - install -Dm644 "${srcdir}/ceph.sudoers" \ + install -Dm640 "${srcdir}/ceph.sudoers" \ "${pkgdir}/etc/sudoers.d/90-ceph" + chmod 750 "${pkgdir}/etc/sudoers.d" + + # Prepare log, state dirs + install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/log/ceph" + install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph" - # remove debian init script - rm -rf "${pkgdir}/etc/init.d" + _print +} - # remove drop.ceph.com ssh stuff - rm -f "${pkgdir}"/usr/share/ceph/id_rsa_drop.ceph.com - rm -f "${pkgdir}"/usr/share/ceph/id_rsa_drop.ceph.com.pub - rm -f "${pkgdir}"/usr/share/ceph/known_hosts_drop.ceph.com +package_ceph-mon() { + pkgdesc='Ceph Storage cluster monitor daemon, for distributed state storage via PAXOS' + depends=( + "ceph-base=${__version}" - # fix bash completions path - install -d -m 755 "${pkgdir}/usr/share/bash-completion" - mv "${pkgdir}"/{etc/bash_completion.d,usr/share/bash-completion/completions} + 'bash' 'boost-libs' 'fmt' 'gperftools' 'snappy' + ) - # fix EnvironmentFile location in systemd service files - sed -i 's|/etc/sysconfig/|/etc/conf.d/|g' "${pkgdir}"/usr/lib/systemd/system/*.service + mv __pkg__/$pkgname/* "$pkgdir" - # prepare some paths and set correct permissions - install -D -d -m755 -o 0 -g 340 "${pkgdir}/etc/ceph" - install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/log/ceph" + # Prepare state dir install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph" - install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph/bootstrap-mds" - install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph/bootstrap-osd" - install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph/bootstrap-rgw" - install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph/mon" - install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph/osd" + install -D -d -m750 -o 340 -g 340 "${pkgdir}/var/lib/ceph/mon" + + _print } package_ceph-mgr() { - depends=("ceph=${pkgver}-${pkgrel}" "ceph-libs=${pkgver}-${pkgrel}" - 'bash' 'boost-libs' 'curl' 'gperftools' 'nss' 'fmt' - 'python' 'python-cherrypy' 'python-pecan' 'python-pyjwt' 'python-more-itertools' - 'python-numpy' 'python-scipy' 'python-six' 'python-coverage' 'python-pytest' 'python-dateutil' - 'python-prettytable' 'python-requests' 'python-pyopenssl' 'python-bcrypt' 'python-yaml' - 'python-werkzeug' 'python-jinja') - optdepends=('python-influxdb: influx module' - 'python-kubernetes: rook module' - 'python-prometheus_client: prometheus module' - 'python-remoto: ssh module') - provides=("ceph-mgr=${pkgver}-${pkgrel}") - conflicts=('ceph<16.2.1-1' 'ceph-mgr-bin') + pkgdesc='Ceph Storage cluster manager daemon, the API gateway for cluster management' + depends=( + "ceph-base=${__version}" "python-cephfs=${__version}" "python-rbd=${__version}" "libcephsqlite=${__version}" - cd "${srcdir}/${pkgbase}-${pkgver}" + 'sqlite' 'python' 'boost-libs' 'fmt' 'gperftools' + + 'python-requests' 'python-typing_extensions' 'python-pyjwt' 'python-coverage' 'python-jinja' + 'python-pyopenssl' 'python-cherrypy' 'python-werkzeug' 'python-prettytable' 'python-pecan' + 'python-scipy' 'python-yaml' 'python-setuptools' 'python-bcrypt' 'python-dateutil' + 'python-cheroot' 'python-urllib3' 'python-jsonpatch' 'python-cryptography' + + ) + optdepends=( + 'cephadm: Required if cluster is managed via cephadm' + 'python-kubernetes: For mgr/module:rook,k8sevents' + 'python-numpy: For mgr/module:diskprediction_local' + 'python-influxdb: For mgr/module:influx' + ) + + # Prepare state dir + install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph" + install -D -d -m750 -o 340 -g 340 "${pkgdir}/var/lib/ceph/mgr" + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_ceph-osd() { + pkgdesc='Ceph Storage cluster object storage daemon, for managing block devices' + depends=( + "ceph-base=${__version}" + + 'fuse3' 'bash' 'boost-libs' 'fmt' 'gperftools' + 'libaio' 'snappy' 'systemd-libs' 'python' + ) + optdepends=( + 'ceph-volume: For preparing block devices for OSD daemons' + 'smartmontools: disk monitoring via S.M.A.R.T' + 'nvme-cli: disk monitoring for NVMe drives' + ) + backup=( + 'etc/sysctl.d/90-ceph-osd.conf' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + + # Prepare state dirs + install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph" + install -D -d -m750 -o 340 -g 340 "${pkgdir}/var/lib/ceph/bootstrap-osd" + install -D -d -m750 -o 340 -g 340 "${pkgdir}/var/lib/ceph/osd" + + # sysctls + sed -i -e '/kernel.pid_max/d' \ + "${srcdir}/${pkgbase}-${pkgver}/etc/sysctl/90-ceph-osd.conf" + install -Dm644 "${srcdir}/${pkgbase}-${pkgver}/etc/sysctl/90-ceph-osd.conf" \ + "${pkgdir}/etc/sysctl.d/90-ceph-osd.conf" + + _print +} + +package_ceph-mds() { + pkgdesc='Ceph Storage cluster metadata server, the API gateway for CephFS' + depends=( + "ceph-base=${__version}" + + 'lua' 'fmt' 'gperftools' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + + # Prepare state dirs + install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph" + install -D -d -m750 -o 340 -g 340 "${pkgdir}/var/lib/ceph/bootstrap-mds" + install -D -d -m750 -o 340 -g 340 "${pkgdir}/var/lib/ceph/mds" + + _print +} + +package_ceph-rgw() { + pkgdesc='Ceph Storage cluster RADOS Object Gateway daemon, for serving RESTful traffic' + depends=( + "librgw=${__version}" + + 'gawk' 'oath-toolkit' 'boost-libs' 'expat' 'gperftools' + 'librabbitmq-c' 'librdkafka' 'lua' + ) + + # Prepare state dirs + install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph" + install -D -d -m750 -o 340 -g 340 "${pkgdir}/var/lib/ceph/bootstrap-rgw" + install -D -d -m750 -o 340 -g 340 "${pkgdir}/var/lib/ceph/rgw" + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +############################################### +# Ceph clients / applications # +############################################### + +package_librbd() { + pkgdesc='Ceph Storage client library for RADOS block devices' + depends=( + "librados=${__version}" + + 'cryptsetup' 'fmt' + ) + provides=( + 'librbd.so' 'libceph_librbd_parent_cache.so' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_ceph-rbd() { + pkgdesc='Ceph Storage utilities and tooling for librbd' + depends=( + "librbd=${__version}" + + 'libnl' 'fmt' 'fuse3' 'gperftools' 'boost-libs' + 'bash' 'cryptsetup' + + ) + backup=( + 'etc/udev/rules.d/50-ceph-rbd.rules' + ) + + # rbd udev rules + install -Dm644 "${srcdir}/${pkgbase}-${pkgver}/udev/50-rbd.rules" \ + "${pkgdir}/etc/udev/rules.d/50-ceph-rbd.rules" + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_libcephfs() { + pkgdesc='Ceph Storage client library for CephFS, a distributed POSIX filesystem' + depends=( + "librados=${__version}" + + 'fmt' + ) + provides=( + 'libcephfs.so' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_ceph-cephfs() { + pkgdesc='Ceph Storage utilities and tooling for libcephfs' + depends=( + "libcephfs=${__version}" + + 'fuse3' 'fmt' 'gperftools' 'libcap-ng' 'lua' 'python' + ) + optdepends=( + "cephfs-shell: Shell access to a CephFS filesystem" + "cephfs-top: Usage and metrics for CephFS, inspired by top(1)" + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_librgw() { + pkgdesc='Ceph Storage client library to RADOS Object Gateway, a S3 and Swift compatible REST API' + depends=( + "librados=${__version}" - # main install - VERBOSE=1 make DESTDIR="${pkgdir}" -C build install + 'librabbitmq-c' 'lua' 'librdkafka' 'expat' 'boost-libs' 'gperftools' + ) + provides=( + 'librgw.so' + ) - # fix sbin dir (cmake opt seems to have no effect) - mv "${pkgdir}"/usr/sbin/* "${pkgdir}/usr/bin/" - rm -rf "${pkgdir}/usr/sbin" + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_ceph-volume() { + pkgdesc='Ceph Storage utility for preparing block devices for use as OSDs' + depends=( + "python-ceph-common=${__version}" "ceph-osd=${__version}" + + 'python' 'lvm2' + + 'python-setuptools' 'python-importlib-metadata' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_cephfs-shell() { + pkgdesc='Ceph Storage utility for accessing a CephFS filesystem shell' + depends=( + "python-ceph-common=${__version}" "python-cephfs=${__version}" + + 'python' 'python-cmd2' 'python-colorama' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_cephfs-top() { + pkgdesc='Ceph Storage utility for a top(1) inspired curses TUI for CephFS metrics' + depends=( + "python-ceph-common=${__version}" "python-cephfs=${__version}" + + 'python' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_ceph-cephadm() { + pkgdesc='Ceph Storage adminstration and configuration utility' + provides=('cephadm') + conflicts=('cephadm') + depends=( + 'python' 'podman' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +############################################### +# Ceph misc. utils # +############################################### + +package_ceph-tools() { + pkgdesc='Ceph Storage miscellaneous tooling and utilities' + depends=( + "ceph-base=${__version}" + + 'bash' 'boost-libs' 'gperftools' 'libaio' 'libcap' + 'snappy' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_ceph-test() { + pkgdesc='Ceph Storage tools for benchmarking and testing live clusters' + depends=( + "ceph-base=${__version}" + + 'libcap' 'libaio' 'boost-libs' 'fmt' 'gperftools' + 'snappy' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_java-cephfs() { + pkgdesc='Ceph Storage JNI bindings for CephFS' + depends=( + "libcephfs=${__version}" + + 'java-runtime' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} - # remove everything except mgr related stuff, rest is in ceph/ceph-libs - rm -rf "${pkgdir}"/usr/lib/{ceph/{compressor,crypto,erasure-code,librbd,denc},rados-classes} - rm -rf "${pkgdir}/usr/include" - find "${pkgdir}/usr/bin" -maxdepth 1 -type f -not -name 'ceph-mgr' -delete - find "${pkgdir}"/usr/lib/systemd/system -maxdepth 1 -type f -not -iname 'ceph-mgr*' -delete - find "${pkgdir}"/usr/lib -maxdepth 1 -type f -delete - find "${pkgdir}"/usr/lib -maxdepth 1 -type l -delete - rm -rf "${pkgdir}"/etc - rm -rf "${pkgdir}"/var - rm -rf "${pkgdir}"/usr/lib/{ceph,sysusers.d,tmpfiles.d} - rm -rf "${pkgdir}"/usr/lib/python* - rm -rf "${pkgdir}"/usr/share/{bash-completion,doc,java,man} +############################################### +# Ceph python packages # +############################################### - # remove debian init script - rm -rf "${pkgdir}/etc/init.d" +package_python-ceph-common() { + pkgdesc='Ceph Storage python module for common classes, objects and types' + depends=( + "ceph-common=${__version}" - # remove drop.ceph.com ssh stuff - rm -f "${pkgdir}"/usr/share/ceph/id_rsa_drop.ceph.com - rm -f "${pkgdir}"/usr/share/ceph/id_rsa_drop.ceph.com.pub - rm -f "${pkgdir}"/usr/share/ceph/known_hosts_drop.ceph.com + 'python' - # fix EnvironmentFile location in systemd service files - sed -i 's|/etc/sysconfig/|/etc/conf.d/|g' "${pkgdir}"/usr/lib/systemd/system/*.service + 'python-setuptools' 'python-prettytable' 'python-yaml' + ) - # prepare some paths and set correct permissions - install -D -d -m755 -o 340 -g 340 "${pkgdir}/var/lib/ceph/mgr" + mv __pkg__/$pkgname/* "$pkgdir" + _print } +package_python-rados() { + pkgdesc='Ceph Storage python library for librados' + depends=( + "python-ceph-common=${__version}" "librados=${__version}" + + 'libxcrypt' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_python-rbd() { + pkgdesc='Ceph Storage python library for librbd' + depends=( + "python-ceph-common=${__version}" "python-rados=${__version}" "librbd=${__version}" + + 'libxcrypt' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_python-cephfs() { + pkgdesc='Ceph Storage python library for libcephfs' + depends=( + "python-ceph-common=${__version}" "python-rados=${__version}" "libcephfs=${__version}" + + 'libxcrypt' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +package_python-rgw() { + pkgdesc='Ceph Storage python library for librgw' + depends=( + "python-ceph-common=${__version}" "python-rados=${__version}" "librgw=${__version}" + + 'libxcrypt' + ) + + mv __pkg__/$pkgname/* "$pkgdir" + _print +} + +############################################### +# Ceph virtual targets # +############################################### + +# No package (excluding other virtual targets) should depend on any of these; +# these are convenience targets for other packagers + +package_ceph-libs() { + pkgdesc='Ceph Storage client libraries [VIRTUAL]' + depends=( + "librados=${__version}" + "librbd=${__version}" + "libcephfs=${__version}" + "librgw=${__version}" + "libcephsqlite=${__version}" + ) +} + +package_ceph-cluster() { + pkgdesc='Ceph Storage cluster daemons and components [VIRTUAL]' + depends=( + "ceph-mon=${__version}" + "ceph-mgr=${__version}" + "ceph-osd=${__version}" + "ceph-mds=${__version}" + "ceph-rgw=${__version}" + "ceph-volume=${__version}" + ) +} + +package_ceph-cli() { + pkgdesc='Ceph Storage CLI utility [VIRTUAL]' + depends=("ceph-base=${__version}") +} + +package_ceph() { + pkgdesc='Ceph Storage full install [VIRTUAL]' + depends=( + "ceph-libs=${__version}" + "ceph-cluster=${__version}" + "ceph-rados=${__version}" + "ceph-rbd=${__version}" + "ceph-cephfs=${__version}" + "ceph-tools=${__version}" + "ceph-test=${__version}" + "python-rados=${__version}" + "python-rbd=${__version}" + "python-cephfs=${__version}" + "python-rgw=${__version}" + ) +} + +#======================================================================================# +#======================================================================================# + # vim:set ts=2 sw=2 et: diff --git a/ceph-17.2.4-test-bluefs-split.patch b/ceph-17.2.4-test-bluefs-split.patch index 186cb8841732..e5a693d43627 100644 --- a/ceph-17.2.4-test-bluefs-split.patch +++ b/ceph-17.2.4-test-bluefs-split.patch @@ -82,10 +82,10 @@ index 3294616b9b2..4d1cac3b4fe 100644 add_executable(unittest_bluestore_types test_bluestore_types.cc diff --git a/src/test/objectstore/test_bluefs.cc b/src/test/objectstore/test_bluefs.cc -index 2a84b27c999..4780669181e 100644 +index 4f77d8597ae..04b25842bcd 100644 --- a/src/test/objectstore/test_bluefs.cc +++ b/src/test/objectstore/test_bluefs.cc -@@ -171,6 +171,7 @@ TEST(BlueFS, small_appends) { +@@ -172,6 +172,7 @@ TEST(BlueFS, small_appends) { fs.umount(); } @@ -93,7 +93,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, very_large_write) { // we'll write a ~5G file, so allocate more than that for the whole fs uint64_t size = 1048576 * 1024 * 6ull; -@@ -244,7 +245,9 @@ TEST(BlueFS, very_large_write) { +@@ -245,7 +246,9 @@ TEST(BlueFS, very_large_write) { g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old)); } @@ -103,7 +103,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, very_large_write2) { // we'll write a ~5G file, so allocate more than that for the whole fs uint64_t size_full = 1048576 * 1024 * 6ull; -@@ -297,6 +300,7 @@ TEST(BlueFS, very_large_write2) { +@@ -298,6 +301,7 @@ TEST(BlueFS, very_large_write2) { g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old)); } @@ -111,7 +111,7 @@ index 2a84b27c999..4780669181e 100644 #define ALLOC_SIZE 4096 -@@ -410,6 +414,7 @@ void join_all(std::vector<std::thread>& v) +@@ -411,6 +415,7 @@ void join_all(std::vector<std::thread>& v) #define NUM_SINGLE_FILE_WRITERS 1 #define NUM_MULTIPLE_FILE_WRITERS 2 @@ -119,7 +119,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_flush_1) { uint64_t size = 1048576 * 128; TempBdev bdev{size}; -@@ -443,7 +448,9 @@ TEST(BlueFS, test_flush_1) { +@@ -444,7 +449,9 @@ TEST(BlueFS, test_flush_1) { } fs.umount(); } @@ -129,7 +129,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_flush_2) { uint64_t size = 1048576 * 256; TempBdev bdev{size}; -@@ -470,7 +477,9 @@ TEST(BlueFS, test_flush_2) { +@@ -471,7 +478,9 @@ TEST(BlueFS, test_flush_2) { } fs.umount(); } @@ -139,7 +139,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_flush_3) { uint64_t size = 1048576 * 256; TempBdev bdev{size}; -@@ -504,7 +513,9 @@ TEST(BlueFS, test_flush_3) { +@@ -505,7 +514,9 @@ TEST(BlueFS, test_flush_3) { } fs.umount(); } @@ -149,7 +149,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_simple_compaction_sync) { g_ceph_context->_conf.set_val( "bluefs_compact_log_sync", -@@ -556,7 +567,9 @@ TEST(BlueFS, test_simple_compaction_sync) { +@@ -557,7 +568,9 @@ TEST(BlueFS, test_simple_compaction_sync) { fs.compact_log(); fs.umount(); } @@ -159,7 +159,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_simple_compaction_async) { g_ceph_context->_conf.set_val( "bluefs_compact_log_sync", -@@ -608,7 +621,9 @@ TEST(BlueFS, test_simple_compaction_async) { +@@ -609,7 +622,9 @@ TEST(BlueFS, test_simple_compaction_async) { fs.compact_log(); fs.umount(); } @@ -169,7 +169,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_compaction_sync) { uint64_t size = 1048576 * 128; TempBdev bdev{size}; -@@ -672,7 +687,9 @@ TEST(BlueFS, test_compaction_sync) { +@@ -673,7 +688,9 @@ TEST(BlueFS, test_compaction_sync) { } fs.umount(); } @@ -179,7 +179,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_compaction_async) { uint64_t size = 1048576 * 128; TempBdev bdev{size}; -@@ -736,7 +753,9 @@ TEST(BlueFS, test_compaction_async) { +@@ -737,7 +754,9 @@ TEST(BlueFS, test_compaction_async) { } fs.umount(); } @@ -189,7 +189,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_replay) { uint64_t size = 1048576 * 128; TempBdev bdev{size}; -@@ -777,7 +796,9 @@ TEST(BlueFS, test_replay) { +@@ -778,7 +797,9 @@ TEST(BlueFS, test_replay) { ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); fs.umount(); } @@ -199,7 +199,7 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_replay_growth) { uint64_t size = 1048576LL * (2 * 1024 + 128); TempBdev bdev{size}; -@@ -815,7 +836,9 @@ TEST(BlueFS, test_replay_growth) { +@@ -816,7 +837,9 @@ TEST(BlueFS, test_replay_growth) { ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); fs.umount(); } @@ -209,40 +209,20 @@ index 2a84b27c999..4780669181e 100644 TEST(BlueFS, test_tracker_50965) { uint64_t size_wal = 1048576 * 64; TempBdev bdev_wal{size_wal}; -@@ -881,7 +904,9 @@ TEST(BlueFS, test_tracker_50965) { - - fs.umount(); - } -+*/ - -+/* - TEST(BlueFS, test_truncate_stable_53129) { - - ConfSaver conf(g_ceph_context->_conf); -@@ -966,7 +991,9 @@ TEST(BlueFS, test_truncate_stable_53129) { - - fs.umount(); - } -+*/ - -+/* - TEST(BlueFS, test_update_ino1_delta_after_replay) { - uint64_t size = 1048576LL * (2 * 1024 + 128); - TempBdev bdev{size}; -@@ -1012,6 +1039,7 @@ TEST(BlueFS, test_update_ino1_delta_after_replay) { +@@ -1013,6 +1036,7 @@ TEST(BlueFS, test_update_ino1_delta_after_replay) { ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); fs.umount(); } +*/ - int main(int argc, char **argv) { - auto args = argv_to_vec(argc, argv); + TEST(BlueFS, broken_unlink_fsync_seq) { + uint64_t size = 1048576 * 128; diff --git a/src/test/objectstore/test_bluefs_compact_async.cc b/src/test/objectstore/test_bluefs_compact_async.cc new file mode 100644 -index 00000000000..45939890459 +index 00000000000..dc5bd2b1e53 --- /dev/null +++ b/src/test/objectstore/test_bluefs_compact_async.cc -@@ -0,0 +1,336 @@ +@@ -0,0 +1,338 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -255,13 +235,14 @@ index 00000000000..45939890459 +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -579,12 +560,13 @@ index 00000000000..45939890459 + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} ++ diff --git a/src/test/objectstore/test_bluefs_compact_sync.cc b/src/test/objectstore/test_bluefs_compact_sync.cc new file mode 100644 -index 00000000000..e6f19fb9e53 +index 00000000000..8ac84a2ca67 --- /dev/null +++ b/src/test/objectstore/test_bluefs_compact_sync.cc -@@ -0,0 +1,336 @@ +@@ -0,0 +1,338 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -597,13 +579,14 @@ index 00000000000..e6f19fb9e53 +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -921,12 +904,13 @@ index 00000000000..e6f19fb9e53 + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} ++ diff --git a/src/test/objectstore/test_bluefs_flush_1.cc b/src/test/objectstore/test_bluefs_flush_1.cc new file mode 100644 -index 00000000000..f34533085ba +index 00000000000..92112624c02 --- /dev/null +++ b/src/test/objectstore/test_bluefs_flush_1.cc -@@ -0,0 +1,254 @@ +@@ -0,0 +1,256 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -939,13 +923,14 @@ index 00000000000..f34533085ba +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -1181,12 +1166,13 @@ index 00000000000..f34533085ba + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} ++ diff --git a/src/test/objectstore/test_bluefs_flush_2.cc b/src/test/objectstore/test_bluefs_flush_2.cc new file mode 100644 -index 00000000000..db588dc22a4 +index 00000000000..622ab391dd1 --- /dev/null +++ b/src/test/objectstore/test_bluefs_flush_2.cc -@@ -0,0 +1,247 @@ +@@ -0,0 +1,249 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -1199,13 +1185,14 @@ index 00000000000..db588dc22a4 +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -1434,12 +1421,13 @@ index 00000000000..db588dc22a4 + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} ++ diff --git a/src/test/objectstore/test_bluefs_flush_3.cc b/src/test/objectstore/test_bluefs_flush_3.cc new file mode 100644 -index 00000000000..9e1e4d05ed1 +index 00000000000..ce3fdf7487d --- /dev/null +++ b/src/test/objectstore/test_bluefs_flush_3.cc -@@ -0,0 +1,254 @@ +@@ -0,0 +1,256 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -1452,13 +1440,14 @@ index 00000000000..9e1e4d05ed1 +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -1694,12 +1683,13 @@ index 00000000000..9e1e4d05ed1 + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} ++ diff --git a/src/test/objectstore/test_bluefs_large_write_1.cc b/src/test/objectstore/test_bluefs_large_write_1.cc new file mode 100644 -index 00000000000..2b9bf34acfd +index 00000000000..3ef45c6e6a0 --- /dev/null +++ b/src/test/objectstore/test_bluefs_large_write_1.cc -@@ -0,0 +1,294 @@ +@@ -0,0 +1,183 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -1712,13 +1702,14 @@ index 00000000000..2b9bf34acfd +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -1852,7 +1843,7 @@ index 00000000000..2b9bf34acfd + delete h; + ASSERT_EQ(0, fs.open_for_read("dir", "bigfile", &h)); + ASSERT_EQ(h->file->fnode.size, total_written); -+ unique_ptr<char> huge_buf(new char[h->file->fnode.size]); ++ auto huge_buf = std::make_unique<char[]>(h->file->fnode.size); + auto l = h->file->fnode.size; + int64_t r = fs.read(h, 0, l, NULL, huge_buf.get()); + ASSERT_EQ(r, l); @@ -1863,118 +1854,6 @@ index 00000000000..2b9bf34acfd + g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old)); +} + -+#define ALLOC_SIZE 4096 -+ -+void write_data(BlueFS &fs, uint64_t rationed_bytes) -+{ -+ int j=0, r=0; -+ uint64_t written_bytes = 0; -+ rationed_bytes -= ALLOC_SIZE; -+ stringstream ss; -+ string dir = "dir."; -+ ss << std::this_thread::get_id(); -+ dir.append(ss.str()); -+ dir.append("."); -+ dir.append(to_string(j)); -+ ASSERT_EQ(0, fs.mkdir(dir)); -+ while (1) { -+ string file = "file."; -+ file.append(to_string(j)); -+ BlueFS::FileWriter *h; -+ ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); -+ ASSERT_NE(nullptr, h); -+ auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); -+ bufferlist bl; -+ std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE); -+ bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); -+ bl.push_back(bp); -+ h->append(bl.c_str(), bl.length()); -+ r = fs.fsync(h); -+ if (r < 0) { -+ break; -+ } -+ written_bytes += g_conf()->bluefs_alloc_size; -+ j++; -+ if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { -+ break; -+ } -+ } -+} -+ -+void create_single_file(BlueFS &fs) -+{ -+ BlueFS::FileWriter *h; -+ stringstream ss; -+ string dir = "dir.test"; -+ ASSERT_EQ(0, fs.mkdir(dir)); -+ string file = "testfile"; -+ ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); -+ bufferlist bl; -+ std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE); -+ bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); -+ bl.push_back(bp); -+ h->append(bl.c_str(), bl.length()); -+ fs.fsync(h); -+ fs.close_writer(h); -+} -+ -+void write_single_file(BlueFS &fs, uint64_t rationed_bytes) -+{ -+ stringstream ss; -+ const string dir = "dir.test"; -+ const string file = "testfile"; -+ uint64_t written_bytes = 0; -+ rationed_bytes -= ALLOC_SIZE; -+ while (1) { -+ BlueFS::FileWriter *h; -+ ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); -+ ASSERT_NE(nullptr, h); -+ auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); -+ bufferlist bl; -+ std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE); -+ bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); -+ bl.push_back(bp); -+ h->append(bl.c_str(), bl.length()); -+ int r = fs.fsync(h); -+ if (r < 0) { -+ break; -+ } -+ written_bytes += g_conf()->bluefs_alloc_size; -+ if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { -+ break; -+ } -+ } -+} -+ -+bool writes_done = false; -+ -+void sync_fs(BlueFS &fs) -+{ -+ while (1) { -+ if (writes_done == true) -+ break; -+ fs.sync_metadata(false); -+ sleep(1); -+ } -+} -+ -+ -+void do_join(std::thread& t) -+{ -+ t.join(); -+} -+ -+void join_all(std::vector<std::thread>& v) -+{ -+ std::for_each(v.begin(),v.end(),do_join); -+} -+ -+#define NUM_WRITERS 3 -+#define NUM_SYNC_THREADS 1 -+ -+#define NUM_SINGLE_FILE_WRITERS 1 -+#define NUM_MULTIPLE_FILE_WRITERS 2 -+ +int main(int argc, char **argv) { + auto args = argv_to_vec(argc, argv); + map<string,string> defaults = { @@ -1996,10 +1875,10 @@ index 00000000000..2b9bf34acfd +} diff --git a/src/test/objectstore/test_bluefs_large_write_2.cc b/src/test/objectstore/test_bluefs_large_write_2.cc new file mode 100644 -index 00000000000..4c44353fa15 +index 00000000000..c71db31a6c1 --- /dev/null +++ b/src/test/objectstore/test_bluefs_large_write_2.cc -@@ -0,0 +1,273 @@ +@@ -0,0 +1,163 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -2012,13 +1891,14 @@ index 00000000000..4c44353fa15 +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -2142,118 +2022,6 @@ index 00000000000..4c44353fa15 + g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old)); +} + -+#define ALLOC_SIZE 4096 -+ -+void write_data(BlueFS &fs, uint64_t rationed_bytes) -+{ -+ int j=0, r=0; -+ uint64_t written_bytes = 0; -+ rationed_bytes -= ALLOC_SIZE; -+ stringstream ss; -+ string dir = "dir."; -+ ss << std::this_thread::get_id(); -+ dir.append(ss.str()); -+ dir.append("."); -+ dir.append(to_string(j)); -+ ASSERT_EQ(0, fs.mkdir(dir)); -+ while (1) { -+ string file = "file."; -+ file.append(to_string(j)); -+ BlueFS::FileWriter *h; -+ ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); -+ ASSERT_NE(nullptr, h); -+ auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); -+ bufferlist bl; -+ std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE); -+ bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); -+ bl.push_back(bp); -+ h->append(bl.c_str(), bl.length()); -+ r = fs.fsync(h); -+ if (r < 0) { -+ break; -+ } -+ written_bytes += g_conf()->bluefs_alloc_size; -+ j++; -+ if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { -+ break; -+ } -+ } -+} -+ -+void create_single_file(BlueFS &fs) -+{ -+ BlueFS::FileWriter *h; -+ stringstream ss; -+ string dir = "dir.test"; -+ ASSERT_EQ(0, fs.mkdir(dir)); -+ string file = "testfile"; -+ ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); -+ bufferlist bl; -+ std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE); -+ bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); -+ bl.push_back(bp); -+ h->append(bl.c_str(), bl.length()); -+ fs.fsync(h); -+ fs.close_writer(h); -+} -+ -+void write_single_file(BlueFS &fs, uint64_t rationed_bytes) -+{ -+ stringstream ss; -+ const string dir = "dir.test"; -+ const string file = "testfile"; -+ uint64_t written_bytes = 0; -+ rationed_bytes -= ALLOC_SIZE; -+ while (1) { -+ BlueFS::FileWriter *h; -+ ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false)); -+ ASSERT_NE(nullptr, h); -+ auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); }); -+ bufferlist bl; -+ std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE); -+ bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get()); -+ bl.push_back(bp); -+ h->append(bl.c_str(), bl.length()); -+ int r = fs.fsync(h); -+ if (r < 0) { -+ break; -+ } -+ written_bytes += g_conf()->bluefs_alloc_size; -+ if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) { -+ break; -+ } -+ } -+} -+ -+bool writes_done = false; -+ -+void sync_fs(BlueFS &fs) -+{ -+ while (1) { -+ if (writes_done == true) -+ break; -+ fs.sync_metadata(false); -+ sleep(1); -+ } -+} -+ -+ -+void do_join(std::thread& t) -+{ -+ t.join(); -+} -+ -+void join_all(std::vector<std::thread>& v) -+{ -+ std::for_each(v.begin(),v.end(),do_join); -+} -+ -+#define NUM_WRITERS 3 -+#define NUM_SYNC_THREADS 1 -+ -+#define NUM_SINGLE_FILE_WRITERS 1 -+#define NUM_MULTIPLE_FILE_WRITERS 2 -+ +int main(int argc, char **argv) { + auto args = argv_to_vec(argc, argv); + map<string,string> defaults = { @@ -2273,12 +2041,13 @@ index 00000000000..4c44353fa15 + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} ++ diff --git a/src/test/objectstore/test_bluefs_regression.cc b/src/test/objectstore/test_bluefs_regression.cc new file mode 100644 -index 00000000000..ab50ee86fd9 +index 00000000000..9588ffc7c56 --- /dev/null +++ b/src/test/objectstore/test_bluefs_regression.cc -@@ -0,0 +1,417 @@ +@@ -0,0 +1,419 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -2291,13 +2060,14 @@ index 00000000000..ab50ee86fd9 +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -2696,12 +2466,13 @@ index 00000000000..ab50ee86fd9 + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} ++ diff --git a/src/test/objectstore/test_bluefs_replay.cc b/src/test/objectstore/test_bluefs_replay.cc new file mode 100644 -index 00000000000..d050a68343e +index 00000000000..be87affdcd1 --- /dev/null +++ b/src/test/objectstore/test_bluefs_replay.cc -@@ -0,0 +1,261 @@ +@@ -0,0 +1,263 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -2714,13 +2485,14 @@ index 00000000000..d050a68343e +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -2963,12 +2735,13 @@ index 00000000000..d050a68343e + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} ++ diff --git a/src/test/objectstore/test_bluefs_replay_growth.cc b/src/test/objectstore/test_bluefs_replay_growth.cc new file mode 100644 -index 00000000000..01e3ecd88b1 +index 00000000000..a02b8e185ef --- /dev/null +++ b/src/test/objectstore/test_bluefs_replay_growth.cc -@@ -0,0 +1,259 @@ +@@ -0,0 +1,260 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + @@ -2981,13 +2754,14 @@ index 00000000000..01e3ecd88b1 +#include <random> +#include <thread> +#include <stack> ++#include <gtest/gtest.h> +#include "global/global_init.h" +#include "common/ceph_argparse.h" +#include "include/stringify.h" +#include "include/scope_guard.h" +#include "common/errno.h" -+#include <gtest/gtest.h> + ++#include "os/bluestore/Allocator.h" +#include "os/bluestore/BlueFS.h" + +using namespace std; @@ -3228,3 +3002,4 @@ index 00000000000..01e3ecd88b1 + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} ++ diff --git a/ceph-17.2.4-tox-cephadm-rm.patch b/ceph-17.2.4-tox-cephadm-rm.patch deleted file mode 100644 index 985d761ef9af..000000000000 --- a/ceph-17.2.4-tox-cephadm-rm.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff --git a/src/cephadm/CMakeLists.txt b/src/cephadm/CMakeLists.txt -index 13be09bae21..eb7ce8afd00 100644 ---- a/src/cephadm/CMakeLists.txt -+++ b/src/cephadm/CMakeLists.txt -@@ -1,4 +1,4 @@ - if(WITH_TESTS) -- include(AddCephTest) -- add_tox_test(cephadm TOX_ENVS py3 mypy flake8) -+ #include(AddCephTest) -+ #add_tox_test(cephadm TOX_ENVS py3 mypy flake8) - endif() diff --git a/ceph-17.2.4-tox-mypy-false-postive.patch b/ceph-17.2.4-tox-mypy-false-postive.patch index df60e8984ab9..836dbd3f18fd 100644 --- a/ceph-17.2.4-tox-mypy-false-postive.patch +++ b/ceph-17.2.4-tox-mypy-false-postive.patch @@ -1,5 +1,5 @@ diff --git a/src/pybind/mgr/tox.ini b/src/pybind/mgr/tox.ini -index d6ae7a29b3d..bb886a2d59e 100644 +index 85e7ae3db11..9a8d009b973 100644 --- a/src/pybind/mgr/tox.ini +++ b/src/pybind/mgr/tox.ini @@ -88,7 +88,6 @@ commands = diff --git a/ceph-17.2.5-fix-iterator-depreciations.patch b/ceph-17.2.5-fix-iterator-depreciations.patch index db57ed696843..af8635d05eea 100644 --- a/ceph-17.2.5-fix-iterator-depreciations.patch +++ b/ceph-17.2.5-fix-iterator-depreciations.patch @@ -1,42 +1,3 @@ -diff --git a/src/crimson/os/seastore/omap_manager/btree/string_kv_node_layout.h b/src/crimson/os/seastore/omap_manager/btree/string_kv_node_layout.h -index 9948a4292fd..505f9826919 100644 ---- a/src/crimson/os/seastore/omap_manager/btree/string_kv_node_layout.h -+++ b/src/crimson/os/seastore/omap_manager/btree/string_kv_node_layout.h -@@ -294,7 +294,13 @@ class StringKVInnerNodeLayout { - friend class delta_inner_t; - public: - template <bool is_const> -- class iter_t : public std::iterator<std::input_iterator_tag, StringKVInnerNodeLayout> { -+ class iter_t { -+ using iterator_category = std::input_iterator_tag; -+ using value_type = StringKVInnerNodeLayout; -+ using difference_type = std::ptrdiff_t; -+ using pointer = StringKVInnerNodeLayout*; -+ using reference = StringKVInnerNodeLayout&; -+ - friend class StringKVInnerNodeLayout; - - template <typename iterator, typename const_iterator> -diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp -index b40d7bf9a57..683a87a28f4 100644 ---- a/src/include/rados/librados.hpp -+++ b/src/include/rados/librados.hpp -@@ -104,8 +104,14 @@ inline namespace v14_2_0 { - }; - CEPH_RADOS_API std::ostream& operator<<(std::ostream& os, const librados::ObjectCursor& oc); - -- class CEPH_RADOS_API NObjectIterator : public std::iterator <std::forward_iterator_tag, ListObject> { -+ class CEPH_RADOS_API NObjectIterator { - public: -+ using iterator_category = std::forward_iterator_tag; -+ using value_type = ListObject; -+ using difference_type = std::ptrdiff_t; -+ using pointer = ListObject*; -+ using reference = ListObject&; -+ - static const NObjectIterator __EndObjectIterator; - NObjectIterator(): impl(NULL) {} - ~NObjectIterator(); diff --git a/src/include/rangeset.h b/src/include/rangeset.h index e7e3d047c72..f19af0b61e4 100644 --- a/src/include/rangeset.h @@ -58,46 +19,6 @@ index e7e3d047c72..f19af0b61e4 100644 //typedef typename map<T,T>::iterator mapit; map<T,T> ranges; -diff --git a/src/include/xlist.h b/src/include/xlist.h -index 733a318a9bd..7e1ae49b246 100644 ---- a/src/include/xlist.h -+++ b/src/include/xlist.h -@@ -159,10 +159,16 @@ public: - remove(_back); - } - -- class iterator: std::iterator<std::forward_iterator_tag, T> { -+ class iterator { - private: - item *cur; - public: -+ using iterator_category = std::forward_iterator_tag; -+ using value_type = T; -+ using difference_type = std::ptrdiff_t; -+ using pointer = T*; -+ using reference = T&; -+ - iterator(item *i = 0) : cur(i) {} - T operator*() { return static_cast<T>(cur->_item); } - iterator& operator++() { -@@ -183,10 +189,16 @@ public: - iterator begin() { return iterator(_front); } - iterator end() { return iterator(NULL); } - -- class const_iterator: std::iterator<std::forward_iterator_tag, T> { -+ class const_iterator { - private: - item *cur; - public: -+ using iterator_category = std::forward_iterator_tag; -+ using value_type = T; -+ using difference_type = std::ptrdiff_t; -+ using pointer = T*; -+ using reference = T&; -+ - const_iterator(item *i = 0) : cur(i) {} - const T operator*() { return static_cast<const T>(cur->_item); } - const_iterator& operator++() { diff --git a/src/msg/async/dpdk/circular_buffer.h b/src/msg/async/dpdk/circular_buffer.h index 2c92c120444..bf5d422dac6 100644 --- a/src/msg/async/dpdk/circular_buffer.h diff --git a/ceph-17.2.6-backport-with-fmt-version.patch b/ceph-17.2.6-backport-with-fmt-version.patch deleted file mode 100644 index 178c149d0909..000000000000 --- a/ceph-17.2.6-backport-with-fmt-version.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt -index b03f76bff80..c670d055854 100644 ---- a/src/CMakeLists.txt -+++ b/src/CMakeLists.txt -@@ -296,7 +296,9 @@ include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/src/xxHash") - include_directories(SYSTEM "${CMAKE_SOURCE_DIR}/src/rapidjson/include") - - option(WITH_FMT_HEADER_ONLY "use header-only version of fmt library" OFF) --find_package(fmt 6.0.0 QUIET) -+set(WITH_FMT_VERSION "7.0.0" CACHE -+ STRING "build with fmt version") -+find_package(fmt ${WITH_FMT_VERSION} EXACT QUIET) - if(fmt_FOUND) - include_directories(SYSTEM "${fmt_INCLUDE_DIR}") - else() diff --git a/ceph-17.2.6-ceph-gcc13-includes.patch b/ceph-17.2.6-ceph-gcc13-includes.patch deleted file mode 100644 index 4d7ec8b986c4..000000000000 --- a/ceph-17.2.6-ceph-gcc13-includes.patch +++ /dev/null @@ -1,49 +0,0 @@ -diff --git a/src/librbd/api/PoolMetadata.h b/src/librbd/api/PoolMetadata.h -index c0a8173596e..8d982879aa9 100644 ---- a/src/librbd/api/PoolMetadata.h -+++ b/src/librbd/api/PoolMetadata.h -@@ -9,6 +9,7 @@ - - #include <map> - #include <string> -+#include <cstdint> - - namespace librbd { - -diff --git a/src/msg/async/compression_onwire.h b/src/msg/async/compression_onwire.h -index dcd6d26c4ca..d3b35a4655c 100644 ---- a/src/msg/async/compression_onwire.h -+++ b/src/msg/async/compression_onwire.h -@@ -4,6 +4,7 @@ - #ifndef CEPH_COMPRESSION_ONWIRE_H - #define CEPH_COMPRESSION_ONWIRE_H - -+#include <cstdint> - #include <optional> - - #include "compressor/Compressor.h" -diff --git a/src/test/librados/op_speed.cc b/src/test/librados/op_speed.cc -index 90c7bdac571..c477f552ab9 100644 ---- a/src/test/librados/op_speed.cc -+++ b/src/test/librados/op_speed.cc -@@ -3,6 +3,8 @@ - - #include "include/rados/librados.hpp" - -+#include <cstdint> -+ - constexpr int to_create = 10'000'000; - - int main() { -diff --git a/src/test/mon/test_log_rss_usage.cc b/src/test/mon/test_log_rss_usage.cc -index f6e85f414c8..53b9f1834df 100644 ---- a/src/test/mon/test_log_rss_usage.cc -+++ b/src/test/mon/test_log_rss_usage.cc -@@ -8,6 +8,7 @@ - #include <stdlib.h> - #include <stdio.h> - #include <unistd.h> -+#include <cstdint> - - using namespace std; - diff --git a/ceph-17.2.6-rocksdb-gcc13-includes.patch b/ceph-17.2.6-rocksdb-gcc13-includes.patch deleted file mode 100644 index 8c986d7255ce..000000000000 --- a/ceph-17.2.6-rocksdb-gcc13-includes.patch +++ /dev/null @@ -1,77 +0,0 @@ -diff --git a/src/rocksdb/table/block_based/data_block_hash_index.h b/src/rocksdb/table/block_based/data_block_hash_index.h -index f356395f329..3215221755d 100644 ---- a/src/rocksdb/table/block_based/data_block_hash_index.h -+++ b/src/rocksdb/table/block_based/data_block_hash_index.h -@@ -5,6 +5,7 @@ - - #pragma once - -+#include <cstdint> - #include <string> - #include <vector> - -diff --git a/src/rocksdb/util/string_util.h b/src/rocksdb/util/string_util.h -index 55d106fff02..11178fd1d7b 100644 ---- a/src/rocksdb/util/string_util.h -+++ b/src/rocksdb/util/string_util.h -@@ -6,6 +6,7 @@ - - #pragma once - -+#include <cstdint> - #include <sstream> - #include <string> - #include <unordered_map> - -diff --git a/src/rocksdb/db/compaction/compaction_iteration_stats.h b/src/rocksdb/db/compaction/compaction_iteration_stats.h -index 963c1d8..73487ed 100644 ---- a/src/rocksdb/db/compaction/compaction_iteration_stats.h -+++ b/src/rocksdb/db/compaction/compaction_iteration_stats.h -@@ -5,6 +5,7 @@ - - #pragma once - -+#include <cstdint> - #include "rocksdb/rocksdb_namespace.h" - - struct CompactionIterationStats { - -diff --git a/src/rocksdb/utilities/checkpoint/checkpoint_impl.h b/src/rocksdb/utilities/checkpoint/checkpoint_impl.h -index dfa8460..9a899e8 100644 ---- a/src/rocksdb/utilities/checkpoint/checkpoint_impl.h -+++ b/src/rocksdb/utilities/checkpoint/checkpoint_impl.h -@@ -8,6 +8,7 @@ - - #include "rocksdb/utilities/checkpoint.h" - -+#include <cstdint> - #include <string> - #include "file/filename.h" - #include "rocksdb/db.h" - -diff --git a/src/rocksdb/include/rocksdb/utilities/checkpoint.h b/src/rocksdb/include/rocksdb/utilities/checkpoint.h -index 1b6a740..ad54ca7 100644 ---- a/src/rocksdb/include/rocksdb/utilities/checkpoint.h -+++ b/src/rocksdb/include/rocksdb/utilities/checkpoint.h -@@ -8,6 +8,7 @@ - #pragma once - #ifndef ROCKSDB_LITE - -+#include <cstdint> - #include <string> - #include <vector> - #include "rocksdb/status.h" - -diff --git a/src/rocksdb/third-party/folly/folly/synchronization/detail/ProxyLockable-inl.h b/src/rocksdb/third-party/folly/folly/synchronization/detail/ProxyLockable-inl.h -index 573330c..ada1e63 100644 ---- a/src/rocksdb/third-party/folly/folly/synchronization/detail/ProxyLockable-inl.h -+++ b/src/rocksdb/third-party/folly/folly/synchronization/detail/ProxyLockable-inl.h -@@ -14,6 +14,7 @@ - #include <mutex> - #include <stdexcept> - #include <utility> -+#include <system_error> - - namespace folly { - namespace detail { - diff --git a/ceph-18.2.0-backport-log-runway-expansion.patch b/ceph-18.2.0-backport-log-runway-expansion.patch new file mode 100644 index 000000000000..92bea1a2c633 --- /dev/null +++ b/ceph-18.2.0-backport-log-runway-expansion.patch @@ -0,0 +1,577 @@ +From a5509f93a7b02e31950d1aba0625dd9996c9608e Mon Sep 17 00:00:00 2001 +From: Pere Diaz Bou <pere-altea@hotmail.com> +Date: Tue, 25 Jul 2023 17:27:14 +0200 +Subject: [PATCH 1/2] os/bluestore: test log runway expansion error + +Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com> +--- + src/test/objectstore/test_bluefs.cc | 153 ++++++++++++++++++++++++++++ + 1 file changed, 153 insertions(+) + +diff --git a/src/test/objectstore/test_bluefs.cc b/src/test/objectstore/test_bluefs.cc +index 4f77d8597ae1d..75496a89d2c39 100644 +--- a/src/test/objectstore/test_bluefs.cc ++++ b/src/test/objectstore/test_bluefs.cc +@@ -1401,6 +1401,159 @@ TEST(BlueFS, test_concurrent_dir_link_and_compact_log_56210) { + } + } + ++TEST(BlueFS, test_log_runway) { ++ uint64_t max_log_runway = 65536; ++ ConfSaver conf(g_ceph_context->_conf); ++ conf.SetVal("bluefs_compact_log_sync", "false"); ++ conf.SetVal("bluefs_min_log_runway", "32768"); ++ conf.SetVal("bluefs_max_log_runway", std::to_string(max_log_runway).c_str()); ++ conf.ApplyChanges(); ++ ++ uint64_t size = 1048576 * 128; ++ TempBdev bdev{size}; ++ BlueFS fs(g_ceph_context); ++ ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); ++ uuid_d fsid; ++ ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); ++ ASSERT_EQ(0, fs.mount()); ++ ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); ++ // longer transaction than current runway ++ std::string longdir(max_log_runway, 'a'); ++ fs.mkdir(longdir); ++ { ++ BlueFS::FileWriter *h; ++ ASSERT_EQ(0, fs.mkdir("dir")); ++ ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false)); ++ h->append("foo", 3); ++ h->append("bar", 3); ++ h->append("baz", 3); ++ fs.fsync(h); ++ fs.close_writer(h); ++ } ++ fs.umount(true); ++ fs.mount(); ++ ++ std::vector<std::string> ls; ++ fs.readdir("dir", &ls); ++ ASSERT_EQ(ls.front(), "file"); ++ uint64_t file_size = 0; ++ utime_t mtime; ++ fs.stat("dir", "file", &file_size, &mtime); ++ ASSERT_EQ(file_size, 9); ++} ++ ++TEST(BlueFS, test_log_runway_2) { ++ uint64_t max_log_runway = 65536; ++ ConfSaver conf(g_ceph_context->_conf); ++ conf.SetVal("bluefs_compact_log_sync", "false"); ++ conf.SetVal("bluefs_min_log_runway", "32768"); ++ conf.SetVal("bluefs_max_log_runway", std::to_string(max_log_runway).c_str()); ++ conf.ApplyChanges(); ++ ++ uint64_t size = 1048576 * 128; ++ TempBdev bdev{size}; ++ BlueFS fs(g_ceph_context); ++ ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); ++ uuid_d fsid; ++ ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); ++ ASSERT_EQ(0, fs.mount()); ++ ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); ++ // longer transaction than current runway ++ std::string longdir(max_log_runway * 2, 'a'); ++ std::string longfile(max_log_runway * 2, 'b'); ++ { ++ BlueFS::FileWriter *h; ++ ASSERT_EQ(0, fs.mkdir(longdir)); ++ ASSERT_EQ(0, fs.open_for_write(longdir, longfile, &h, false)); ++ h->append("canary", 6); ++ fs.fsync(h); ++ fs.close_writer(h); ++ fs.sync_metadata(true); ++ } ++ { ++ BlueFS::FileWriter *h; ++ ASSERT_EQ(0, fs.mkdir("dir")); ++ ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false)); ++ h->append("foo", 3); ++ h->append("bar", 3); ++ h->append("baz", 3); ++ fs.fsync(h); ++ fs.close_writer(h); ++ } ++ fs.umount(true); ++ fs.mount(); ++ ++ std::vector<std::string> ls; ++ fs.readdir("dir", &ls); ++ ASSERT_EQ(ls.front(), "file"); ++ uint64_t file_size = 0; ++ utime_t mtime; ++ fs.stat("dir", "file", &file_size, &mtime); ++ ASSERT_EQ(file_size, 9); ++ fs.stat(longdir, longfile, &file_size, &mtime); ++ ASSERT_EQ(file_size, 6); ++} ++ ++TEST(BlueFS, test_log_runway_3) { ++ uint64_t max_log_runway = 65536; ++ ConfSaver conf(g_ceph_context->_conf); ++ conf.SetVal("bluefs_alloc_size", "4096"); ++ conf.SetVal("bluefs_shared_alloc_size", "4096"); ++ conf.SetVal("bluefs_compact_log_sync", "false"); ++ conf.SetVal("bluefs_min_log_runway", "32768"); ++ conf.SetVal("bluefs_max_log_runway", std::to_string(max_log_runway).c_str()); ++ conf.ApplyChanges(); ++ ++ uint64_t size = 1048576 * 128; ++ TempBdev bdev{size}; ++ BlueFS fs(g_ceph_context); ++ ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576)); ++ uuid_d fsid; ++ ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false })); ++ ASSERT_EQ(0, fs.mount()); ++ ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); ++ // longer transaction than current runway ++ for (size_t m = 0; m < 40; m++) { ++ std::string longdir(max_log_runway + m, 'A' + m); ++ std::string longfile(max_log_runway + m, 'A' + m); ++ BlueFS::FileWriter *h; ++ ASSERT_EQ(0, fs.mkdir(longdir)); ++ ASSERT_EQ(0, fs.open_for_write(longdir, longfile, &h, false)); ++ h->append("canary", 6); ++ fs.fsync(h); ++ fs.close_writer(h); ++ fs.sync_metadata(true); ++ } ++ { ++ BlueFS::FileWriter *h; ++ ASSERT_EQ(0, fs.mkdir("dir")); ++ ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false)); ++ h->append("foo", 3); ++ h->append("bar", 3); ++ h->append("baz", 3); ++ fs.fsync(h); ++ fs.close_writer(h); ++ } ++ fs.umount(true); ++ fs.mount(); ++ ++ std::vector<std::string> ls; ++ fs.readdir("dir", &ls); ++ ASSERT_EQ(ls.front(), "file"); ++ uint64_t file_size = 0; ++ utime_t mtime; ++ fs.stat("dir", "file", &file_size, &mtime); ++ ASSERT_EQ(file_size, 9); ++ for (size_t m = 0; m < 40; m++) { ++ uint64_t file_size = 0; ++ utime_t mtime; ++ std::string longdir(max_log_runway + m, 'A' + m); ++ std::string longfile(max_log_runway + m, 'A' + m); ++ fs.stat(longdir, longfile, &file_size, &mtime); ++ ASSERT_EQ(file_size, 6); ++ } ++} ++ + int main(int argc, char **argv) { + auto args = argv_to_vec(argc, argv); + map<string,string> defaults = { + +From e97aa857c241a9694871de3d3a4079c4d9a120d3 Mon Sep 17 00:00:00 2001 +From: Pere Diaz Bou <pere-altea@hotmail.com> +Date: Tue, 25 Jul 2023 17:28:14 +0200 +Subject: [PATCH 2/2] os/bluestore: fix bluefs log runway enospc + +With these changes, every call to log compaction will try to expand its +runway in case of insufficient log space. async compaction will ignore +the `log_forbidden_to_expand` atomic since we know it should't be +harmful. In any other case, expansion of log will wait until compaction +is completed. + +in order to ensure op_file_update_inc fits on disk we increase the size +of logs as previously used in _maybe_extend_log. This means we too bring +back _maybe_extend_log with a different usage. + +_maybe_extend_log increases the size of the log if the runway is less +than the min runway and if the current transaction is too big to fit. + +Fixes: https://tracker.ceph.com/issues/58759 +Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com> +--- + src/os/bluestore/BlueFS.cc | 157 ++++++++++++++-------------- + src/os/bluestore/BlueFS.h | 7 +- + src/os/bluestore/bluefs_types.cc | 19 ++++ + src/os/bluestore/bluefs_types.h | 2 + + src/test/objectstore/test_bluefs.cc | 9 +- + 5 files changed, 110 insertions(+), 84 deletions(-) + +diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc +index 8454ddaf8078c..19953b130d62a 100644 +--- a/src/os/bluestore/BlueFS.cc ++++ b/src/os/bluestore/BlueFS.cc +@@ -20,15 +20,12 @@ using TOPNSPC::common::cmd_getval; + + using std::byte; + using std::list; +-using std::make_pair; + using std::map; + using std::ostream; +-using std::pair; + using std::set; + using std::string; + using std::to_string; + using std::vector; +-using std::chrono::duration; + using std::chrono::seconds; + + using ceph::bufferlist; +@@ -2745,14 +2742,19 @@ void BlueFS::_compact_log_async_LD_LNF_D() //also locks FW for new_writer + // Part 0. + // Lock the log and forbid its expansion and other compactions + ++ // lock log's run-time structures for a while ++ log.lock.lock(); ++ ++ // Extend log in case of having a big transaction waiting before starting compaction. ++ _maybe_extend_log(); ++ + // only one compaction allowed at one time + bool old_is_comp = std::atomic_exchange(&log_is_compacting, true); + if (old_is_comp) { + dout(10) << __func__ << " ongoing" <<dendl; ++ log.lock.unlock(); + return; + } +- // lock log's run-time structures for a while +- log.lock.lock(); + auto t0 = mono_clock::now(); + + // Part 1. +@@ -2763,7 +2765,7 @@ void BlueFS::_compact_log_async_LD_LNF_D() //also locks FW for new_writer + // During that, no one else can write to log, otherwise we risk jumping backwards. + // We need to sync log, because we are injecting discontinuity, and writer is not prepared for that. + +- //signal _maybe_extend_log that expansion of log is temporary inacceptable ++ //signal _extend_log that expansion of log is temporary inacceptable + bool old_forbidden = atomic_exchange(&log_forbidden_to_expand, true); + ceph_assert(old_forbidden == false); + +@@ -2779,9 +2781,9 @@ void BlueFS::_compact_log_async_LD_LNF_D() //also locks FW for new_writer + + // 1.1 allocate new log extents and store them at fnode_tail + File *log_file = log.writer->file.get(); ++ + old_log_jump_to = log_file->fnode.get_allocated(); + bluefs_fnode_t fnode_tail; +- uint64_t runway = log_file->fnode.get_allocated() - log.writer->get_effective_write_pos(); + dout(10) << __func__ << " old_log_jump_to 0x" << std::hex << old_log_jump_to + << " need 0x" << cct->_conf->bluefs_max_log_runway << std::dec << dendl; + int r = _allocate(vselector->select_prefer_bdev(log_file->vselector_hint), +@@ -2809,7 +2811,7 @@ void BlueFS::_compact_log_async_LD_LNF_D() //also locks FW for new_writer + // TODO - think - if _flush_and_sync_log_jump will not add dirty files nor release pending allocations + // then flush_bdev() will not be necessary + _flush_bdev(); +- _flush_and_sync_log_jump_D(old_log_jump_to, runway); ++ _flush_and_sync_log_jump_D(old_log_jump_to); + + // + // Part 2. +@@ -3054,54 +3056,68 @@ void BlueFS::_consume_dirty(uint64_t seq) + } + } + +-// Extends log if its free space is smaller then bluefs_min_log_runway. +-// Returns space available *BEFORE* adding new space. Signed for additional <0 detection. +-int64_t BlueFS::_maybe_extend_log() +-{ ++int64_t BlueFS::_maybe_extend_log() { ++ uint64_t runway = log.writer->file->fnode.get_allocated() - log.writer->get_effective_write_pos(); ++ // increasing the size of the log involves adding a OP_FILE_UPDATE_INC which its size will ++ // increase with respect the number of extents. bluefs_min_log_runway should cover the max size ++ // a log can get. ++ // inject new allocation in case log is too big ++ size_t expected_log_size = 0; ++ log.t.bound_encode(expected_log_size); ++ if (expected_log_size + cct->_conf->bluefs_min_log_runway > runway) { ++ _extend_log(expected_log_size + cct->_conf->bluefs_max_log_runway); ++ } else if (runway < cct->_conf->bluefs_min_log_runway) { ++ _extend_log(cct->_conf->bluefs_max_log_runway); ++ } ++ runway = log.writer->file->fnode.get_allocated() - log.writer->get_effective_write_pos(); ++ return runway; ++} ++ ++void BlueFS::_extend_log(uint64_t amount) { + ceph_assert(ceph_mutex_is_locked(log.lock)); +- // allocate some more space (before we run out)? +- // BTW: this triggers `flush()` in the `page_aligned_appender` of `log.writer`. +- int64_t runway = log.writer->file->fnode.get_allocated() - +- log.writer->get_effective_write_pos(); +- if (runway < (int64_t)cct->_conf->bluefs_min_log_runway) { +- dout(10) << __func__ << " allocating more log runway (0x" +- << std::hex << runway << std::dec << " remaining)" << dendl; +- /* +- * Usually, when we are low on space in log, we just allocate new extent, +- * put update op(log) to log and we are fine. +- * Problem - it interferes with log compaction: +- * New log produced in compaction will include - as last op - jump into some offset (anchor) of current log. +- * It is assumed that log region (anchor - end) will contain all changes made by bluefs since +- * full state capture into new log. +- * Putting log update into (anchor - end) region is illegal, because any update there must be compatible with +- * both logs, but old log is different then new log. +- * +- * Possible solutions: +- * - stall extending log until we finish compacting and switch log (CURRENT) +- * - re-run compaction with more runway for old log +- * - add OP_FILE_ADDEXT that adds extent; will be compatible with both logs +- */ +- if (log_forbidden_to_expand.load() == true) { +- return -EWOULDBLOCK; +- } +- vselector->sub_usage(log.writer->file->vselector_hint, log.writer->file->fnode); +- int r = _allocate( ++ std::unique_lock<ceph::mutex> ll(log.lock, std::adopt_lock); ++ while (log_forbidden_to_expand.load() == true) { ++ log_cond.wait(ll); ++ } ++ ll.release(); ++ uint64_t allocated_before_extension = log.writer->file->fnode.get_allocated(); ++ vselector->sub_usage(log.writer->file->vselector_hint, log.writer->file->fnode); ++ amount = round_up_to(amount, super.block_size); ++ int r = _allocate( + vselector->select_prefer_bdev(log.writer->file->vselector_hint), +- cct->_conf->bluefs_max_log_runway, ++ amount, + 0, + &log.writer->file->fnode); +- ceph_assert(r == 0); +- vselector->add_usage(log.writer->file->vselector_hint, log.writer->file->fnode); +- log.t.op_file_update_inc(log.writer->file->fnode); ++ ceph_assert(r == 0); ++ dout(10) << "extended log by 0x" << std::hex << amount << " bytes " << dendl; ++ vselector->add_usage(log.writer->file->vselector_hint, log.writer->file->fnode); ++ ++ bluefs_transaction_t log_extend_transaction; ++ log_extend_transaction.seq = log.t.seq; ++ log_extend_transaction.uuid = log.t.uuid; ++ log_extend_transaction.op_file_update_inc(log.writer->file->fnode); ++ ++ bufferlist bl; ++ bl.reserve(super.block_size); ++ encode(log_extend_transaction, bl); ++ _pad_bl(bl, super.block_size); ++ log.writer->append(bl); ++ ceph_assert(allocated_before_extension >= log.writer->get_effective_write_pos()); ++ log.t.seq = log.seq_live; ++ ++ // before sync_core we advance the seq ++ { ++ std::unique_lock<ceph::mutex> l(dirty.lock); ++ _log_advance_seq(); + } +- return runway; + } + +-void BlueFS::_flush_and_sync_log_core(int64_t runway) ++void BlueFS::_flush_and_sync_log_core() + { + ceph_assert(ceph_mutex_is_locked(log.lock)); + dout(10) << __func__ << " " << log.t << dendl; + ++ + bufferlist bl; + bl.reserve(super.block_size); + encode(log.t, bl); +@@ -3113,10 +3129,11 @@ void BlueFS::_flush_and_sync_log_core(int64_t runway) + logger->inc(l_bluefs_log_write_count, 1); + logger->inc(l_bluefs_logged_bytes, bl.length()); + +- if (true) { +- ceph_assert(bl.length() <= runway); // if we write this, we will have an unrecoverable data loss +- // transaction will not fit extents before growth -> data loss on _replay +- } ++ uint64_t runway = log.writer->file->fnode.get_allocated() - log.writer->get_effective_write_pos(); ++ // ensure runway is big enough, this should be taken care of by _maybe_extend_log, ++ // but let's keep this here just in case. ++ ceph_assert(bl.length() <= runway); ++ + + log.writer->append(bl); + +@@ -3185,31 +3202,15 @@ void BlueFS::_release_pending_allocations(vector<interval_set<uint64_t>>& to_rel + + int BlueFS::_flush_and_sync_log_LD(uint64_t want_seq) + { +- int64_t available_runway; +- do { +- log.lock.lock(); +- dirty.lock.lock(); +- if (want_seq && want_seq <= dirty.seq_stable) { +- dout(10) << __func__ << " want_seq " << want_seq << " <= seq_stable " +- << dirty.seq_stable << ", done" << dendl; +- dirty.lock.unlock(); +- log.lock.unlock(); +- return 0; +- } +- +- available_runway = _maybe_extend_log(); +- if (available_runway == -EWOULDBLOCK) { +- // we are in need of adding runway, but we are during log-switch from compaction +- dirty.lock.unlock(); +- //instead log.lock.unlock() do move ownership +- std::unique_lock<ceph::mutex> ll(log.lock, std::adopt_lock); +- while (log_forbidden_to_expand.load()) { +- log_cond.wait(ll); +- } +- } else { +- ceph_assert(available_runway >= 0); +- } +- } while (available_runway < 0); ++ log.lock.lock(); ++ dirty.lock.lock(); ++ if (want_seq && want_seq <= dirty.seq_stable) { ++ dout(10) << __func__ << " want_seq " << want_seq << " <= seq_stable " ++ << dirty.seq_stable << ", done" << dendl; ++ dirty.lock.unlock(); ++ log.lock.unlock(); ++ return 0; ++ } + + ceph_assert(want_seq == 0 || want_seq <= dirty.seq_live); // illegal to request seq that was not created yet + uint64_t seq =_log_advance_seq(); +@@ -3218,7 +3219,8 @@ int BlueFS::_flush_and_sync_log_LD(uint64_t want_seq) + to_release.swap(dirty.pending_release); + dirty.lock.unlock(); + +- _flush_and_sync_log_core(available_runway); ++ _maybe_extend_log(); ++ _flush_and_sync_log_core(); + _flush_bdev(log.writer); + logger->set(l_bluefs_log_bytes, log.writer->file->fnode.size); + //now log.lock is no longer needed +@@ -3232,8 +3234,7 @@ int BlueFS::_flush_and_sync_log_LD(uint64_t want_seq) + } + + // Flushes log and immediately adjusts log_writer pos. +-int BlueFS::_flush_and_sync_log_jump_D(uint64_t jump_to, +- int64_t available_runway) ++int BlueFS::_flush_and_sync_log_jump_D(uint64_t jump_to) + { + ceph_assert(ceph_mutex_is_locked(log.lock)); + +@@ -3246,7 +3247,7 @@ int BlueFS::_flush_and_sync_log_jump_D(uint64_t jump_to, + vector<interval_set<uint64_t>> to_release(dirty.pending_release.size()); + to_release.swap(dirty.pending_release); + dirty.lock.unlock(); +- _flush_and_sync_log_core(available_runway); ++ _flush_and_sync_log_core(); + + dout(10) << __func__ << " jumping log offset from 0x" << std::hex + << log.writer->pos << " -> 0x" << jump_to << std::dec << dendl; +diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h +index adfc8eb0a235b..4c89baea3a6c1 100644 +--- a/src/os/bluestore/BlueFS.h ++++ b/src/os/bluestore/BlueFS.h +@@ -453,15 +453,14 @@ class BlueFS { + #endif + + int64_t _maybe_extend_log(); +- void _extend_log(); ++ void _extend_log(uint64_t amount); + uint64_t _log_advance_seq(); + void _consume_dirty(uint64_t seq); + void _clear_dirty_set_stable_D(uint64_t seq_stable); + void _release_pending_allocations(std::vector<interval_set<uint64_t>>& to_release); + +- void _flush_and_sync_log_core(int64_t available_runway); +- int _flush_and_sync_log_jump_D(uint64_t jump_to, +- int64_t available_runway); ++ void _flush_and_sync_log_core(); ++ int _flush_and_sync_log_jump_D(uint64_t jump_to); + int _flush_and_sync_log_LD(uint64_t want_seq = 0); + + uint64_t _estimate_transaction_size(bluefs_transaction_t* t); +diff --git a/src/os/bluestore/bluefs_types.cc b/src/os/bluestore/bluefs_types.cc +index c8d2ede7bed92..70c8a4fbf1c56 100644 +--- a/src/os/bluestore/bluefs_types.cc ++++ b/src/os/bluestore/bluefs_types.cc +@@ -4,6 +4,7 @@ + #include <algorithm> + #include "bluefs_types.h" + #include "common/Formatter.h" ++#include "include/denc.h" + #include "include/uuid.h" + #include "include/stringify.h" + +@@ -218,6 +219,23 @@ std::ostream& operator<<(std::ostream& out, const bluefs_fnode_delta_t& delta) + + // bluefs_transaction_t + ++DENC_HELPERS ++void bluefs_transaction_t::bound_encode(size_t &s) const { ++ uint32_t crc = op_bl.crc32c(-1); ++ DENC_START(1, 1, s); ++ denc(uuid, s); ++ denc_varint(seq, s); ++ // not using bufferlist encode method, as it merely copies the bufferptr and not ++ // contents, meaning we're left with fragmented target bl ++ __u32 len = op_bl.length(); ++ denc(len, s); ++ for (auto& it : op_bl.buffers()) { ++ s += it.length(); ++ } ++ denc(crc, s); ++ DENC_FINISH(s); ++} ++ + void bluefs_transaction_t::encode(bufferlist& bl) const + { + uint32_t crc = op_bl.crc32c(-1); +@@ -282,3 +300,4 @@ ostream& operator<<(ostream& out, const bluefs_transaction_t& t) + << " crc 0x" << t.op_bl.crc32c(-1) + << std::dec << ")"; + } ++ +diff --git a/src/os/bluestore/bluefs_types.h b/src/os/bluestore/bluefs_types.h +index d5d8ee5a62826..b0ce7c5c9d38d 100644 +--- a/src/os/bluestore/bluefs_types.h ++++ b/src/os/bluestore/bluefs_types.h +@@ -308,6 +308,7 @@ struct bluefs_transaction_t { + encode(delta, op_bl); + file.reset_delta(); + } ++ + void op_file_remove(uint64_t ino) { + using ceph::encode; + encode((__u8)OP_FILE_REMOVE, op_bl); +@@ -328,6 +329,7 @@ struct bluefs_transaction_t { + op_bl.claim_append(from.op_bl); + } + ++ void bound_encode(size_t &s) const; + void encode(ceph::buffer::list& bl) const; + void decode(ceph::buffer::list::const_iterator& p); + void dump(ceph::Formatter *f) const; +diff --git a/src/test/objectstore/test_bluefs.cc b/src/test/objectstore/test_bluefs.cc +index 75496a89d2c39..6d3ff1218a437 100644 +--- a/src/test/objectstore/test_bluefs.cc ++++ b/src/test/objectstore/test_bluefs.cc +@@ -1459,8 +1459,9 @@ TEST(BlueFS, test_log_runway_2) { + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false })); + // longer transaction than current runway +- std::string longdir(max_log_runway * 2, 'a'); +- std::string longfile(max_log_runway * 2, 'b'); ++ size_t name_length = max_log_runway * 2; ++ std::string longdir(name_length, 'a'); ++ std::string longfile(name_length, 'b'); + { + BlueFS::FileWriter *h; + ASSERT_EQ(0, fs.mkdir(longdir)); +@@ -1492,6 +1493,10 @@ TEST(BlueFS, test_log_runway_2) { + ASSERT_EQ(file_size, 9); + fs.stat(longdir, longfile, &file_size, &mtime); + ASSERT_EQ(file_size, 6); ++ ++ std::vector<std::string> ls_longdir; ++ fs.readdir(longdir, &ls_longdir); ++ ASSERT_EQ(ls_longdir.front(), longfile); + } + + TEST(BlueFS, test_log_runway_3) { diff --git a/ceph-18.2.0-fix-ecode-shec-test.patch b/ceph-18.2.0-fix-ecode-shec-test.patch new file mode 100644 index 000000000000..24ea9b1af0d6 --- /dev/null +++ b/ceph-18.2.0-fix-ecode-shec-test.patch @@ -0,0 +1,19 @@ +diff --git a/src/test/erasure-code/TestErasureCodeShec_arguments.cc b/src/test/erasure-code/TestErasureCodeShec_arguments.cc +index 075c6383eed..74403eaf6ed 100644 +--- a/src/test/erasure-code/TestErasureCodeShec_arguments.cc ++++ b/src/test/erasure-code/TestErasureCodeShec_arguments.cc +@@ -86,12 +86,12 @@ void create_table_shec432() { + continue; + } + if (std::popcount(avails) == 4) { +- auto a = to_array<std::initializer_list<int>>({ ++ std::vector<std::initializer_list<int>> a = { + {0,1,2,3}, {0,1,2,4}, {0,1,2,6}, {0,1,3,4}, {0,1,3,6}, {0,1,4,6}, + {0,2,3,4}, {0,2,3,5}, {0,2,4,5}, {0,2,4,6}, {0,2,5,6}, {0,3,4,5}, + {0,3,4,6}, {0,3,5,6}, {0,4,5,6}, {1,2,3,4}, {1,2,3,5}, {1,2,4,5}, + {1,2,4,6}, {1,2,5,6}, {1,3,4,5}, {1,3,4,6}, {1,3,5,6}, {1,4,5,6}, +- {2,3,4,5}, {2,4,5,6}, {3,4,5,6}}); ++ {2,3,4,5}, {2,4,5,6}, {3,4,5,6}}; + if (ranges::any_of(a, std::bind_front(cmp_equal<uint, int>, avails), + getint)) { + vec.push_back(avails); diff --git a/ceph-18.2.0-fmt10-fixes.patch b/ceph-18.2.0-fmt10-fixes.patch new file mode 100644 index 000000000000..10ae19ee36e1 --- /dev/null +++ b/ceph-18.2.0-fmt10-fixes.patch @@ -0,0 +1,206 @@ +diff --git a/src/common/LogEntry.h b/src/common/LogEntry.h +index 3ddebbd3043..b9096e2850a 100644 +--- a/src/common/LogEntry.h ++++ b/src/common/LogEntry.h +@@ -15,7 +15,11 @@ + #ifndef CEPH_LOGENTRY_H + #define CEPH_LOGENTRY_H + ++#include <fmt/core.h> + #include <fmt/format.h> ++#if FMT_VERSION >= 90000 ++#include <fmt/ostream.h> ++#endif + + #include "include/utime.h" + #include "msg/msg_fmt.h" +@@ -194,19 +198,17 @@ inline std::ostream& operator<<(std::ostream& out, const LogEntry& e) + << e.channel << " " << e.prio << " " << e.msg; + } + +-template <> struct fmt::formatter<EntityName> : fmt::formatter<std::string_view> { +- template <typename FormatContext> +- auto format(const EntityName& e, FormatContext& ctx) { +- return formatter<std::string_view>::format(e.to_str(), ctx); +- } +-}; ++template <> ++struct fmt::formatter<clog_type>: fmt::ostream_formatter {}; + +-template <> struct fmt::formatter<LogEntry> : fmt::formatter<std::string_view> { +- template <typename FormatContext> +- auto format(const LogEntry& e, FormatContext& ctx) { +- return fmt::format_to(ctx.out(), "{} {} ({}) {} : {} {} {}", +- e.stamp, e.name, e.rank, e.seq, e.channel, e.prio, e.msg); ++template <> ++struct fmt::formatter<EntityName> : fmt::formatter<std::string_view> { ++ auto format(const EntityName& e, format_context& ctx) { ++ return fmt::formatter<std::string_view>::format(e.to_str(), ctx); + } + }; + ++template <> ++struct fmt::formatter<LogEntry> : fmt::ostream_formatter {}; ++ + #endif +diff --git a/src/include/byteorder.h b/src/include/byteorder.h +index eb6d5e102b4..9a4d0be877a 100644 +--- a/src/include/byteorder.h ++++ b/src/include/byteorder.h +@@ -53,3 +53,8 @@ inline ceph_les16 init_les16(__s16 x) { + v = x; + return v; + } ++ ++template <typename T> ++auto format_as(ceph_le<T> c) { ++ return (T)c; ++} +diff --git a/src/include/neorados/RADOS_fmt.hpp b/src/include/neorados/RADOS_fmt.hpp +new file mode 100644 +index 00000000000..1512ec965fe +--- /dev/null ++++ b/src/include/neorados/RADOS_fmt.hpp +@@ -0,0 +1,16 @@ ++// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- ++// vim: ts=8 sw=2 smarttab ++#pragma once ++/** ++ * \file fmtlib formatters for some neorados types ++ */ ++ ++#include <fmt/core.h> ++#if FMT_VERSION >= 90000 ++#include <fmt/ostream.h> ++#endif ++ ++#include <include/neorados/RADOS.hpp> ++ ++template <> ++struct fmt::formatter<neorados::Object> : fmt::ostream_formatter {}; +diff --git a/src/include/types_fmt.h b/src/include/types_fmt.h +new file mode 100644 +index 00000000000..3d40085f0b2 +--- /dev/null ++++ b/src/include/types_fmt.h +@@ -0,0 +1,16 @@ ++// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- ++// vim: ts=8 sw=2 smarttab ++#pragma once ++/** ++ * \file fmtlib formatters for some types.h classes ++ */ ++ ++#include <fmt/core.h> ++#if FMT_VERSION >= 90000 ++#include <fmt/ostream.h> ++#endif ++ ++#include <include/types.h> ++ ++template <> ++struct fmt::formatter<shard_id_t> : fmt::ostream_formatter {}; +diff --git a/src/osd/SnapMapper.cc b/src/osd/SnapMapper.cc +index 7893bc08fdc..e8d34cd25bc 100644 +--- a/src/osd/SnapMapper.cc ++++ b/src/osd/SnapMapper.cc +@@ -211,7 +211,7 @@ string SnapMapper::get_prefix(int64_t pool, snapid_t snap) + return fmt::sprintf("%s%lld_%.16X_", + MAPPING_PREFIX, + pool, +- snap); ++ (uint64_t)snap); + } + + string SnapMapper::to_raw_key( +@@ -650,7 +650,7 @@ string SnapMapper::make_purged_snap_key(int64_t pool, snapid_t last) + return fmt::sprintf("%s_%lld_%016llx", + PURGED_SNAP_PREFIX, + pool, +- last); ++ (uint64_t)last); + } + + void SnapMapper::make_purged_snap_key_value( +@@ -866,7 +866,7 @@ string SnapMapper::get_legacy_prefix(snapid_t snap) + { + return fmt::sprintf("%s%.16X_", + LEGACY_MAPPING_PREFIX, +- snap); ++ (uint64_t)snap); + } + + string SnapMapper::to_legacy_raw_key( +diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h +index afed5fa8351..e374369e8ba 100644 +--- a/src/osd/osd_types.h ++++ b/src/osd/osd_types.h +@@ -35,6 +35,7 @@ + #include "msg/msg_types.h" + #include "include/compat.h" + #include "include/types.h" ++#include "include/types_fmt.h" + #include "include/utime.h" + #include "include/CompatSet.h" + #include "common/ceph_context.h" +diff --git a/src/osd/osd_types_fmt.h b/src/osd/osd_types_fmt.h +index 8d48134106e..65a751469f7 100644 +--- a/src/osd/osd_types_fmt.h ++++ b/src/osd/osd_types_fmt.h +@@ -57,7 +57,7 @@ struct fmt::formatter<chunk_info_t> { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + template <typename FormatContext> +- auto format(const chunk_info_t& ci, FormatContext& ctx) ++ auto format(const chunk_info_t& ci, FormatContext& ctx) const + { + return fmt::format_to(ctx.out(), "(len: {} oid: {} offset: {} flags: {})", + ci.length, ci.oid, ci.offset, +@@ -169,7 +169,7 @@ struct fmt::formatter<pg_info_t> { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + template <typename FormatContext> +- auto format(const pg_info_t& pgi, FormatContext& ctx) ++ auto format(const pg_info_t& pgi, FormatContext& ctx) const + { + fmt::format_to(ctx.out(), "{}({}", pgi.pgid, (pgi.dne() ? " DNE" : "")); + if (pgi.is_empty()) { +@@ -211,7 +211,7 @@ struct fmt::formatter<SnapSet> { + } + + template <typename FormatContext> +- auto format(const SnapSet& snps, FormatContext& ctx) ++ auto format(const SnapSet& snps, FormatContext& ctx) const + { + if (verbose) { + // similar to SnapSet::dump() +@@ -265,7 +265,7 @@ struct fmt::formatter<ScrubMap::object> { + + ///\todo: consider passing the 'D" flag to control snapset dump + template <typename FormatContext> +- auto format(const ScrubMap::object& so, FormatContext& ctx) ++ auto format(const ScrubMap::object& so, FormatContext& ctx) const + { + fmt::format_to(ctx.out(), + "so{{ sz:{} dd:{} od:{} ", +@@ -308,7 +308,7 @@ struct fmt::formatter<ScrubMap> { + } + + template <typename FormatContext> +- auto format(const ScrubMap& smap, FormatContext& ctx) ++ auto format(const ScrubMap& smap, FormatContext& ctx) const + { + fmt::format_to(ctx.out(), + "smap{{ valid:{} incr-since:{} #:{}", +diff --git a/src/tools/neorados.cc b/src/tools/neorados.cc +index 24966d2aee5..44ee1cf199c 100644 +--- a/src/tools/neorados.cc ++++ b/src/tools/neorados.cc +@@ -36,6 +36,7 @@ + #include "include/buffer.h" // :( + + #include "include/neorados/RADOS.hpp" ++#include "include/neorados/RADOS_fmt.hpp" + + using namespace std::literals; + @@ -16,25 +16,26 @@ build: @$Say Building @{{PkgBuild}} via chroot makechrootpkg -c -r {{ChrootPath}} -d "/tmp:/tmp" -C -n -l {{PkgVer}}_{{PkgRel}} +# Repackage without rebuilding +repackage: + @$Say Repackaging @{{PkgBuild}} via chroot + makechrootpkg -r {{ChrootPath}} -l {{PkgVer}}_{{PkgRel}} -- --skipint --noprepare --noextract --nocheck --repackage --force + +# Run a command in the chroot environment +cexec +args: (_cexec "." args) + +# Run ctest in the chroot environment +ctest *args: (_cexec "build" "ctest" args) + # Create and update the base chroot chroot: (_update_chroot ChrootBase) # Initialize the base chroot for building packages mkchroot: (_mkchroot ChrootBase) -# Watch build log streams, optionally filtering them with the given regex and options -watch $filter=None $opts="-iP": _mkloglist - @$Say Watching build {{BuildTriple}} logs ${filter:+"(filter: $filter $opts)"} - tail -F -n +1 --silent $(cat {{LogFileList}} | xargs) 2>/dev/null {{ if filter == None { None } else { '| rg ' + opts + ' "' + filter + '"' } }} - -# Print build logs, optionally filtering them with the given regex and options -logs $filter=None $opts="-iP": _mkloglist - @$Say Printing {{BuildTriple}} logs ${filter:+"(filter: $filter $opts)"} - cat *.log 2>/dev/null {{ if filter == None { None } else { '| rg ' + opts + ' "' + filter + '"' } }} - # Install required dependencies deps: - pacman -S base-devel sudo devtools ripgrep --needed --noconfirm + pacman -S base-devel util-linux sudo devtools ripgrep --needed --noconfirm # Clean one or more of: chroot|deps|artifacts|logs clean +what="chroot": @@ -63,7 +64,7 @@ clean +what="chroot": done # Upload built artifacts to Github, using the associated release -upload pkg="ceph,ceph-libs,ceph-mgr": (_upload pkg) +upload pkg="@all": (_upload pkg) # Initialize the chroot @_mkchroot $cbase: @@ -75,24 +76,25 @@ upload pkg="ceph,ceph-libs,ceph-mgr": (_upload pkg) $Say Updating chroot packages @$cbase arch-nspawn $cbase pacman -Syu -@_mkloglist: - mkdir -p $(dirname {{LogFileList}}) - echo \ - ceph-{{BuildTriple}}-{build,prepare,check,package_ceph{,-libs,-mgr}}.log \ - ceph-{,mgr-,libs-}{{BuildTriple}}.pkg.tar.zst-namcap.log \ - PKGBUILD-namcap.log \ - > {{LogFileList}} +# Exec into the chroot to a path relative to the workdir, and run the given args +_cexec path +args: + arch-nspawn {{ChrootActive}} --chdir /build/{{PkgBase}}/src/{{PkgBase}}-{{PkgVer}}/{{path}} sh -c {{quote(trim(args))}} # Script to upload a comma separated list of packages to the active Github release _upload $pkgstring: #!/usr/bin/env bash set -euo pipefail - [[ -n "$GITHUB_TOKEN" ]] || { $Say "Error: GITHUB_TOKEN must be set" && exit 4; } + [[ -v GITHUB_TOKEN ]] || { $Say "Error: GITHUB_TOKEN must be set" && exit 4; } IFS=', ' read -r -a PKGS <<<"$pkgstring" + if printf '%s\0' "${PKGS[@]}" | grep -zxqF -- '@all'; then + $Say Expanding '@all' to package set + PKGS=($(rg -P --only-matching --replace '$1' '^package_(.+)\(\) {' {{PkgBuild}} | sort | xargs)) + fi - $Say "uploading package(s): { ${PKGS[@]} } to {{GithubRepo}}/releases/v{{PkgVer}}-{{PkgRel}}" + $Say "Uploading ${#PKGS[@]} package(s) to {{GithubRepo}}/releases/v{{PkgVer}}-{{PkgRel}}" + printf ' > %s %s %s %s %s\n' "${PKGS[@]}" | column -t declare -A FILES for pkg in "${PKGS[@]}"; do @@ -137,7 +139,6 @@ GitCommitish := if `git tag --points-at HEAD` != None { } BuildId := "[" + C_YELLOW + PkgBase + C_RESET + "/" + C_GREEN + PkgVer + ":" + PkgRel + C_RESET + "@" + C_CYAN + GitCommitish + C_RESET + "]" BuildTriple := PkgVer + "-" + PkgRel + "-" + "x86_64" -LogFileList := env_var_or_default("TEMP", "/tmp") / PkgBase + ".temp" / "logfiles" GithubRepo := "bazaah/aur-ceph" # ~~~ Color Codes ~~~ diff --git a/scripts.gh-upload-artifact.sh b/scripts.gh-upload-artifact.sh index 6747c52aea92..df01b07b152f 100755 --- a/scripts.gh-upload-artifact.sh +++ b/scripts.gh-upload-artifact.sh @@ -137,7 +137,12 @@ main() { -H "$GH_AUTH" \ -H "Content-Type: $(file -b --mime-type $artifact)" \ --data-binary @"$artifact" \ + -o "$GH_RESPONSE" \ $upload_uri + + (( $? > 0 )) \ + && die 1 "$(cat <(printf "Failed to upload $filename to $repo:\n") $GH_RESPONSE)" \ + || jq '.' "$GH_RESPONSE" fi } |