- added patch to make MinIO/S3 object deletion delay configurable

 - modified man page and server configuration to reflect above changes

5 files changed, 106 insertions, 11 deletions

diff --git a/.SRCINFO b/.SRCINFO
index faab4532cb9a..f63b43b70c0d 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,7 +1,7 @@
 pkgbase = ente-server-git
 	pkgdesc = Self hosted server for Ente (mobile) clients
-	pkgver = r1.40abb5c
-	pkgrel = 4
+	pkgver = r1.7490199
+	pkgrel = 5
 	url = https://github.com/ente-io
 	arch = x86_64
 	license = AGPL-3.0-only
@@ -20,6 +20,7 @@ pkgbase = ente-server-git
 	options = !debug
 	backup = etc/ente-server/local.yaml
 	source = https://raw.githubusercontent.com/ente-io/ente/main/LICENSE
+	source = ente-server-configurable-delete-object-delay.patch
 	source = ente-server-man.1.md
 	source = ente-server-nginx.conf
 	source = ente-server-sysusers.conf
@@ -30,12 +31,13 @@ pkgbase = ente-server-git
 	source = https_security_headers.conf
 	source = usr.bin.ente-server
 	sha256sums = 0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0
-	sha256sums = d4d6648ded8043d80d056cbafcaf87cacf9a5e79b22c02e0302506e6568c177c
+	sha256sums = f3624560a2c332724967e1e64689f8549a936fb85fc557ccc4bcbb7e57e373e8
+	sha256sums = 9151cd1072cf33f88c355761ce931346a58555da49cb6241e9498b4c1dd0b87b
 	sha256sums = 2d5221aaa83f32bbc8c75c2d7c70f9ff8021d451b544f230c99fe29b84fcba75
 	sha256sums = 72c23c4ba9d3468a1b089d182917123cb15b8bf8b52b3955b98a0357d29b5cbd
 	sha256sums = 6ba953245f2a285dbd82ce65635d19410eab1dcd92821c398bdf7ffba9451a9b
 	sha256sums = f5ae64093463a66fa66ecc4627f603ff0f9e17841e1d681dbcc68b1bad95100e
-	sha256sums = 9a10d967adca272d3d04666adf6d39c74122941f2431d95a4bc2f8acd4f4cffb
+	sha256sums = c3e54eacff7f6b4a406dff4b871120c6a97dc5dca179347055514a19d10cfb72
 	sha256sums = 405365bd47efa25b8bcefc93a5c0535fd50cce22b5d8dcea070098aa432ff87e
 	sha256sums = a1149c57e233f7be2f12668f5ef0f03409bd5ad37b1a223bb56d2ae865cf6358
 	sha256sums = 297bc7d90c473758c9054aaaa6155b4e7232d0dfea761a4e55ed8b743f289f86
diff --git a/PKGBUILD b/PKGBUILD
index 5cae401d4dc2..c56f8d725fb7 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -2,8 +2,8 @@
 pkgname=ente-server-git
 _pkgname_alt=museum
 _pkg_git_src=https://github.com/ente-io/ente.git
-pkgver=r1.40abb5c
-pkgrel=4
+pkgver=r1.7490199
+pkgrel=5
 pkgdesc="Self hosted server for Ente (mobile) clients"
 arch=(x86_64)
 url="https://github.com/ente-io"
@@ -19,6 +19,7 @@ backup=(
 options=('!debug')
 source=(
     "https://raw.githubusercontent.com/ente-io/ente/main/LICENSE"
+    "${pkgname%-git}-configurable-delete-object-delay.patch"
     "${pkgname%-git}-man.1.md"
     "${pkgname%-git}-nginx.conf"
     "${pkgname%-git}-sysusers.conf"
@@ -31,12 +32,13 @@ source=(
 )
 sha256sums=(
     "0d96a4ff68ad6d4b6f1f30f713b18d5184912ba8dd389f86aa7710db079abcb0"
-    "d4d6648ded8043d80d056cbafcaf87cacf9a5e79b22c02e0302506e6568c177c"
+    "f3624560a2c332724967e1e64689f8549a936fb85fc557ccc4bcbb7e57e373e8"
+    "9151cd1072cf33f88c355761ce931346a58555da49cb6241e9498b4c1dd0b87b"
     "2d5221aaa83f32bbc8c75c2d7c70f9ff8021d451b544f230c99fe29b84fcba75"
     "72c23c4ba9d3468a1b089d182917123cb15b8bf8b52b3955b98a0357d29b5cbd"
     "6ba953245f2a285dbd82ce65635d19410eab1dcd92821c398bdf7ffba9451a9b"
     "f5ae64093463a66fa66ecc4627f603ff0f9e17841e1d681dbcc68b1bad95100e"
-    "9a10d967adca272d3d04666adf6d39c74122941f2431d95a4bc2f8acd4f4cffb"
+    "c3e54eacff7f6b4a406dff4b871120c6a97dc5dca179347055514a19d10cfb72"
     "405365bd47efa25b8bcefc93a5c0535fd50cce22b5d8dcea070098aa432ff87e"
     "a1149c57e233f7be2f12668f5ef0f03409bd5ad37b1a223bb56d2ae865cf6358"
     "297bc7d90c473758c9054aaaa6155b4e7232d0dfea761a4e55ed8b743f289f86"
@@ -67,6 +69,11 @@ prepare() {
 }
 
 build() {
+  # patch ente-server: make MinIO/S3 object deletion delay configurable
+  cd "${srcdir}/${pkgname}/"
+  patch -p1 < "${srcdir}/${pkgname%-git}-configurable-delete-object-delay.patch"
+
+  # build ente-server binary
   cd "${srcdir}/${pkgname}/server/cmd/${_pkgname_alt}"
   export GOPATH="${srcdir}/gopath"
   export CGO_CPPFLAGS="${CPPFLAGS}"
@@ -75,9 +82,9 @@ build() {
   export CGO_LDFLAGS="${LDFLAGS}"
   export GOFLAGS="-buildmode=pie -trimpath -ldflags=-linkmode=external -modcacherw -mod=mod"
   mkdir -p ../../build
-  # patch cleanup -> reduce delay from 45 days to 5 minutes
-  sed -i 's@DeleteObjectQueue:         45 \* 24 \* 60, \/\/ 45 days in minutes@DeleteObjectQueue:                    5, // 5 minutes@' "${srcdir}/${pkgname}/server/pkg/repo/queue.go"
   go build -o ../../build/ente-server -v
+
+  # build ente-server tool binaries
   cd ../../tools
   for tool in *; do
       [ "${tool}" == 'pkg' ] && continue
@@ -85,6 +92,8 @@ build() {
       go build -o ../../build -v
       cd -
   done
+
+  # cleanup go module cache
   go clean -modcache
 }
 
diff --git a/ente-server-configurable-delete-object-delay.patch b/ente-server-configurable-delete-object-delay.patch
new file mode 100644
index 000000000000..528567846bf0
--- /dev/null
+++ b/ente-server-configurable-delete-object-delay.patch
@@ -0,0 +1,80 @@
+diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go
+index 84c34189d..8a7e3f3bc 100644
+--- a/server/cmd/museum/main.go
++++ b/server/cmd/museum/main.go
+@@ -154,7 +154,7 @@ func main() {
+ 	dataCleanupRepository := &datacleanup.Repository{DB: db}
+ 
+ 	notificationHistoryRepo := &repo.NotificationHistoryRepository{DB: db}
+-	queueRepo := &repo.QueueRepository{DB: db}
++	queueRepo := &repo.QueueRepository{DB: db, DeleteObjectDelay: viper.GetString("internal.delete-object-delay")}
+ 	objectRepo := &repo.ObjectRepository{DB: db, QueueRepo: queueRepo}
+ 	objectCleanupRepo := &repo.ObjectCleanupRepository{DB: db}
+ 	objectCopiesRepo := &repo.ObjectCopiesRepository{DB: db}
+diff --git a/server/configurations/local.yaml b/server/configurations/local.yaml
+index 7785f5601..e205368eb 100644
+--- a/server/configurations/local.yaml
++++ b/server/configurations/local.yaml
+@@ -282,6 +282,10 @@ internal:
+     #     local-domain-value: 123456
+     # List of user IDs that can use the admin API endpoints.
+     admins: []
++    # Cleanup delay of S3/MinIO objects in minutes (default: 45 days = 64800 minutes)
++    # WARNING: Only change this value when you know what you're doing, since this can cause things to break if configured incorrectly.
++    # For example, if replication is enabled then this value should be such that the deletion is never attempted before compliance lock passes.
++    # delete-object-delay: 64800
+ 
+ # Replication config
+ #
+diff --git a/server/pkg/repo/queue.go b/server/pkg/repo/queue.go
+index 49544dbc8..b39da98bb 100644
+--- a/server/pkg/repo/queue.go
++++ b/server/pkg/repo/queue.go
+@@ -15,17 +15,7 @@ import (
+ // QueueRepository defines methods to insert, delete items from queue
+ type QueueRepository struct {
+ 	DB *sql.DB
+-}
+-
+-// itemDeletionDelayInMinMap tracks the delay (in min) after which an item is ready to be processed.
+-// -ve entry indicates that the item should be processed immediately, without any delay.
+-var itemDeletionDelayInMinMap = map[string]int64{
+-	DropFileEncMedataQueue:    -1 * 24 * 60, // -ve value to ensure attributes are immediately removed
+-	DeleteObjectQueue:         45 * 24 * 60, // 45 days in minutes
+-	DeleteEmbeddingsQueue:     -1 * 24 * 60, // -ve value to ensure embeddings are immediately removed
+-	TrashCollectionQueueV3:    -1 * 24 * 60, // -ve value to ensure collections are immediately marked as trashed
+-	TrashEmptyQueue:           -1 * 24 * 60, // -ve value to ensure empty trash request are processed in next cron run
+-	RemoveComplianceHoldQueue: -1 * 24 * 60, // -ve value to ensure compliance hold is removed in next cron run
++    DeleteObjectDelay string
+ }
+ 
+ const (
+@@ -124,8 +114,28 @@ func (repo *QueueRepository) DeleteItem(queueName string, item string) error {
+ 	return stacktrace.Propagate(err, "")
+ }
+ 
++func getDeleteObjectDelayInt(DeleteObjectDelay *string) int64 {
++    DeleteObjectDelayInt, err := strconv.ParseInt(*DeleteObjectDelay, 10, 64)
++    if err != nil || DeleteObjectDelayInt == 0 {
++        DeleteObjectDelayInt = 45 * 24 * 60 // defaults to 45 days in minutes
++    }
++    logrus.Debugf("DeleteObjectDelay: %d minutes", DeleteObjectDelayInt)
++    return DeleteObjectDelayInt
++}
++
+ // GetItemsReadyForDeletion method, for a given queue name, returns a list of QueueItem  which are ready for deletion
+ func (repo *QueueRepository) GetItemsReadyForDeletion(queueName string, count int) ([]QueueItem, error) {
++    // itemDeletionDelayInMinMap tracks the delay (in min) after which an item is ready to be processed.
++    // -ve entry indicates that the item should be processed immediately, without any delay.
++    var itemDeletionDelayInMinMap = map[string]int64{
++    	DropFileEncMedataQueue:    -1 * 24 * 60, // -ve value to ensure attributes are immediately removed
++    	DeleteObjectQueue:         getDeleteObjectDelayInt(&repo.DeleteObjectDelay),
++    	DeleteEmbeddingsQueue:     -1 * 24 * 60, // -ve value to ensure embeddings are immediately removed
++    	TrashCollectionQueueV3:    -1 * 24 * 60, // -ve value to ensure collections are immediately marked as trashed
++    	TrashEmptyQueue:           -1 * 24 * 60, // -ve value to ensure empty trash request are processed in next cron run
++    	RemoveComplianceHoldQueue: -1 * 24 * 60, // -ve value to ensure compliance hold is removed in next cron run
++    }
++
+ 	delayInMin, ok := itemDeletionDelayInMinMap[queueName]
+ 	if !ok {
+ 		return nil, stacktrace.Propagate(fmt.Errorf("missing delay for %s", queueName), "")
diff --git a/ente-server-man.1.md b/ente-server-man.1.md
index ccd10e837997..e719b4bab29a 100644
--- a/ente-server-man.1.md
+++ b/ente-server-man.1.md
@@ -20,7 +20,7 @@ For example when the Ente photos client is used, the client encrypts the photos,
 
 This AUR package is meant for self hosting in contrary to the by default provided dockerized Ente server. If a dockerized or hosted solution is required, please go to <https://ente.io> for more information.
 
-Note: Since self hosted server space is often available in limited quantity, this AUR package has been modified to cleanup old unlinked Ente server MinIO objects much faster. The default upstream Ente server cleans these object every 45 days by default, but this package decreased the delay to cleanup every 5 minutes instead.
+Note: Since self hosted server space is often available in limited quantity, this AUR package added the ability to change the cleanup delay cleanup before old unlinked ente-server `MinIO` objects will be actually deleted. This can be configured via the `internal:delete-object-delay` `ente-server` configuration parameter. The default upstream Ente server cleans these objects every 45 days by default, but when replication is not required, this setting can be set to cleanup every 5 minutes for example. Do note that this setting is **NOT** supported by the Ente developers, due to the possible risk of breaking `ente-server`. The full warning has been described in the `ente-server` configuration file.
 
 INSTALLATION
 ------------
diff --git a/ente-server.yaml b/ente-server.yaml
index 8396a4e05a76..aeb47b340471 100644
--- a/ente-server.yaml
+++ b/ente-server.yaml
@@ -129,6 +129,10 @@ internal:
     #     local-domain-value: 123456
     # List of user IDs that can use the admin API endpoints.
     admins: []
+    # Cleanup delay of S3/MinIO objects in minutes (default: 45 days = 64800 minutes)
+    # WARNING: Only change this value when you know what you're doing, since this can cause things to break if configured incorrectly.
+    # For example, if replication is enabled then this value should be such that the deletion is never attempted before compliance lock passes.
+    # delete-object-delay: 64800
 
 # Replication config
 #