1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
|
diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go
index 84c34189d..8a7e3f3bc 100644
--- a/server/cmd/museum/main.go
+++ b/server/cmd/museum/main.go
@@ -154,7 +154,7 @@ func main() {
dataCleanupRepository := &datacleanup.Repository{DB: db}
notificationHistoryRepo := &repo.NotificationHistoryRepository{DB: db}
- queueRepo := &repo.QueueRepository{DB: db}
+ queueRepo := &repo.QueueRepository{DB: db, DeleteObjectDelay: viper.GetString("internal.delete-object-delay")}
objectRepo := &repo.ObjectRepository{DB: db, QueueRepo: queueRepo}
objectCleanupRepo := &repo.ObjectCleanupRepository{DB: db}
objectCopiesRepo := &repo.ObjectCopiesRepository{DB: db}
diff --git a/server/configurations/local.yaml b/server/configurations/local.yaml
index 7785f5601..e205368eb 100644
--- a/server/configurations/local.yaml
+++ b/server/configurations/local.yaml
@@ -282,6 +282,10 @@ internal:
# local-domain-value: 123456
# List of user IDs that can use the admin API endpoints.
admins: []
+ # Cleanup delay of S3/MinIO objects in minutes (default: 45 days = 64800 minutes)
+ # WARNING: Only change this value when you know what you're doing, since this can cause things to break if configured incorrectly.
+ # For example, if replication is enabled then this value should be such that the deletion is never attempted before compliance lock passes.
+ # delete-object-delay: 64800
# Replication config
#
diff --git a/server/pkg/repo/queue.go b/server/pkg/repo/queue.go
index 49544dbc8..b39da98bb 100644
--- a/server/pkg/repo/queue.go
+++ b/server/pkg/repo/queue.go
@@ -15,17 +15,7 @@ import (
// QueueRepository defines methods to insert, delete items from queue
type QueueRepository struct {
DB *sql.DB
-}
-
-// itemDeletionDelayInMinMap tracks the delay (in min) after which an item is ready to be processed.
-// -ve entry indicates that the item should be processed immediately, without any delay.
-var itemDeletionDelayInMinMap = map[string]int64{
- DropFileEncMedataQueue: -1 * 24 * 60, // -ve value to ensure attributes are immediately removed
- DeleteObjectQueue: 45 * 24 * 60, // 45 days in minutes
- DeleteEmbeddingsQueue: -1 * 24 * 60, // -ve value to ensure embeddings are immediately removed
- TrashCollectionQueueV3: -1 * 24 * 60, // -ve value to ensure collections are immediately marked as trashed
- TrashEmptyQueue: -1 * 24 * 60, // -ve value to ensure empty trash request are processed in next cron run
- RemoveComplianceHoldQueue: -1 * 24 * 60, // -ve value to ensure compliance hold is removed in next cron run
+ DeleteObjectDelay string
}
const (
@@ -124,8 +114,28 @@ func (repo *QueueRepository) DeleteItem(queueName string, item string) error {
return stacktrace.Propagate(err, "")
}
+func getDeleteObjectDelayInt(DeleteObjectDelay *string) int64 {
+ DeleteObjectDelayInt, err := strconv.ParseInt(*DeleteObjectDelay, 10, 64)
+ if err != nil || DeleteObjectDelayInt == 0 {
+ DeleteObjectDelayInt = 45 * 24 * 60 // defaults to 45 days in minutes
+ }
+ logrus.Debugf("DeleteObjectDelay: %d minutes", DeleteObjectDelayInt)
+ return DeleteObjectDelayInt
+}
+
// GetItemsReadyForDeletion method, for a given queue name, returns a list of QueueItem which are ready for deletion
func (repo *QueueRepository) GetItemsReadyForDeletion(queueName string, count int) ([]QueueItem, error) {
+ // itemDeletionDelayInMinMap tracks the delay (in min) after which an item is ready to be processed.
+ // -ve entry indicates that the item should be processed immediately, without any delay.
+ var itemDeletionDelayInMinMap = map[string]int64{
+ DropFileEncMedataQueue: -1 * 24 * 60, // -ve value to ensure attributes are immediately removed
+ DeleteObjectQueue: getDeleteObjectDelayInt(&repo.DeleteObjectDelay),
+ DeleteEmbeddingsQueue: -1 * 24 * 60, // -ve value to ensure embeddings are immediately removed
+ TrashCollectionQueueV3: -1 * 24 * 60, // -ve value to ensure collections are immediately marked as trashed
+ TrashEmptyQueue: -1 * 24 * 60, // -ve value to ensure empty trash request are processed in next cron run
+ RemoveComplianceHoldQueue: -1 * 24 * 60, // -ve value to ensure compliance hold is removed in next cron run
+ }
+
delayInMin, ok := itemDeletionDelayInMinMap[queueName]
if !ok {
return nil, stacktrace.Propagate(fmt.Errorf("missing delay for %s", queueName), "")
|