aboutsummaryrefslogtreecommitdiffstats
path: root/plugins/mod_http_file_share.lua
diff options
context:
space:
mode:
authorKim Alvefur <zash@zash.se>2021-09-12 01:38:33 +0200
committerKim Alvefur <zash@zash.se>2021-09-12 01:38:33 +0200
commitae7f6c34f0b2f00c616102a7c5fa3a9fe0ed9541 (patch)
tree4c56ff10f957c2a6f7ed0931384dadb1ca48b311 /plugins/mod_http_file_share.lua
parent97aa610a42dd563d6e23c84de13550997f036cd6 (diff)
downloadprosody-ae7f6c34f0b2f00c616102a7c5fa3a9fe0ed9541.tar.gz
prosody-ae7f6c34f0b2f00c616102a7c5fa3a9fe0ed9541.zip
mod_http_file_share: Add optional global quota on total storage usage
Before, maximum storage usage (assuming all users upload as much as they could) would depend on the quota, retention period and number of users. Since number of users can vary, this makes it hard to know how much storage will be needed. Adding a limit to the total overall storage use solves this, making it simple to set it to some number based on what storage is actually available. Summary job run less often than the prune job since it touches the entire archive; and started before the prune job since it's needed before the first upload.
Diffstat (limited to 'plugins/mod_http_file_share.lua')
-rw-r--r--plugins/mod_http_file_share.lua58
1 files changed, 55 insertions, 3 deletions
diff --git a/plugins/mod_http_file_share.lua b/plugins/mod_http_file_share.lua
index 03274120..ef6da4a1 100644
--- a/plugins/mod_http_file_share.lua
+++ b/plugins/mod_http_file_share.lua
@@ -37,6 +37,7 @@ local file_types = module:get_option_set(module.name .. "_allowed_file_types", {
local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
+local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil);
local access = module:get_option_set(module.name .. "_access", {});
@@ -58,11 +59,15 @@ local upload_errors = errors.init(module.name, namespace, {
};
filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
+ unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
+ outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
});
local upload_cache = cache.new(1024);
local quota_cache = cache.new(1024);
+local total_storage_usage = nil;
+
local measure_upload_cache_size = module:measure("upload_cache", "amount");
local measure_quota_cache_size = module:measure("quota_cache", "amount");
@@ -126,6 +131,15 @@ function may_upload(uploader, filename, filesize, filetype) -- > boolean, error
return false, upload_errors.new("filesize");
end
+ if total_storage_limit then
+ if not total_storage_usage then
+ return false, upload_errors.new("unknowntotal");
+ elseif total_storage_usage + filesize > total_storage_limit then
+ module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
+ return false, upload_errors.new("outofdisk");
+ end
+ end
+
local uploader_quota = get_daily_quota(uploader);
if uploader_quota + filesize > daily_quota then
return false, upload_errors.new("quota");
@@ -193,6 +207,11 @@ function handle_slot_request(event)
return true;
end
+ if total_storage_usage then
+ total_storage_usage = total_storage_usage + filesize;
+ module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+ end
+
local cached_quota = quota_cache:get(uploader);
if cached_quota and cached_quota.time > os.time()-86400 then
cached_quota.size = cached_quota.size + filesize;
@@ -433,13 +452,16 @@ if expiry >= 0 and not external_base_url then
end
module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
+ module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
local obsolete_uploads = array();
local i = 0;
- for slot_id in iter do
+ local size_sum = 0;
+ for slot_id, slot_info in iter do
i = i + 1;
obsolete_uploads:push(slot_id);
upload_cache:set(slot_id, nil);
+ size_sum = size_sum + tonumber(slot_info.attr.size);
end
sleep(0.1);
@@ -463,7 +485,11 @@ if expiry >= 0 and not external_base_url then
local deletion_query = {["end"] = boundary_time};
if not problem_deleting then
- module:log("info", "All (%d) expired files successfully deleted", n);
+ module:log("info", "All (%d, %s) expired files successfully deleted", n, B(size_sum));
+ if total_storage_usage then
+ total_storage_usage = total_storage_usage - size_sum;
+ module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+ end
-- we can delete based on time
else
module:log("warn", "%d out of %d expired files could not be deleted", n-#obsolete_uploads, n);
@@ -471,6 +497,7 @@ if expiry >= 0 and not external_base_url then
-- successfully deleted, and then try again with the failed ones.
-- eventually the admin ought to notice and fix the permissions or
-- whatever the problem is.
+ -- total_storage_limit will be inaccurate until this has been resolved
deletion_query = {ids = obsolete_uploads};
end
@@ -489,12 +516,37 @@ if expiry >= 0 and not external_base_url then
prune_done();
end);
- module:add_timer(1, function ()
+ module:add_timer(5, function ()
reaper_task:run(os.time()-expiry);
return 60*60;
end);
end
+if total_storage_limit then
+ local async = require "util.async";
+
+ local summarizer_task = async.runner(function()
+ local summary_done = module:measure("summary", "times");
+ local iter = assert(uploads:find(nil));
+
+ local count, sum = 0, 0;
+ for _, file in iter do
+ sum = sum + tonumber(file.attr.size);
+ count = count + 1;
+ end
+
+ module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
+ total_storage_usage = sum;
+ module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+ summary_done();
+ end);
+
+ module:add_timer(1, function()
+ summarizer_task:run(true);
+ return 11 * 60 * 60;
+ end);
+end
+
-- Reachable from the console
function check_files(query)
local issues = {};