From ae7f6c34f0b2f00c616102a7c5fa3a9fe0ed9541 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 12 Sep 2021 01:38:33 +0200 Subject: mod_http_file_share: Add optional global quota on total storage usage Before, maximum storage usage (assuming all users upload as much as they could) would depend on the quota, retention period and number of users. Since number of users can vary, this makes it hard to know how much storage will be needed. Adding a limit to the total overall storage use solves this, making it simple to set it to some number based on what storage is actually available. Summary job run less often than the prune job since it touches the entire archive; and started before the prune job since it's needed before the first upload. --- plugins/mod_http_file_share.lua | 58 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/plugins/mod_http_file_share.lua b/plugins/mod_http_file_share.lua index 03274120..ef6da4a1 100644 --- a/plugins/mod_http_file_share.lua +++ b/plugins/mod_http_file_share.lua @@ -37,6 +37,7 @@ local file_types = module:get_option_set(module.name .. "_allowed_file_types", { local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"}); local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400); local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day +local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil); local access = module:get_option_set(module.name .. "_access", {}); @@ -58,11 +59,15 @@ local upload_errors = errors.init(module.name, namespace, { }; filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; }; quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; }; + unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" }; + outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" }; }); local upload_cache = cache.new(1024); local quota_cache = cache.new(1024); +local total_storage_usage = nil; + local measure_upload_cache_size = module:measure("upload_cache", "amount"); local measure_quota_cache_size = module:measure("quota_cache", "amount"); @@ -126,6 +131,15 @@ function may_upload(uploader, filename, filesize, filetype) -- > boolean, error return false, upload_errors.new("filesize"); end + if total_storage_limit then + if not total_storage_usage then + return false, upload_errors.new("unknowntotal"); + elseif total_storage_usage + filesize > total_storage_limit then + module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage)); + return false, upload_errors.new("outofdisk"); + end + end + local uploader_quota = get_daily_quota(uploader); if uploader_quota + filesize > daily_quota then return false, upload_errors.new("quota"); @@ -193,6 +207,11 @@ function handle_slot_request(event) return true; end + if total_storage_usage then + total_storage_usage = total_storage_usage + filesize; + module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); + end + local cached_quota = quota_cache:get(uploader); if cached_quota and cached_quota.time > os.time()-86400 then cached_quota.size = cached_quota.size + filesize; @@ -433,13 +452,16 @@ if expiry >= 0 and not external_base_url then end module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time)); + module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); local obsolete_uploads = array(); local i = 0; - for slot_id in iter do + local size_sum = 0; + for slot_id, slot_info in iter do i = i + 1; obsolete_uploads:push(slot_id); upload_cache:set(slot_id, nil); + size_sum = size_sum + tonumber(slot_info.attr.size); end sleep(0.1); @@ -463,7 +485,11 @@ if expiry >= 0 and not external_base_url then local deletion_query = {["end"] = boundary_time}; if not problem_deleting then - module:log("info", "All (%d) expired files successfully deleted", n); + module:log("info", "All (%d, %s) expired files successfully deleted", n, B(size_sum)); + if total_storage_usage then + total_storage_usage = total_storage_usage - size_sum; + module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); + end -- we can delete based on time else module:log("warn", "%d out of %d expired files could not be deleted", n-#obsolete_uploads, n); @@ -471,6 +497,7 @@ if expiry >= 0 and not external_base_url then -- successfully deleted, and then try again with the failed ones. -- eventually the admin ought to notice and fix the permissions or -- whatever the problem is. + -- total_storage_limit will be inaccurate until this has been resolved deletion_query = {ids = obsolete_uploads}; end @@ -489,12 +516,37 @@ if expiry >= 0 and not external_base_url then prune_done(); end); - module:add_timer(1, function () + module:add_timer(5, function () reaper_task:run(os.time()-expiry); return 60*60; end); end +if total_storage_limit then + local async = require "util.async"; + + local summarizer_task = async.runner(function() + local summary_done = module:measure("summary", "times"); + local iter = assert(uploads:find(nil)); + + local count, sum = 0, 0; + for _, file in iter do + sum = sum + tonumber(file.attr.size); + count = count + 1; + end + + module:log("info", "Uploaded files total: %s in %d files", B(sum), count); + total_storage_usage = sum; + module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit)); + summary_done(); + end); + + module:add_timer(1, function() + summarizer_task:run(true); + return 11 * 60 * 60; + end); +end + -- Reachable from the console function check_files(query) local issues = {}; -- cgit v1.2.3