mod_http_file_share: Add optional global quota on total storage usage

Before, maximum storage usage (assuming all users upload as much as they
could) would depend on the quota, retention period and number of users.
Since number of users can vary, this makes it hard to know how much
storage will be needed.

Adding a limit to the total overall storage use solves this, making it
simple to set it to some number based on what storage is actually
available.

Summary job run less often than the prune job since it touches the
entire archive; and started before the prune job since it's needed
before the first upload.

1 files changed, 55 insertions, 3 deletions

diff --git a/plugins/mod_http_file_share.lua b/plugins/mod_http_file_share.lua
index 03274120..ef6da4a1 100644
--- a/plugins/mod_http_file_share.lua
+++ b/plugins/mod_http_file_share.lua
@@ -37,6 +37,7 @@ local file_types = module:get_option_set(module.name .. "_allowed_file_types", {
 local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
 local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
 local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
+local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil);
 
 local access = module:get_option_set(module.name .. "_access", {});
 
@@ -58,11 +59,15 @@ local upload_errors = errors.init(module.name, namespace, {
 	};
 	filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
 	quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
+	unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
+	outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
 });
 
 local upload_cache = cache.new(1024);
 local quota_cache = cache.new(1024);
 
+local total_storage_usage = nil;
+
 local measure_upload_cache_size = module:measure("upload_cache", "amount");
 local measure_quota_cache_size = module:measure("quota_cache", "amount");
 
@@ -126,6 +131,15 @@ function may_upload(uploader, filename, filesize, filetype) -- > boolean, error
 		return false, upload_errors.new("filesize");
 	end
 
+	if total_storage_limit then
+		if not total_storage_usage  then
+			return false, upload_errors.new("unknowntotal");
+		elseif total_storage_usage + filesize > total_storage_limit then
+			module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
+			return false, upload_errors.new("outofdisk");
+		end
+	end
+
 	local uploader_quota = get_daily_quota(uploader);
 	if uploader_quota + filesize > daily_quota then
 		return false, upload_errors.new("quota");
@@ -193,6 +207,11 @@ function handle_slot_request(event)
 		return true;
 	end
 
+	if total_storage_usage then
+		total_storage_usage = total_storage_usage + filesize;
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+	end
+
 	local cached_quota = quota_cache:get(uploader);
 	if cached_quota and cached_quota.time > os.time()-86400 then
 		cached_quota.size = cached_quota.size + filesize;
@@ -433,13 +452,16 @@ if expiry >= 0 and not external_base_url then
 		end
 
 		module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
 
 		local obsolete_uploads = array();
 		local i = 0;
-		for slot_id in iter do
+		local size_sum = 0;
+		for slot_id, slot_info in iter do
 			i = i + 1;
 			obsolete_uploads:push(slot_id);
 			upload_cache:set(slot_id, nil);
+			size_sum = size_sum + tonumber(slot_info.attr.size);
 		end
 
 		sleep(0.1);
@@ -463,7 +485,11 @@ if expiry >= 0 and not external_base_url then
 
 		local deletion_query = {["end"] = boundary_time};
 		if not problem_deleting then
-			module:log("info", "All (%d) expired files successfully deleted", n);
+			module:log("info", "All (%d, %s) expired files successfully deleted", n, B(size_sum));
+			if total_storage_usage then
+				total_storage_usage = total_storage_usage - size_sum;
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+			end
 			-- we can delete based on time
 		else
 			module:log("warn", "%d out of %d expired files could not be deleted", n-#obsolete_uploads, n);
@@ -471,6 +497,7 @@ if expiry >= 0 and not external_base_url then
 			-- successfully deleted, and then try again with the failed ones.
 			-- eventually the admin ought to notice and fix the permissions or
 			-- whatever the problem is.
+			-- total_storage_limit will be inaccurate until this has been resolved
 			deletion_query = {ids = obsolete_uploads};
 		end
 
@@ -489,12 +516,37 @@ if expiry >= 0 and not external_base_url then
 		prune_done();
 	end);
 
-	module:add_timer(1, function ()
+	module:add_timer(5, function ()
 		reaper_task:run(os.time()-expiry);
 		return 60*60;
 	end);
 end
 
+if total_storage_limit then
+	local async = require "util.async";
+
+	local summarizer_task = async.runner(function()
+		local summary_done = module:measure("summary", "times");
+		local iter = assert(uploads:find(nil));
+
+		local count, sum = 0, 0;
+		for _, file in iter do
+			sum = sum + tonumber(file.attr.size);
+			count = count + 1;
+		end
+
+		module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
+		total_storage_usage = sum;
+		module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
+		summary_done();
+	end);
+
+	module:add_timer(1, function()
+		summarizer_task:run(true);
+		return 11 * 60 * 60;
+	end);
+end
+
 -- Reachable from the console
 function check_files(query)
 	local issues = {};