aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorKim Alvefur <zash@zash.se>2022-01-11 04:15:29 +0100
committerKim Alvefur <zash@zash.se>2022-01-11 04:15:29 +0100
commitec9ec1b9a0398d3eed10609376e7d78e25369840 (patch)
tree2224e14dfea03f858f251dc8daf8a483b7da2610
parent2a0311098cedfd93ecf0f0a83b8bcc9066531d09 (diff)
downloadprosody-ec9ec1b9a0398d3eed10609376e7d78e25369840.tar.gz
prosody-ec9ec1b9a0398d3eed10609376e7d78e25369840.zip
mod_http_file_share: Always measure total disk usage for statistics!
Metrics available or not depending on configuration is weird, even tho it might be expensive to calculate and it's only really needed when there is a global quota. Default quota is set to infinity, which is essentially what it was. Reports NaN if there is an error, which should count as over the infinite default quota.
-rw-r--r--plugins/mod_http_file_share.lua95
1 files changed, 46 insertions, 49 deletions
diff --git a/plugins/mod_http_file_share.lua b/plugins/mod_http_file_share.lua
index 447a9ecc..8773f3a4 100644
--- a/plugins/mod_http_file_share.lua
+++ b/plugins/mod_http_file_share.lua
@@ -20,6 +20,9 @@ local hi = require "util.human.units";
local cache = require "util.cache";
local lfs = require "lfs";
+local unknown = math.abs(0/0);
+local unlimited = math.huge;
+
local namespace = "urn:xmpp:http:upload:0";
module:depends("disco");
@@ -38,7 +41,7 @@ local file_types = module:get_option_set(module.name .. "_allowed_file_types", {
local safe_types = module:get_option_set(module.name .. "_safe_file_types", {"image/*","video/*","audio/*","text/plain"});
local expiry = module:get_option_number(module.name .. "_expires_after", 7 * 86400);
local daily_quota = module:get_option_number(module.name .. "_daily_quota", file_size_limit*10); -- 100 MB / day
-local total_storage_limit = module:get_option_number(module.name.."_global_quota", nil);
+local total_storage_limit = module:get_option_number(module.name.."_global_quota", unlimited);
local access = module:get_option_set(module.name .. "_access", {});
@@ -60,30 +63,29 @@ local upload_errors = errors.init(module.name, namespace, {
};
filesizefmt = { type = "modify"; condition = "bad-request"; text = "File size must be positive integer"; };
quota = { type = "wait"; condition = "resource-constraint"; text = "Daily quota reached"; };
- unknowntotal = { type = "wait"; condition = "undefined-condition"; text = "Server storage usage not yet calculated" };
outofdisk = { type = "wait"; condition = "resource-constraint"; text = "Server global storage quota reached" };
});
local upload_cache = cache.new(1024);
local quota_cache = cache.new(1024);
-local total_storage_usage = nil;
+local total_storage_usage = unknown;
local measure_upload_cache_size = module:measure("upload_cache", "amount");
local measure_quota_cache_size = module:measure("quota_cache", "amount");
-local measure_total_storage_usage = nil;
-if total_storage_limit then
+local measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" });
+
+do
local total, err = persist_stats:get(nil, "total");
- if not err then total_storage_usage = tonumber(total) or 0; end
- measure_total_storage_usage = module:measure("total_storage", "amount", { unit = "bytes" });
+ if not err then
+ total_storage_usage = tonumber(total) or 0;
+ end
end
module:hook_global("stats-update", function ()
measure_upload_cache_size(upload_cache:count());
measure_quota_cache_size(quota_cache:count());
- if total_storage_limit and total_storage_usage then
- measure_total_storage_usage(total_storage_usage);
- end
+ measure_total_storage_usage(total_storage_usage);
end);
local buckets = {};
@@ -95,7 +97,14 @@ end
local measure_uploads = module:measure("upload", "sizes", {buckets = buckets});
-- Convenience wrapper for logging file sizes
-local function B(bytes) return hi.format(bytes, "B", "b"); end
+local function B(bytes)
+ if bytes ~= bytes then
+ return "unknown"
+ elseif bytes == unlimited then
+ return "unlimited";
+ end
+ return hi.format(bytes, "B", "b");
+end
local function get_filename(slot, create)
return dm.getpath(slot, module.host, module.name, "bin", create)
@@ -141,13 +150,9 @@ function may_upload(uploader, filename, filesize, filetype) -- > boolean, error
return false, upload_errors.new("filesize");
end
- if total_storage_limit then
- if not total_storage_usage then
- return false, upload_errors.new("unknowntotal");
- elseif total_storage_usage + filesize > total_storage_limit then
- module:log("warn", "Global storage quota reached, at %s!", B(total_storage_usage));
- return false, upload_errors.new("outofdisk");
- end
+ if total_storage_usage + filesize > total_storage_limit then
+ module:log("warn", "Global storage quota reached, at %s / %s!", B(total_storage_usage), B(total_storage_limit));
+ return false, upload_errors.new("outofdisk");
end
local uploader_quota = get_daily_quota(uploader);
@@ -217,10 +222,8 @@ function handle_slot_request(event)
return true;
end
- if total_storage_usage then
- total_storage_usage = total_storage_usage + filesize;
- module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
- end
+ total_storage_usage = total_storage_usage + filesize;
+ module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
local cached_quota = quota_cache:get(uploader);
if cached_quota and cached_quota.time > os.time()-86400 then
@@ -472,11 +475,7 @@ if expiry >= 0 and not external_base_url then
end
module:log("info", "Pruning expired files uploaded earlier than %s", dt.datetime(boundary_time));
- if total_storage_usage then
- module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
- elseif total_storage_limit then
- module:log("debug", "Global quota %s / %s", "not yet calculated", B(total_storage_limit));
- end
+ module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
local obsolete_uploads = array();
local num_expired = 0;
@@ -513,11 +512,9 @@ if expiry >= 0 and not external_base_url then
deletion_query = {ids = obsolete_uploads};
end
- if total_storage_usage then
- total_storage_usage = total_storage_usage - size_sum;
- module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
- persist_stats:set(nil, "total", total_storage_usage);
- end
+ total_storage_usage = total_storage_usage - size_sum;
+ module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
+ persist_stats:set(nil, "total", total_storage_usage);
if #obsolete_uploads == 0 then
module:log("debug", "No metadata to remove");
@@ -535,27 +532,27 @@ if expiry >= 0 and not external_base_url then
end);
end
-if total_storage_limit then
- local summary_start = module:measure("summary", "times");
+local summary_start = module:measure("summary", "times");
- module:weekly("Global quota check", function()
- local summary_done = summary_start();
- local iter = assert(uploads:find(nil));
+module:weekly("Calculate total storage usage", function()
+ local summary_done = summary_start();
+ local iter = assert(uploads:find(nil));
- local count, sum = 0, 0;
- for _, file in iter do
- sum = sum + tonumber(file.attr.size);
- count = count + 1;
- end
+ local count, sum = 0, 0;
+ for _, file in iter do
+ sum = sum + tonumber(file.attr.size);
+ count = count + 1;
+ end
- module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
+ module:log("info", "Uploaded files total: %s in %d files", B(sum), count);
+ if persist_stats:set(nil, "total", sum) then
total_storage_usage = sum;
- module:log("debug", "Global quota %s / %s", B(total_storage_usage), B(total_storage_limit));
- persist_stats:set(nil, "total", sum);
- summary_done();
- end);
-
-end
+ else
+ total_storage_usage = unknown;
+ end
+ module:log("debug", "Total storage usage: %s / %s", B(total_storage_usage), B(total_storage_limit));
+ summary_done();
+end);
-- Reachable from the console
function check_files(query)