From c68690726162f0ab0efb62f1cb455001c06b0fa7 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 3 Jan 2019 17:25:43 +0100 Subject: mod_mam: Perform message expiry based on building an index by date For each day, store a set of users that have new messages. To expire messages, we collect the union of sets of users from dates that fall outside the cleanup range. The previous algoritm did not work well with many users, especially with the default settings. --- plugins/mod_mam/mod_mam.lua | 70 +++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 31 deletions(-) (limited to 'plugins/mod_mam/mod_mam.lua') diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 94bedbb1..18f84752 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -33,7 +33,7 @@ local is_stanza = st.is_stanza; local tostring = tostring; local time_now = os.time; local m_min = math.min; -local timestamp, timestamp_parse = require "util.datetime".datetime, require "util.datetime".parse; +local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date"); local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50); local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://jabber.org/protocol/chatstates" }); @@ -46,13 +46,8 @@ if not archive.find then end local use_total = module:get_option_boolean("mam_include_total", true); -local cleanup; - -local function schedule_cleanup(username) - if cleanup and not cleanup[username] then - table.insert(cleanup, username); - cleanup[username] = true; - end +function schedule_cleanup() + -- replaced by non-noop later if cleanup is enabled end -- Handle prefs. @@ -96,7 +91,6 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event) local qid = query.attr.queryid; get_prefs(origin.username, true); - schedule_cleanup(origin.username); -- Search query parameters local qwith, qstart, qend; @@ -212,6 +206,7 @@ end local function shall_store(user, who) -- TODO Cache this? if not um.user_exists(user, host) then + module:log("debug", "%s@%s does not exist", user, host) return false; end local prefs = get_prefs(user); @@ -329,6 +324,9 @@ module:hook("pre-message/full", strip_stanza_id_after_other_events, -1); local cleanup_after = module:get_option_string("archive_expires_after", "1w"); local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60); if cleanup_after ~= "never" then + local cleanup_storage = module:open_store("archive_cleanup"); + local cleanup_map = module:open_store("archive_cleanup", "map"); + local day = 86400; local multipliers = { d = day, w = day * 7, m = 31 * day, y = 365.2425 * day }; local n, m = cleanup_after:lower():match("(%d+)%s*([dwmy]?)"); @@ -346,33 +344,43 @@ if cleanup_after ~= "never" then return false; end - -- Set of known users to do message expiry for - -- Populated either below or when new messages are added - cleanup = {}; + -- For each day, store a set of users that have new messages. To expire + -- messages, we collect the union of sets of users from dates that fall + -- outside the cleanup range. - -- Iterating over users is not supported by all authentication modules - -- Catch and ignore error if not supported - pcall(function () - -- If this works, then we schedule cleanup for all known users on startup - for user in um.users(module.host) do - schedule_cleanup(user); - end - end); + function schedule_cleanup(username, date) + cleanup_map:set(date or datestamp(), username, true); + end - -- At odd intervals, delete old messages for one user - module:add_timer(math.random(10, 60), function() - local user = table.remove(cleanup, 1); - if user then - module:log("debug", "Removing old messages for user %q", user); + cleanup_runner = require "util.async".runner(function () + local users = {}; + local cut_off = datestamp(os.time() - cleanup_after); + for date in cleanup_storage:users() do + if date < cut_off then + module:log("debug", "Messages from %q should be expired", date); + local messages_this_day = cleanup_storage:get(date); + if messages_this_day then + for user in pairs(messages_this_day) do + users[user] = true; + end + cleanup_storage:set(date, nil); + end + end + end + local sum, num_users = 0, 0; + for user in pairs(users) do local ok, err = archive:delete(user, { ["end"] = os.time() - cleanup_after; }) - if not ok then - module:log("warn", "Could not expire archives for user %s: %s", user, err); - elseif type(ok) == "number" then - module:log("debug", "Removed %d messages", ok); + if ok then + num_users = num_users + 1; + sum = sum + tonumber(ok) or 0; end - cleanup[user] = nil; end - return math.random(cleanup_interval, cleanup_interval * 2); + module:log("info", "Deleted expired %d messages for %d users", sum, num_users); + end); + + cleanup_task = module:add_timer(1, function () + cleanup_runner:run(true); + return cleanup_interval; end); else module:log("debug", "Archive expiry disabled"); -- cgit v1.2.3 From 3738686f06f0156da85df269050f3f754c7603cd Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 4 Jan 2019 10:14:55 +0100 Subject: mod_mam: Fix word order in log message --- plugins/mod_mam/mod_mam.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/mod_mam/mod_mam.lua') diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 18f84752..35a4b9a0 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -375,7 +375,7 @@ if cleanup_after ~= "never" then sum = sum + tonumber(ok) or 0; end end - module:log("info", "Deleted expired %d messages for %d users", sum, num_users); + module:log("info", "Deleted %d expired messages for %d users", sum, num_users); end); cleanup_task = module:add_timer(1, function () -- cgit v1.2.3 From 2ac699495592895c1cde86cb0ba2dc25c254a4eb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 6 Jan 2019 09:34:59 +0100 Subject: mod_mam: Measure how long it takes to run the message expiry job job --- plugins/mod_mam/mod_mam.lua | 3 +++ 1 file changed, 3 insertions(+) (limited to 'plugins/mod_mam/mod_mam.lua') diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 35a4b9a0..5ba04d68 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -351,8 +351,10 @@ if cleanup_after ~= "never" then function schedule_cleanup(username, date) cleanup_map:set(date or datestamp(), username, true); end + local cleanup_time = module:measure("cleanup", "times"); cleanup_runner = require "util.async".runner(function () + local cleanup_done = cleanup_time(); local users = {}; local cut_off = datestamp(os.time() - cleanup_after); for date in cleanup_storage:users() do @@ -376,6 +378,7 @@ if cleanup_after ~= "never" then end end module:log("info", "Deleted %d expired messages for %d users", sum, num_users); + cleanup_done(); end); cleanup_task = module:add_timer(1, function () -- cgit v1.2.3 From bdfc36fc8caa83c2919c0df1f46b91232af09096 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 6 Jan 2019 09:44:55 +0100 Subject: mod_mam: Handle expiry of messages that expire in the middle of the cut-off day --- plugins/mod_mam/mod_mam.lua | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'plugins/mod_mam/mod_mam.lua') diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 5ba04d68..d2ca709b 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -358,14 +358,18 @@ if cleanup_after ~= "never" then local users = {}; local cut_off = datestamp(os.time() - cleanup_after); for date in cleanup_storage:users() do - if date < cut_off then + if date <= cut_off then module:log("debug", "Messages from %q should be expired", date); local messages_this_day = cleanup_storage:get(date); if messages_this_day then for user in pairs(messages_this_day) do users[user] = true; end - cleanup_storage:set(date, nil); + if date < cut_off then + -- Messages from the same day as the cut-off might not have expired yet, + -- but all earlier will have, so clear storage for those days. + cleanup_storage:set(date, nil); + end end end end -- cgit v1.2.3 From de09c462e53b412b30463d55400328b4316a4c45 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 20 Jan 2019 20:24:17 +0100 Subject: mod_mam: Fix operator precedence (thanks mimi89999) --- plugins/mod_mam/mod_mam.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/mod_mam/mod_mam.lua') diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index d2ca709b..67bf177e 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -378,7 +378,7 @@ if cleanup_after ~= "never" then local ok, err = archive:delete(user, { ["end"] = os.time() - cleanup_after; }) if ok then num_users = num_users + 1; - sum = sum + tonumber(ok) or 0; + sum = sum + (tonumber(ok) or 0); end end module:log("info", "Deleted %d expired messages for %d users", sum, num_users); -- cgit v1.2.3 From 9dce1de7674543e1daa3664c75a709845b1330fb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 7 Nov 2017 18:58:52 +0100 Subject: mod_mam: Trim archive when quota has been exceeded --- plugins/mod_mam/mod_mam.lua | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'plugins/mod_mam/mod_mam.lua') diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 88cbaa08..5be4bc24 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -40,6 +40,10 @@ local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://ja local archive_store = module:get_option_string("archive_store", "archive"); local archive = module:open_store(archive_store, "archive"); +local cleanup_after = module:get_option_string("archive_expires_after", "1w"); +local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60); +local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000); + if not archive.find then error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n" .."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information"); @@ -295,7 +299,20 @@ local function message_handler(event, c2s) log("debug", "Archiving stanza: %s", stanza:top_tag()); -- And stash it - local ok = archive:append(store_user, nil, clone_for_storage, time_now(), with); + local time = time_now(); + local ok, err = archive:append(store_user, nil, clone_for_storage, time, with); + if not ok and err == "quota-limit" then + if archive.caps and archive.caps.truncate then + module:log("debug", "User '%s' over quota, trimming archive", store_user); + local truncated = archive:delete(store_user, { + truncate = archive_item_limit - 1; + ["end"] = type(cleanup_after) == "number" and (os.time() - cleanup_after) or nil; + }); + if truncated then + ok, err = archive:append(store_user, nil, clone_for_storage, time, with); + end + end + end if ok then local clone_for_other_handlers = st.clone(stanza); local id = ok; @@ -321,8 +338,6 @@ end module:hook("pre-message/bare", strip_stanza_id_after_other_events, -1); module:hook("pre-message/full", strip_stanza_id_after_other_events, -1); -local cleanup_after = module:get_option_string("archive_expires_after", "1w"); -local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60); if cleanup_after ~= "never" then local cleanup_storage = module:open_store("archive_cleanup"); local cleanup_map = module:open_store("archive_cleanup", "map"); -- cgit v1.2.3 From 2fed4a88c282ca6aa62d9641f3360a021ec0cebe Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:22:21 +0100 Subject: mod_mam: On quota hit, separately delete by time and by item count This is to work around a possible SQL issue where offsets and time stamps don't interact correctly. --- plugins/mod_mam/mod_mam.lua | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'plugins/mod_mam/mod_mam.lua') diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 5be4bc24..632de9ea 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -302,11 +302,19 @@ local function message_handler(event, c2s) local time = time_now(); local ok, err = archive:append(store_user, nil, clone_for_storage, time, with); if not ok and err == "quota-limit" then - if archive.caps and archive.caps.truncate then - module:log("debug", "User '%s' over quota, trimming archive", store_user); + if type(cleanup_after) == "number" then + module:log("debug", "User '%s' over quota, cleaning archive", store_user); + local cleaned = archive:delete(store_user, { + ["end"] = (os.time() - cleanup_after); + }); + if cleaned then + ok, err = archive:append(store_user, nil, clone_for_storage, time, with); + end + end + if not ok and (archive.caps and archive.caps.truncate) then + module:log("debug", "User '%s' over quota, truncating archive", store_user); local truncated = archive:delete(store_user, { truncate = archive_item_limit - 1; - ["end"] = type(cleanup_after) == "number" and (os.time() - cleanup_after) or nil; }); if truncated then ok, err = archive:append(store_user, nil, clone_for_storage, time, with); -- cgit v1.2.3 From fcda870911ff3a9ae66a3051c0e1a281f7b99772 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 07:16:05 +0200 Subject: mod_mam: Propagate item-not-found to client (fixes #1325) --- plugins/mod_mam/mod_mam.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'plugins/mod_mam/mod_mam.lua') diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 632de9ea..317ddac1 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -142,7 +142,11 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event) }); if not data then - origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err)); + if err == "item-not-found" then + origin.send(st.error_reply(stanza, "modify", "item-not-found")); + else + origin.send(st.error_reply(stanza, "cancel", "internal-server-error")); + end return true; end local total = tonumber(err); -- cgit v1.2.3