From c68690726162f0ab0efb62f1cb455001c06b0fa7 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 3 Jan 2019 17:25:43 +0100
Subject: mod_mam: Perform message expiry based on building an index by date

For each day, store a set of users that have new messages. To expire
messages, we collect the union of sets of users from dates that fall
outside the cleanup range.

The previous algoritm did not work well with many users, especially with
the default settings.
---
 plugins/mod_mam/mod_mam.lua | 70 +++++++++++++++++++++++++--------------------
 1 file changed, 39 insertions(+), 31 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 94bedbb1..18f84752 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -33,7 +33,7 @@ local is_stanza = st.is_stanza;
 local tostring = tostring;
 local time_now = os.time;
 local m_min = math.min;
-local timestamp, timestamp_parse = require "util.datetime".datetime, require "util.datetime".parse;
+local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date");
 local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50);
 local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://jabber.org/protocol/chatstates" });
 
@@ -46,13 +46,8 @@ if not archive.find then
 end
 local use_total = module:get_option_boolean("mam_include_total", true);
 
-local cleanup;
-
-local function schedule_cleanup(username)
-	if cleanup and not cleanup[username] then
-		table.insert(cleanup, username);
-		cleanup[username] = true;
-	end
+function schedule_cleanup()
+	-- replaced by non-noop later if cleanup is enabled
 end
 
 -- Handle prefs.
@@ -96,7 +91,6 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local qid = query.attr.queryid;
 
 	get_prefs(origin.username, true);
-	schedule_cleanup(origin.username);
 
 	-- Search query parameters
 	local qwith, qstart, qend;
@@ -212,6 +206,7 @@ end
 local function shall_store(user, who)
 	-- TODO Cache this?
 	if not um.user_exists(user, host) then
+		module:log("debug", "%s@%s does not exist", user, host)
 		return false;
 	end
 	local prefs = get_prefs(user);
@@ -329,6 +324,9 @@ module:hook("pre-message/full", strip_stanza_id_after_other_events, -1);
 local cleanup_after = module:get_option_string("archive_expires_after", "1w");
 local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
 if cleanup_after ~= "never" then
+	local cleanup_storage = module:open_store("archive_cleanup");
+	local cleanup_map = module:open_store("archive_cleanup", "map");
+
 	local day = 86400;
 	local multipliers = { d = day, w = day * 7, m = 31 * day, y = 365.2425 * day };
 	local n, m = cleanup_after:lower():match("(%d+)%s*([dwmy]?)");
@@ -346,33 +344,43 @@ if cleanup_after ~= "never" then
 		return false;
 	end
 
-	-- Set of known users to do message expiry for
-	-- Populated either below or when new messages are added
-	cleanup = {};
+	-- For each day, store a set of users that have new messages. To expire
+	-- messages, we collect the union of sets of users from dates that fall
+	-- outside the cleanup range.
 
-	-- Iterating over users is not supported by all authentication modules
-	-- Catch and ignore error if not supported
-	pcall(function ()
-		-- If this works, then we schedule cleanup for all known users on startup
-		for user in um.users(module.host) do
-			schedule_cleanup(user);
-		end
-	end);
+	function schedule_cleanup(username, date)
+		cleanup_map:set(date or datestamp(), username, true);
+	end
 
-	-- At odd intervals, delete old messages for one user
-	module:add_timer(math.random(10, 60), function()
-		local user = table.remove(cleanup, 1);
-		if user then
-			module:log("debug", "Removing old messages for user %q", user);
+	cleanup_runner = require "util.async".runner(function ()
+		local users = {};
+		local cut_off = datestamp(os.time() - cleanup_after);
+		for date in cleanup_storage:users() do
+			if date < cut_off then
+				module:log("debug", "Messages from %q should be expired", date);
+				local messages_this_day = cleanup_storage:get(date);
+				if messages_this_day then
+					for user in pairs(messages_this_day) do
+						users[user] = true;
+					end
+					cleanup_storage:set(date, nil);
+				end
+			end
+		end
+		local sum, num_users = 0, 0;
+		for user in pairs(users) do
 			local ok, err = archive:delete(user, { ["end"] = os.time() - cleanup_after; })
-			if not ok then
-				module:log("warn", "Could not expire archives for user %s: %s", user, err);
-			elseif type(ok) == "number" then
-				module:log("debug", "Removed %d messages", ok);
+			if ok then
+				num_users = num_users + 1;
+				sum = sum + tonumber(ok) or 0;
 			end
-			cleanup[user] = nil;
 		end
-		return math.random(cleanup_interval, cleanup_interval * 2);
+		module:log("info", "Deleted expired %d messages for %d users", sum, num_users);
+	end);
+
+	cleanup_task = module:add_timer(1, function ()
+		cleanup_runner:run(true);
+		return cleanup_interval;
 	end);
 else
 	module:log("debug", "Archive expiry disabled");
-- 
cgit v1.2.3


From 3738686f06f0156da85df269050f3f754c7603cd Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Fri, 4 Jan 2019 10:14:55 +0100
Subject: mod_mam: Fix word order in log message

---
 plugins/mod_mam/mod_mam.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 18f84752..35a4b9a0 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -375,7 +375,7 @@ if cleanup_after ~= "never" then
 				sum = sum + tonumber(ok) or 0;
 			end
 		end
-		module:log("info", "Deleted expired %d messages for %d users", sum, num_users);
+		module:log("info", "Deleted %d expired messages for %d users", sum, num_users);
 	end);
 
 	cleanup_task = module:add_timer(1, function ()
-- 
cgit v1.2.3


From 2ac699495592895c1cde86cb0ba2dc25c254a4eb Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Sun, 6 Jan 2019 09:34:59 +0100
Subject: mod_mam: Measure how long it takes to run the message expiry job job

---
 plugins/mod_mam/mod_mam.lua | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 35a4b9a0..5ba04d68 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -351,8 +351,10 @@ if cleanup_after ~= "never" then
 	function schedule_cleanup(username, date)
 		cleanup_map:set(date or datestamp(), username, true);
 	end
+	local cleanup_time = module:measure("cleanup", "times");
 
 	cleanup_runner = require "util.async".runner(function ()
+		local cleanup_done = cleanup_time();
 		local users = {};
 		local cut_off = datestamp(os.time() - cleanup_after);
 		for date in cleanup_storage:users() do
@@ -376,6 +378,7 @@ if cleanup_after ~= "never" then
 			end
 		end
 		module:log("info", "Deleted %d expired messages for %d users", sum, num_users);
+		cleanup_done();
 	end);
 
 	cleanup_task = module:add_timer(1, function ()
-- 
cgit v1.2.3


From bdfc36fc8caa83c2919c0df1f46b91232af09096 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Sun, 6 Jan 2019 09:44:55 +0100
Subject: mod_mam: Handle expiry of messages that expire in the middle of the
 cut-off day

---
 plugins/mod_mam/mod_mam.lua | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 5ba04d68..d2ca709b 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -358,14 +358,18 @@ if cleanup_after ~= "never" then
 		local users = {};
 		local cut_off = datestamp(os.time() - cleanup_after);
 		for date in cleanup_storage:users() do
-			if date < cut_off then
+			if date <= cut_off then
 				module:log("debug", "Messages from %q should be expired", date);
 				local messages_this_day = cleanup_storage:get(date);
 				if messages_this_day then
 					for user in pairs(messages_this_day) do
 						users[user] = true;
 					end
-					cleanup_storage:set(date, nil);
+					if date < cut_off then
+						-- Messages from the same day as the cut-off might not have expired yet,
+						-- but all earlier will have, so clear storage for those days.
+						cleanup_storage:set(date, nil);
+					end
 				end
 			end
 		end
-- 
cgit v1.2.3


From de09c462e53b412b30463d55400328b4316a4c45 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Sun, 20 Jan 2019 20:24:17 +0100
Subject: mod_mam: Fix operator precedence (thanks mimi89999)

---
 plugins/mod_mam/mod_mam.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index d2ca709b..67bf177e 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -378,7 +378,7 @@ if cleanup_after ~= "never" then
 			local ok, err = archive:delete(user, { ["end"] = os.time() - cleanup_after; })
 			if ok then
 				num_users = num_users + 1;
-				sum = sum + tonumber(ok) or 0;
+				sum = sum + (tonumber(ok) or 0);
 			end
 		end
 		module:log("info", "Deleted %d expired messages for %d users", sum, num_users);
-- 
cgit v1.2.3


From 9dce1de7674543e1daa3664c75a709845b1330fb Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 7 Nov 2017 18:58:52 +0100
Subject: mod_mam: Trim archive when quota has been exceeded

---
 plugins/mod_mam/mod_mam.lua | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 88cbaa08..5be4bc24 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -40,6 +40,10 @@ local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://ja
 local archive_store = module:get_option_string("archive_store", "archive");
 local archive = module:open_store(archive_store, "archive");
 
+local cleanup_after = module:get_option_string("archive_expires_after", "1w");
+local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
+
 if not archive.find then
 	error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n"
 		.."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information");
@@ -295,7 +299,20 @@ local function message_handler(event, c2s)
 		log("debug", "Archiving stanza: %s", stanza:top_tag());
 
 		-- And stash it
-		local ok = archive:append(store_user, nil, clone_for_storage, time_now(), with);
+		local time = time_now();
+		local ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+		if not ok and err == "quota-limit" then
+			if archive.caps and archive.caps.truncate then
+				module:log("debug", "User '%s' over quota, trimming archive", store_user);
+				local truncated = archive:delete(store_user, {
+					truncate = archive_item_limit - 1;
+					["end"] = type(cleanup_after) == "number" and (os.time() - cleanup_after) or nil;
+				});
+				if truncated then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+		end
 		if ok then
 			local clone_for_other_handlers = st.clone(stanza);
 			local id = ok;
@@ -321,8 +338,6 @@ end
 module:hook("pre-message/bare", strip_stanza_id_after_other_events, -1);
 module:hook("pre-message/full", strip_stanza_id_after_other_events, -1);
 
-local cleanup_after = module:get_option_string("archive_expires_after", "1w");
-local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
 if cleanup_after ~= "never" then
 	local cleanup_storage = module:open_store("archive_cleanup");
 	local cleanup_map = module:open_store("archive_cleanup", "map");
-- 
cgit v1.2.3


From 2fed4a88c282ca6aa62d9641f3360a021ec0cebe Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Fri, 22 Mar 2019 02:22:21 +0100
Subject: mod_mam: On quota hit, separately delete by time and by item count

This is to work around a possible SQL issue where offsets and time
stamps don't interact correctly.
---
 plugins/mod_mam/mod_mam.lua | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 5be4bc24..632de9ea 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -302,11 +302,19 @@ local function message_handler(event, c2s)
 		local time = time_now();
 		local ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
 		if not ok and err == "quota-limit" then
-			if archive.caps and archive.caps.truncate then
-				module:log("debug", "User '%s' over quota, trimming archive", store_user);
+			if type(cleanup_after) == "number" then
+				module:log("debug", "User '%s' over quota, cleaning archive", store_user);
+				local cleaned = archive:delete(store_user, {
+					["end"] = (os.time() - cleanup_after);
+				});
+				if cleaned then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+			if not ok and (archive.caps and archive.caps.truncate) then
+				module:log("debug", "User '%s' over quota, truncating archive", store_user);
 				local truncated = archive:delete(store_user, {
 					truncate = archive_item_limit - 1;
-					["end"] = type(cleanup_after) == "number" and (os.time() - cleanup_after) or nil;
 				});
 				if truncated then
 					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
-- 
cgit v1.2.3


From fcda870911ff3a9ae66a3051c0e1a281f7b99772 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Sun, 5 May 2019 07:16:05 +0200
Subject: mod_mam: Propagate item-not-found to client (fixes #1325)

---
 plugins/mod_mam/mod_mam.lua | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 632de9ea..317ddac1 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -142,7 +142,11 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
-		origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+		if err == "item-not-found" then
+			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
+		else
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		end
 		return true;
 	end
 	local total = tonumber(err);
-- 
cgit v1.2.3


From 6ab914c24a5bc606f13f69cd5d358074e0e1f788 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 11 Jul 2019 23:22:59 +0200
Subject: mod_mam: Make log message more compact

---
 plugins/mod_mam/mod_mam.lua | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 4e0cf531..0b3b0b82 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -118,10 +118,11 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query, id %s with %s from %s until %s",
-		tostring(qid), qwith or "anyone",
-		qstart and timestamp(qstart) or "the dawn of time",
-		qend and timestamp(qend) or "now");
+	module:log("debug", "Archive query id=%s with=%s when=%s...%s",
+		qid or "-",
+		qwith or "*",
+		qstart and timestamp(qstart) or "",
+		qend and timestamp(qend) or "");
 
 	-- RSM stuff
 	local qset = rsm.get(query);
-- 
cgit v1.2.3


From 219d20f1ba270123c0f739fa2e5f10b3af6b61a3 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 11 Jul 2019 23:30:58 +0200
Subject: mod_mam: Include username that performed query

Not always easy to find from surrounding logs
---
 plugins/mod_mam/mod_mam.lua | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 0b3b0b82..0c50bd2a 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -118,7 +118,8 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query id=%s with=%s when=%s...%s",
+	module:log("debug", "Archive query by %s id=%s with=%s when=%s...%s",
+		origin.username,
 		qid or "-",
 		qwith or "*",
 		qstart and timestamp(qstart) or "",
-- 
cgit v1.2.3


From b396fc61d55211c7ebd43b90677a2b5c24302bd7 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 11 Jul 2019 23:26:02 +0200
Subject: mod_mam: Use stanza id in log message as fallback if no query id

---
 plugins/mod_mam/mod_mam.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 0c50bd2a..cc06b617 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -120,7 +120,7 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 
 	module:log("debug", "Archive query by %s id=%s with=%s when=%s...%s",
 		origin.username,
-		qid or "-",
+		qid or stanza.attr.id,
 		qwith or "*",
 		qstart and timestamp(qstart) or "",
 		qend and timestamp(qend) or "");
-- 
cgit v1.2.3


From c733e10ba2d06213e1eb45d2e31baf3f9919d592 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 11 Jul 2019 23:31:13 +0200
Subject: mod_mam: Include query id in final log message

Should make it easier to find related log messages
---
 plugins/mod_mam/mod_mam.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index cc06b617..5c796f9e 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -197,7 +197,7 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	end
 
 	-- That's all folks!
-	module:log("debug", "Archive query %s completed", tostring(qid));
+	module:log("debug", "Archive query id=%s completed", qid or stanza.attr.id);
 
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
-- 
cgit v1.2.3


From 460efc6b3e2713183e1f3e1fdd0018b90c08c83f Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 11 Jul 2019 23:31:51 +0200
Subject: mod_mam: Log more details in final log message

Saves you from counting messages sent
---
 plugins/mod_mam/mod_mam.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 5c796f9e..613d045a 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -197,7 +197,7 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	end
 
 	-- That's all folks!
-	module:log("debug", "Archive query id=%s completed", qid or stanza.attr.id);
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, #results);
 
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
-- 
cgit v1.2.3


From 006983308dd9300755cf992b2699d73744033a42 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 11 Jul 2019 23:27:01 +0200
Subject: mod_mam: Log query failure reason

The storage engine will usually make a lot of noise for serious errors,
but not always.
---
 plugins/mod_mam/mod_mam.lua | 1 +
 1 file changed, 1 insertion(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 613d045a..a3bf9c42 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -143,6 +143,7 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
+		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);
 		if err == "item-not-found" then
 			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
 		else
-- 
cgit v1.2.3


From c8528f9004a6053ad1a999e2ecc90d07267d5abd Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 11 Jul 2019 23:27:45 +0200
Subject: mod_mam: Log RSM parameters

Helps when trying to figure out why some queries fail, ie when paging
using an archive id that doesn't exist.
---
 plugins/mod_mam/mod_mam.lua | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index a3bf9c42..82bc6b5f 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -131,6 +131,9 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local reverse = qset and qset.before or false;
 	local before, after = qset and qset.before, qset and qset.after;
 	if type(before) ~= "string" then before = nil; end
+	if qset then
+		module:log("debug", "Archive query id=%s rsm=%q", qid or stanza.attr.id, qset);
+	end
 
 	-- Load all the data!
 	local data, err = archive:find(origin.username, {
-- 
cgit v1.2.3


From 3a4f54f014d4bb0714ef910c92500a8a4202c158 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 11 Jul 2019 23:29:59 +0200
Subject: mod_mam: Move final log message to end of query procedure

---
 plugins/mod_mam/mod_mam.lua | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 82bc6b5f..2f6ab531 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -200,13 +200,13 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		first, last = last, first;
 	end
 
-	-- That's all folks!
-	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, #results);
-
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 			:add_child(rsm.generate {
 				first = first, last = last, count = total }));
+
+	-- That's all folks!
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, #results);
 	return true;
 end);
 
-- 
cgit v1.2.3


From 41c458eac5f9df61ab8af811c74c8d89e7f38e98 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Sat, 13 Jul 2019 19:57:43 +0200
Subject: mod_mam: Report correct count of results for forward queries

#results is only correct for backwards queries, the table is unused for
forward queries.
---
 plugins/mod_mam/mod_mam.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 2f6ab531..855e1974 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -206,7 +206,7 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 				first = first, last = last, count = total }));
 
 	-- That's all folks!
-	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, #results);
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);
 	return true;
 end);
 
-- 
cgit v1.2.3


From 40b1e3e0ed1223517f29bcf136fc08a6f33b17f2 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 30 Jul 2019 02:29:36 +0200
Subject: plugins: Remove tostring call from logging

Taken care of by loggingmanager now

Mass-rewrite using lua pattern like `tostring%b()`
---
 plugins/mod_mam/mod_mam.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 855e1974..cfa92ff5 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -224,13 +224,13 @@ local function shall_store(user, who)
 	end
 	local prefs = get_prefs(user);
 	local rule = prefs[who];
-	module:log("debug", "%s's rule for %s is %s", user, who, tostring(rule));
+	module:log("debug", "%s's rule for %s is %s", user, who, rule);
 	if rule ~= nil then
 		return rule;
 	end
 	-- Below could be done by a metatable
 	local default = prefs[false];
-	module:log("debug", "%s's default rule is %s", user, tostring(default));
+	module:log("debug", "%s's default rule is %s", user, default);
 	if default == "roster" then
 		return has_in_roster(user, who);
 	end
-- 
cgit v1.2.3


From 1c3988b1719e164b38c75eff73aa3df52c531ae7 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Mon, 23 Dec 2019 23:30:45 +0100
Subject: mod_mam: More careful validation of MAM query form

Adapted from mod_muc_mam
---
 plugins/mod_mam/mod_mam.lua | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index e9528d52..018aef77 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -25,6 +25,7 @@ local jid_bare = require "util.jid".bare;
 local jid_split = require "util.jid".split;
 local jid_prepped_split = require "util.jid".prepped_split;
 local dataform = require "util.dataforms".new;
+local get_form_type = require "util.dataforms".get_type;
 local host = module.host;
 
 local rm_load_roster = require "core.rostermanager".load_roster;
@@ -101,7 +102,14 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local qwith, qstart, qend;
 	local form = query:get_child("x", "jabber:x:data");
 	if form then
-		local err;
+		local form_type, err = get_form_type(form);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid dataform: "..err));
+			return true;
+		elseif form_type ~= xmlns_mam then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'"));
+			return true;
+		end
 		form, err = query_form:data(form);
 		if err then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
-- 
cgit v1.2.3


From 5817e5f02d048aa6c17d5430d55f11c4a7a40a96 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 21 Apr 2020 00:53:21 +0200
Subject: mod_mam: Factor out "should we store this" into a function

Meant to improve readability and ease further improvements to this
algorithm.
---
 plugins/mod_mam/mod_mam.lua | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 8229eb4e..27b4796d 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -263,11 +263,28 @@ local function strip_stanza_id(stanza, user)
 	return stanza;
 end
 
+local function should_store(stanza) --> boolean, reason: string
+	local orig_type = stanza.attr.type or "normal";
+	-- We store chat messages or normal messages that have a body
+	if not(orig_type == "chat" or (orig_type == "normal" and stanza:get_child("body")) ) then
+		return false, "type";
+	end
+
+	-- or if hints suggest we shouldn't
+	if not stanza:get_child("store", "urn:xmpp:hints") then -- No hint telling us we should store
+		if stanza:get_child("no-permanent-store", "urn:xmpp:hints")
+			or stanza:get_child("no-store", "urn:xmpp:hints") then -- Hint telling us we should NOT store
+			return false, "hint";
+		end
+	end
+
+	return true, "default";
+end
+
 -- Handle messages
 local function message_handler(event, c2s)
 	local origin, stanza = event.origin, event.stanza;
 	local log = c2s and origin.log or module._log;
-	local orig_type = stanza.attr.type or "normal";
 	local orig_from = stanza.attr.from;
 	local orig_to = stanza.attr.to or orig_from;
 	-- Stanza without 'to' are treated as if it was to their own bare jid
@@ -280,21 +297,12 @@ local function message_handler(event, c2s)
 	-- Filter out <stanza-id> that claim to be from us
 	event.stanza = strip_stanza_id(stanza, store_user);
 
-	-- We store chat messages or normal messages that have a body
-	if not(orig_type == "chat" or (orig_type == "normal" and stanza:get_child("body")) ) then
-		log("debug", "Not archiving stanza: %s (type)", stanza:top_tag());
+	local should, why = should_store(stanza);
+	if not should then
+		log("debug", "Not archiving stanza: %s (%s)", stanza:top_tag(), why);
 		return;
 	end
 
-	-- or if hints suggest we shouldn't
-	if not stanza:get_child("store", "urn:xmpp:hints") then -- No hint telling us we should store
-		if stanza:get_child("no-permanent-store", "urn:xmpp:hints")
-			or stanza:get_child("no-store", "urn:xmpp:hints") then -- Hint telling us we should NOT store
-			log("debug", "Not archiving stanza: %s (hint)", stanza:top_tag());
-			return;
-		end
-	end
-
 	local clone_for_storage;
 	if not strip_tags:empty() then
 		clone_for_storage = st.clone(stanza);
-- 
cgit v1.2.3


From 962d36d401a1a55346a92fe3b30c6b215583fada Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 21 Apr 2020 00:53:23 +0200
Subject: mod_mam: Log 'why' a stanza is archived

Logging of 'why not' is already done. Why not both? Will make more sense
when the rules evolve a bit.
---
 plugins/mod_mam/mod_mam.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 27b4796d..9c00cc99 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -323,7 +323,7 @@ local function message_handler(event, c2s)
 
 	-- Check with the users preferences
 	if shall_store(store_user, with) then
-		log("debug", "Archiving stanza: %s", stanza:top_tag());
+		log("debug", "Archiving stanza: %s (%s)", stanza:top_tag(), why);
 
 		-- And stash it
 		local time = time_now();
-- 
cgit v1.2.3


From e5d6376c58d026003fbcee2dff60a7b244debbdb Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 21 Apr 2020 00:56:56 +0200
Subject: mod_mam: Invert check for type

This is based on code in mod_csi_simple and aiming towards being more
flexible and maintainable than a couple of tests for when not to store.
---
 plugins/mod_mam/mod_mam.lua | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 9c00cc99..028c3b8f 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -264,11 +264,8 @@ local function strip_stanza_id(stanza, user)
 end
 
 local function should_store(stanza) --> boolean, reason: string
-	local orig_type = stanza.attr.type or "normal";
-	-- We store chat messages or normal messages that have a body
-	if not(orig_type == "chat" or (orig_type == "normal" and stanza:get_child("body")) ) then
-		return false, "type";
-	end
+	local st_type = stanza.attr.type or "normal";
+	local st_to_full = (stanza.attr.to or ""):find("/");
 
 	-- or if hints suggest we shouldn't
 	if not stanza:get_child("store", "urn:xmpp:hints") then -- No hint telling us we should store
@@ -277,6 +274,17 @@ local function should_store(stanza) --> boolean, reason: string
 			return false, "hint";
 		end
 	end
+	if st_type == "headline" then
+		-- Headline messages are ephemeral by definition
+		return false, "headline";
+	end
+	if st_type == "groupchat" and st_to_full then
+		-- MUC messages always go to the full JID, usually archived by the MUC
+		return false, "groupchat";
+	end
+	if stanza:get_child("body") then
+		return true, "body";
+	end
 
 	return true, "default";
 end
-- 
cgit v1.2.3


From 0613c5b47f429e34ca3cbe4eec671e2f281321a8 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 21 Apr 2020 01:01:25 +0200
Subject: mod_mam: Rework hints handling

Improved readability and early returns definite yes/no answer.
---
 plugins/mod_mam/mod_mam.lua | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 028c3b8f..57f399ae 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -267,13 +267,6 @@ local function should_store(stanza) --> boolean, reason: string
 	local st_type = stanza.attr.type or "normal";
 	local st_to_full = (stanza.attr.to or ""):find("/");
 
-	-- or if hints suggest we shouldn't
-	if not stanza:get_child("store", "urn:xmpp:hints") then -- No hint telling us we should store
-		if stanza:get_child("no-permanent-store", "urn:xmpp:hints")
-			or stanza:get_child("no-store", "urn:xmpp:hints") then -- Hint telling us we should NOT store
-			return false, "hint";
-		end
-	end
 	if st_type == "headline" then
 		-- Headline messages are ephemeral by definition
 		return false, "headline";
@@ -282,6 +275,12 @@ local function should_store(stanza) --> boolean, reason: string
 		-- MUC messages always go to the full JID, usually archived by the MUC
 		return false, "groupchat";
 	end
+	if stanza:get_child("no-permanent-store", "urn:xmpp:hints") then
+		return false, "hint";
+	end
+	if stanza:get_child("store", "urn:xmpp:hints") then
+		return true, "hint";
+	end
 	if stanza:get_child("body") then
 		return true, "body";
 	end
-- 
cgit v1.2.3


From e84179de9814eded7c4543848d871fd163ac31d2 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 21 Apr 2020 01:17:55 +0200
Subject: mod_mam: Add more positive hints for storage

Mostly just lifted from mod_csi_simple
---
 plugins/mod_mam/mod_mam.lua | 12 ++++++++++++
 1 file changed, 12 insertions(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 57f399ae..385a0ad2 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -284,6 +284,18 @@ local function should_store(stanza) --> boolean, reason: string
 	if stanza:get_child("body") then
 		return true, "body";
 	end
+	if stanza:get_child("subject") then
+		-- XXX Who would send a message with a subject but with a body?
+		return true, "subject";
+	end
+	if stanza:get_child("encryption", "urn:xmpp:eme:0") then
+		-- Since we can't know what an encrypted message contains, we assume it's important
+		return true, "encrypted";
+	end
+	if stanza:get_child("x", "jabber:x:conference")
+	or stanza:find("{http://jabber.org/protocol/muc#user}x/invite") then
+		return true, "invite";
+	end
 
 	return true, "default";
 end
-- 
cgit v1.2.3


From c8db26be662085caef941923286bdcce9f600950 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 21 Apr 2020 01:18:54 +0200
Subject: mod_mam: Store XEP-0184 receipts and requests

Happy now Ge0rG?
---
 plugins/mod_mam/mod_mam.lua | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 385a0ad2..6e522d73 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -292,6 +292,11 @@ local function should_store(stanza) --> boolean, reason: string
 		-- Since we can't know what an encrypted message contains, we assume it's important
 		return true, "encrypted";
 	end
+	if stanza:get_child(nil, "urn:xmpp:receipts") then
+		-- If it's important enough to ask for a receipt then it's important enough to archive
+		-- and the same applies to the receipt
+		return true, "receipt";
+	end
 	if stanza:get_child("x", "jabber:x:conference")
 	or stanza:find("{http://jabber.org/protocol/muc#user}x/invite") then
 		return true, "invite";
-- 
cgit v1.2.3


From 78668f1be3eb2e0c8301995a65fd7aa87e67aefa Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 21 Apr 2020 01:29:58 +0200
Subject: mod_mam: Check sender of error instead of receiver

The intent is to capture errors to stanzas sent by the local user, so
that they can see why a message failed to be delivered even if the error
came after they went offline.
---
 plugins/mod_mam/mod_mam.lua | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 6e522d73..db0e7f8d 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -266,6 +266,9 @@ end
 local function should_store(stanza) --> boolean, reason: string
 	local st_type = stanza.attr.type or "normal";
 	local st_to_full = (stanza.attr.to or ""):find("/");
+	if st_type == "error" then
+		st_to_full = (stanza.attr.from or ""):find("/");
+	end
 
 	if st_type == "headline" then
 		-- Headline messages are ephemeral by definition
-- 
cgit v1.2.3


From 617ac066385fea7bc1841252c87ea786bc84eafc Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 21 Apr 2020 19:41:43 +0200
Subject: mod_mam: Prefer not archiving if no interesting payloads are found

---
 plugins/mod_mam/mod_mam.lua | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index db0e7f8d..69282857 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -305,7 +305,9 @@ local function should_store(stanza) --> boolean, reason: string
 		return true, "invite";
 	end
 
-	return true, "default";
+	 -- The IM-NG thing to do here would be to return `not st_to_full`
+	 -- One day ...
+	return false, "default";
 end
 
 -- Handle messages
-- 
cgit v1.2.3


From b0dbacb69cac71c39088328f740b7e4b59831b81 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 21 Apr 2020 23:06:55 +0200
Subject: mod_mam: Fix typo in comment

If it is with a body then it execution does not get this far
---
 plugins/mod_mam/mod_mam.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 69282857..6d493131 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -288,7 +288,7 @@ local function should_store(stanza) --> boolean, reason: string
 		return true, "body";
 	end
 	if stanza:get_child("subject") then
-		-- XXX Who would send a message with a subject but with a body?
+		-- XXX Who would send a message with a subject but without a body?
 		return true, "subject";
 	end
 	if stanza:get_child("encryption", "urn:xmpp:eme:0") then
-- 
cgit v1.2.3


From 8c1161cf773fa9eb5371e2bffdb054b97a71dad1 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Wed, 22 Apr 2020 18:47:06 +0200
Subject: mod_mam: Respect no-store hint (thanks Ge0rG)

no-store is used in an example in XEP-0313, so obviously this is the
preferred hint
---
 plugins/mod_mam/mod_mam.lua | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 6d493131..7ac2a512 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -278,7 +278,8 @@ local function should_store(stanza) --> boolean, reason: string
 		-- MUC messages always go to the full JID, usually archived by the MUC
 		return false, "groupchat";
 	end
-	if stanza:get_child("no-permanent-store", "urn:xmpp:hints") then
+	if stanza:get_child("no-store", "urn:xmpp:hints")
+	or stanza:get_child("no-permanent-store", "urn:xmpp:hints") then
 		return false, "hint";
 	end
 	if stanza:get_child("store", "urn:xmpp:hints") then
-- 
cgit v1.2.3


From 9bb75c85c478a5da62e697aa6006c570db15eae2 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Wed, 22 Apr 2020 18:48:27 +0200
Subject: mod_mam: Keep chat markers (thanks Ge0rG)

---
 plugins/mod_mam/mod_mam.lua | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 7ac2a512..06a20f4d 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -301,6 +301,9 @@ local function should_store(stanza) --> boolean, reason: string
 		-- and the same applies to the receipt
 		return true, "receipt";
 	end
+	if stanza:get_child(nil, "urn:xmpp:chat-markers:0") then
+		return true, "marker";
+	end
 	if stanza:get_child("x", "jabber:x:conference")
 	or stanza:find("{http://jabber.org/protocol/muc#user}x/invite") then
 		return true, "invite";
-- 
cgit v1.2.3


From 92150bd359689a2b3401ae9dcdc8ba51fbaf8e1f Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Wed, 22 Apr 2020 18:50:30 +0200
Subject: mod_mam: Save delivery failures (thanks Ge0rG)

Makes it possible to learn of delivery failure even if it came bouncing
back while you were offline.
---
 plugins/mod_mam/mod_mam.lua | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 06a20f4d..b74a92ea 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -274,6 +274,9 @@ local function should_store(stanza) --> boolean, reason: string
 		-- Headline messages are ephemeral by definition
 		return false, "headline";
 	end
+	if st_type == "error" then
+		return true, "bounce";
+	end
 	if st_type == "groupchat" and st_to_full then
 		-- MUC messages always go to the full JID, usually archived by the MUC
 		return false, "groupchat";
-- 
cgit v1.2.3


From 966c3c56043bb8903acbbd5c8f9ee36bc23d5388 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Wed, 22 Apr 2020 18:53:50 +0200
Subject: mod_mam: Make note of Experimental (or Deferred) XEPs

Since these XEPs are subject to change we may need come back and double
check these in the future.
---
 plugins/mod_mam/mod_mam.lua | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index b74a92ea..a9d389f5 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -283,6 +283,7 @@ local function should_store(stanza) --> boolean, reason: string
 	end
 	if stanza:get_child("no-store", "urn:xmpp:hints")
 	or stanza:get_child("no-permanent-store", "urn:xmpp:hints") then
+		-- XXX Experimental XEP
 		return false, "hint";
 	end
 	if stanza:get_child("store", "urn:xmpp:hints") then
@@ -297,6 +298,7 @@ local function should_store(stanza) --> boolean, reason: string
 	end
 	if stanza:get_child("encryption", "urn:xmpp:eme:0") then
 		-- Since we can't know what an encrypted message contains, we assume it's important
+		-- XXX Experimental XEP
 		return true, "encrypted";
 	end
 	if stanza:get_child(nil, "urn:xmpp:receipts") then
@@ -305,6 +307,7 @@ local function should_store(stanza) --> boolean, reason: string
 		return true, "receipt";
 	end
 	if stanza:get_child(nil, "urn:xmpp:chat-markers:0") then
+		-- XXX Experimental XEP
 		return true, "marker";
 	end
 	if stanza:get_child("x", "jabber:x:conference")
-- 
cgit v1.2.3


From ba452d69231d6cbf4c2efb4f5e99df40d9a11ede Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 23 Apr 2020 00:55:34 +0200
Subject: mod_mam: Don't store any groupchat messages

The intent was to not store MUC groupchat messages, which are sent from
the MUC to local full JIDs, while allowing for potential future
account based group chat. However, since this function handles messages
in both directions and outgoing MUC messages are sent to the bare room
JID, those were stored.
---
 plugins/mod_mam/mod_mam.lua | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index a9d389f5..65859afd 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -269,6 +269,8 @@ local function should_store(stanza) --> boolean, reason: string
 	if st_type == "error" then
 		st_to_full = (stanza.attr.from or ""):find("/");
 	end
+	-- FIXME pass direction of stanza and use that along with bare/full JID addressing
+	-- for more accurate MUC / type=groupchat check
 
 	if st_type == "headline" then
 		-- Headline messages are ephemeral by definition
@@ -277,7 +279,7 @@ local function should_store(stanza) --> boolean, reason: string
 	if st_type == "error" then
 		return true, "bounce";
 	end
-	if st_type == "groupchat" and st_to_full then
+	if st_type == "groupchat" then
 		-- MUC messages always go to the full JID, usually archived by the MUC
 		return false, "groupchat";
 	end
-- 
cgit v1.2.3


From 80aaa484c38af613e61ad2f1af0ca186933ab3b8 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Thu, 23 Apr 2020 01:05:34 +0200
Subject: mod_mam: Remove unused variables [luacheck]

Logic using full vs bare JID addressing may return in the future.
---
 plugins/mod_mam/mod_mam.lua | 4 ----
 1 file changed, 4 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 65859afd..d61d4883 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -265,10 +265,6 @@ end
 
 local function should_store(stanza) --> boolean, reason: string
 	local st_type = stanza.attr.type or "normal";
-	local st_to_full = (stanza.attr.to or ""):find("/");
-	if st_type == "error" then
-		st_to_full = (stanza.attr.from or ""):find("/");
-	end
 	-- FIXME pass direction of stanza and use that along with bare/full JID addressing
 	-- for more accurate MUC / type=groupchat check
 
-- 
cgit v1.2.3


From e5327bcc31cb602df89d19f9c9c73aa90d4bec37 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Sun, 26 Apr 2020 14:28:00 +0200
Subject: mod_mam: Store only incoming errors

Unclear if clients normally ever send error messages, but there may be
locally generated bounces sent on behalf of local sessions.
---
 plugins/mod_mam/mod_mam.lua | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index d61d4883..72b7639a 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -263,7 +263,7 @@ local function strip_stanza_id(stanza, user)
 	return stanza;
 end
 
-local function should_store(stanza) --> boolean, reason: string
+local function should_store(stanza, c2s) --> boolean, reason: string
 	local st_type = stanza.attr.type or "normal";
 	-- FIXME pass direction of stanza and use that along with bare/full JID addressing
 	-- for more accurate MUC / type=groupchat check
@@ -272,7 +272,8 @@ local function should_store(stanza) --> boolean, reason: string
 		-- Headline messages are ephemeral by definition
 		return false, "headline";
 	end
-	if st_type == "error" then
+	if st_type == "error" and not c2s then
+		-- Store delivery failure notifications so you know if your own messages were not delivered
 		return true, "bounce";
 	end
 	if st_type == "groupchat" then
@@ -334,7 +335,7 @@ local function message_handler(event, c2s)
 	-- Filter out <stanza-id> that claim to be from us
 	event.stanza = strip_stanza_id(stanza, store_user);
 
-	local should, why = should_store(stanza);
+	local should, why = should_store(stanza, c2s);
 	if not should then
 		log("debug", "Not archiving stanza: %s (%s)", stanza:top_tag(), why);
 		return;
-- 
cgit v1.2.3


From d146cc6f582196d9d9a79b6d3c2bf030cc26f7b2 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Tue, 5 May 2020 22:21:39 +0200
Subject: MAM: Remove 1% of contents when reaching limits, fix #1545

With mod\_storage\_internal this counts out to 100 messages out of 10 000,
meaning should not hit the quota limit immediately until that many
messages have been added again.
---
 plugins/mod_mam/mod_mam.lua | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 72b7639a..8695fe65 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -44,6 +44,8 @@ local archive = module:open_store(archive_store, "archive");
 local cleanup_after = module:get_option_string("archive_expires_after", "1w");
 local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
 local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
+local archive_truncate = math.floor(archive_item_limit * 0.99);
+
 if not archive.find then
 	error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n"
 		.."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information");
@@ -379,7 +381,7 @@ local function message_handler(event, c2s)
 			if not ok and (archive.caps and archive.caps.truncate) then
 				module:log("debug", "User '%s' over quota, truncating archive", store_user);
 				local truncated = archive:delete(store_user, {
-					truncate = archive_item_limit - 1;
+					truncate = archive_truncate;
 				});
 				if truncated then
 					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
-- 
cgit v1.2.3


From 1f8daf736d2ebfee0222b2bbccad5ed41eaf16a8 Mon Sep 17 00:00:00 2001
From: Kim Alvefur <zash@zash.se>
Date: Sat, 9 May 2020 00:57:42 +0200
Subject: mod_mam: Archive XEP-0353: Jingle Message Initiation

---
 plugins/mod_mam/mod_mam.lua | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'plugins/mod_mam')

diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index 8695fe65..4fe714eb 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -315,6 +315,10 @@ local function should_store(stanza, c2s) --> boolean, reason: string
 	or stanza:find("{http://jabber.org/protocol/muc#user}x/invite") then
 		return true, "invite";
 	end
+	if stanza:get_child(nil, "urn:xmpp:jingle-message:0") then
+		-- XXX Experimental XEP stuck in Proposed for almost a year at the time of this comment
+		return true, "jingle call";
+	end
 
 	 -- The IM-NG thing to do here would be to return `not st_to_full`
 	 -- One day ...
-- 
cgit v1.2.3