From d7761bd914bd38e43de12c248196bc81307c71c5 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Fri, 20 Oct 2017 12:53:53 +0200 Subject: mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733) --- plugins/mod_storage_internal.lua | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index 42b451bd..d812c0e9 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -1,3 +1,4 @@ +local cache = require "util.cache"; local datamanager = require "core.storagemanager".olddm; local array = require "util.array"; local datetime = require "util.datetime"; @@ -7,6 +8,9 @@ local id = require "util.id".medium; local host = module.host; +local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000); +local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000)); + local driver = {}; function driver:open(store, typ) @@ -54,28 +58,56 @@ function archive:append(username, key, value, when, with) value.attr.stamp = datetime.datetime(when); value.attr.stamp_legacy = datetime.legacy(when); + local item_count = archive_item_count_cache:get(username); + if key then local items, err = datamanager.list_load(username, host, self.store); if not items and err then return items, err; end + + -- Check the quota + item_count = items and #items or 0; + archive_item_count_cache:set(username, item_count); + if item_count >= archive_item_limit then + module:log("debug", "%s reached or over quota, not adding to store", username); + return nil, "quota-limit"; + end + if items then + -- Filter out any item with the same key as the one being added items = array(items); items:filter(function (item) return item.key ~= key; end); + value.key = key; items:push(value); local ok, err = datamanager.list_store(username, host, self.store, items); if not ok then return ok, err; end + archive_item_count_cache:set(username, #items); return key; end else + if not item_count then -- Item count not cached? + -- We need to load the list to get the number of items currently stored + local items, err = datamanager.list_load(username, host, self.store); + if not items and err then return items, err; end + item_count = items and #items or 0; + archive_item_count_cache:set(username, item_count); + end + if item_count >= archive_item_limit then + module:log("debug", "%s reached or over quota, not adding to store", username); + return nil, "quota-limit"; + end key = id(); end + module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit); + value.key = key; local ok, err = datamanager.list_append(username, host, self.store, value); if not ok then return ok, err; end + archive_item_count_cache:set(username, item_count+1); return key; end @@ -158,6 +190,7 @@ end function archive:delete(username, query) if not query or next(query) == nil then + archive_item_count_cache:set(username, nil); return datamanager.list_store(username, host, self.store, nil); end local items, err = datamanager.list_load(username, host, self.store); @@ -165,6 +198,7 @@ function archive:delete(username, query) if err then return items, err; end + archive_item_count_cache:set(username, 0); -- Store is empty return 0; end @@ -214,6 +248,7 @@ function archive:delete(username, query) end local ok, err = datamanager.list_store(username, host, self.store, items); if not ok then return ok, err; end + archive_item_count_cache:set(username, #items); return count; end -- cgit v1.2.3 From 0028ea46e2aed1e0522da59b3d31912afea2c54a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 18:01:24 +0100 Subject: mod_storage_internal,_sql: Expose archive capabilities feature set This was planned to be added long ago but was forgotten. --- plugins/mod_storage_internal.lua | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index d812c0e9..c21fe0dc 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -47,6 +47,12 @@ end local archive = {}; driver.archive = { __index = archive }; +archive.caps = { + total = true; + quota = archive_item_limit; + truncate = true; +}; + function archive:append(username, key, value, when, with) when = when or now(); if not st.is_stanza(value) then -- cgit v1.2.3 From 9eb4885f38891261621cd18aa206883851acbaab Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 18:02:27 +0100 Subject: mod_storage_internal,_sql: Key item count cache on both username and store --- plugins/mod_storage_internal.lua | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index c21fe0dc..52ca4da8 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -5,6 +5,7 @@ local datetime = require "util.datetime"; local st = require "util.stanza"; local now = require "util.time".now; local id = require "util.id".medium; +local jid_join = require "util.jid".join; local host = module.host; @@ -64,7 +65,8 @@ function archive:append(username, key, value, when, with) value.attr.stamp = datetime.datetime(when); value.attr.stamp_legacy = datetime.legacy(when); - local item_count = archive_item_count_cache:get(username); + local cache_key = jid_join(username, host, self.store); + local item_count = archive_item_count_cache:get(cache_key); if key then local items, err = datamanager.list_load(username, host, self.store); @@ -72,7 +74,7 @@ function archive:append(username, key, value, when, with) -- Check the quota item_count = items and #items or 0; - archive_item_count_cache:set(username, item_count); + archive_item_count_cache:set(cache_key, item_count); if item_count >= archive_item_limit then module:log("debug", "%s reached or over quota, not adding to store", username); return nil, "quota-limit"; @@ -89,7 +91,7 @@ function archive:append(username, key, value, when, with) items:push(value); local ok, err = datamanager.list_store(username, host, self.store, items); if not ok then return ok, err; end - archive_item_count_cache:set(username, #items); + archive_item_count_cache:set(cache_key, #items); return key; end else @@ -98,7 +100,7 @@ function archive:append(username, key, value, when, with) local items, err = datamanager.list_load(username, host, self.store); if not items and err then return items, err; end item_count = items and #items or 0; - archive_item_count_cache:set(username, item_count); + archive_item_count_cache:set(cache_key, item_count); end if item_count >= archive_item_limit then module:log("debug", "%s reached or over quota, not adding to store", username); @@ -113,7 +115,7 @@ function archive:append(username, key, value, when, with) local ok, err = datamanager.list_append(username, host, self.store, value); if not ok then return ok, err; end - archive_item_count_cache:set(username, item_count+1); + archive_item_count_cache:set(cache_key, item_count+1); return key; end @@ -195,8 +197,9 @@ function archive:dates(username) end function archive:delete(username, query) + local cache_key = jid_join(username, host, self.store); if not query or next(query) == nil then - archive_item_count_cache:set(username, nil); + archive_item_count_cache:set(cache_key, nil); return datamanager.list_store(username, host, self.store, nil); end local items, err = datamanager.list_load(username, host, self.store); @@ -204,7 +207,7 @@ function archive:delete(username, query) if err then return items, err; end - archive_item_count_cache:set(username, 0); + archive_item_count_cache:set(cache_key, 0); -- Store is empty return 0; end @@ -254,7 +257,7 @@ function archive:delete(username, query) end local ok, err = datamanager.list_store(username, host, self.store, items); if not ok then return ok, err; end - archive_item_count_cache:set(username, #items); + archive_item_count_cache:set(cache_key, #items); return count; end -- cgit v1.2.3 From c4b5bfdc5f2da4cdbe085a5a95c6418f802ec56e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:04:34 +0100 Subject: mod_storage_internal: Increase default quota to 10 000 Performance doesn't seem great but 10k should be far enough from limits inherited by the Lua parser. 1000 messages seemed pretty close to what an active user might produce in one week. --- plugins/mod_storage_internal.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index 52ca4da8..cb88f10f 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -9,7 +9,7 @@ local jid_join = require "util.jid".join; local host = module.host; -local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000); +local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000); local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000)); local driver = {}; -- cgit v1.2.3 From 5bb703f07f52906b1280daaac164a3886f09a373 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:24:48 +0100 Subject: mod_storage_internal: Include store name when reporting quota status --- plugins/mod_storage_internal.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index cb88f10f..c87d01be 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -109,7 +109,7 @@ function archive:append(username, key, value, when, with) key = id(); end - module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit); + module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store); value.key = key; -- cgit v1.2.3 From 582fa3f46f105fe55447d607dceb43b5ac61d440 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 25 Feb 2019 15:48:28 +0100 Subject: mod_storage_internal: Implement a summary API returning message counts per contact --- plugins/mod_storage_internal.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index c87d01be..aa5c3c8a 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -196,6 +196,16 @@ function archive:dates(username) return array(items):pluck("when"):map(datetime.date):unique(); end +function archive:summary(username, query) + local iter, err = self:find(username, query) + if not iter then return iter, err; end + local summary = {}; + for _, _, _, with in iter do + summary[with] = (summary[with] or 0) + 1; + end + return summary; +end + function archive:delete(username, query) local cache_key = jid_join(username, host, self.store); if not query or next(query) == nil then -- cgit v1.2.3 From 637b52a13e2c9e655a2595a4d8e274280e2fd612 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 08:12:16 +0200 Subject: mod_storage_internal,memory: Only return total count if requested --- plugins/mod_storage_internal.lua | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index aa5c3c8a..f921747e 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -124,11 +124,14 @@ function archive:find(username, query) if not items then if err then return items, err; - else - return function () end, 0; + elseif query then + if query.total then + return function () end, 0; + end end + return function () end; end - local count = #items; + local count = nil; local i = 0; if query then items = array(items); @@ -152,11 +155,13 @@ function archive:find(username, query) return item.when <= query["end"]; end); end - count = #items; + if query.total then + count = #items; + end if query.reverse then items:reverse(); if query.before then - for j = 1, count do + for j = 1, #items do if (items[j].key or tostring(j)) == query.before then i = j; break; @@ -164,7 +169,7 @@ function archive:find(username, query) end end elseif query.after then - for j = 1, count do + for j = 1, #items do if (items[j].key or tostring(j)) == query.after then i = j; break; -- cgit v1.2.3 From ef95de88710184351f130090685c5bc7ae7fbcaf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 16:07:16 +0200 Subject: mod_storage_internal: Add support for iterating over users in archive stores May help with writing a better migrator --- plugins/mod_storage_internal.lua | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index f921747e..96780b33 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -211,6 +211,10 @@ function archive:summary(username, query) return summary; end +function archive:users() + return datamanager.users(host, self.store, "list"); +end + function archive:delete(username, query) local cache_key = jid_join(username, host, self.store); if not query or next(query) == nil then -- cgit v1.2.3 From b772308c93a1c78995a073964ebf963f759ef81c Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 5 Mar 2019 00:12:30 +0100 Subject: mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325) --- plugins/mod_storage_internal.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index 96780b33..fdce3c98 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -161,20 +161,30 @@ function archive:find(username, query) if query.reverse then items:reverse(); if query.before then + local found = false; for j = 1, #items do if (items[j].key or tostring(j)) == query.before then + found = true; i = j; break; end end + if not found then + return nil, "item-not-found"; + end end elseif query.after then + local found = false; for j = 1, #items do if (items[j].key or tostring(j)) == query.after then + found = true; i = j; break; end end + if not found then + return nil, "item-not-found"; + end end if query.limit and #items - i > query.limit then items[i+query.limit+1] = nil; -- cgit v1.2.3 From 99f6c695075dc4a93deb5aea4e8acaf635d48fdb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 07:24:12 +0200 Subject: mod_storage_internal: Return appropriate error even with empty archive --- plugins/mod_storage_internal.lua | 3 +++ 1 file changed, 3 insertions(+) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index fdce3c98..2556224d 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -125,6 +125,9 @@ function archive:find(username, query) if err then return items, err; elseif query then + if query.before or query.after then + return nil, "item-not-found"; + end if query.total then return function () end, 0; end -- cgit v1.2.3 From 39cb87a158a017441e129ed8514f83bfbfdae64f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 23 Aug 2019 01:04:00 +0200 Subject: mod_storage_*: Tweak :summary API to allow future expansion with more fields Eg might want to include last message, timestamp, chat state or other info. --- plugins/mod_storage_internal.lua | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index 2556224d..28caa071 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -217,11 +217,13 @@ end function archive:summary(username, query) local iter, err = self:find(username, query) if not iter then return iter, err; end - local summary = {}; + local counts = {}; for _, _, _, with in iter do - summary[with] = (summary[with] or 0) + 1; + counts[with] = (counts[with] or 0) + 1; end - return summary; + return { + counts = counts; + }; end function archive:users() -- cgit v1.2.3 From d3c559bcc89ef91ebecde3cc3b4520873a98ea4d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 23 Aug 2019 01:10:27 +0200 Subject: mod_storage_*: Include timestamp of latest message in :summary API Clients may want to show a list of conversations ordered by how timestamp of most recent message. The counts allow a badge with unread message counter. --- plugins/mod_storage_internal.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index 28caa071..1ee860d6 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -218,11 +218,14 @@ function archive:summary(username, query) local iter, err = self:find(username, query) if not iter then return iter, err; end local counts = {}; - for _, _, _, with in iter do + local latest = {}; + for _, _, when, with in iter do counts[with] = (counts[with] or 0) + 1; + latest[with] = when; end return { counts = counts; + latest = latest; }; end -- cgit v1.2.3 From 9d5e31fd481a38bc7e346697cdacb8ef5cbd1ae5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 23 Aug 2019 01:15:44 +0200 Subject: mod_storage_*: Also include timestmap of first message in :summary API For completeness along with most recent timestamp. Might be nice to be able to order by oldest unread message. --- plugins/mod_storage_internal.lua | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index 1ee860d6..b3702fe1 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -218,13 +218,18 @@ function archive:summary(username, query) local iter, err = self:find(username, query) if not iter then return iter, err; end local counts = {}; + local earliest = {}; local latest = {}; for _, _, when, with in iter do counts[with] = (counts[with] or 0) + 1; + if earliest[with] == nil then + earliest[with] = when; + end latest[with] = when; end return { counts = counts; + earliest = earliest; latest = latest; }; end -- cgit v1.2.3 From d45b4c026faac9849324a8db0747e2729f19362c Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 23 Aug 2019 01:28:53 +0200 Subject: mod_storage_internal: Include last text message A protocol built on this API now allows showing a list of unread conversations with a counter, ordered by either oldest or newest message, along with the text body itself. --- plugins/mod_storage_internal.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'plugins/mod_storage_internal.lua') diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index b3702fe1..c8b902cf 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -220,17 +220,20 @@ function archive:summary(username, query) local counts = {}; local earliest = {}; local latest = {}; - for _, _, when, with in iter do + local body = {}; + for _, stanza, when, with in iter do counts[with] = (counts[with] or 0) + 1; if earliest[with] == nil then earliest[with] = when; end latest[with] = when; + body[with] = stanza:get_child_text("body") or body[with]; end return { counts = counts; earliest = earliest; latest = latest; + body = body; }; end -- cgit v1.2.3