aboutsummaryrefslogtreecommitdiffstats
path: root/plugins/mod_storage_internal.lua
diff options
context:
space:
mode:
Diffstat (limited to 'plugins/mod_storage_internal.lua')
-rw-r--r--plugins/mod_storage_internal.lua238
1 files changed, 163 insertions, 75 deletions
diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua
index fa87e495..a43dd272 100644
--- a/plugins/mod_storage_internal.lua
+++ b/plugins/mod_storage_internal.lua
@@ -1,17 +1,20 @@
-local cache = require "util.cache";
-local datamanager = require "core.storagemanager".olddm;
-local array = require "util.array";
-local datetime = require "util.datetime";
-local st = require "util.stanza";
-local now = require "util.time".now;
-local id = require "util.id".medium;
-local jid_join = require "util.jid".join;
-local set = require "util.set";
+local cache = require "prosody.util.cache";
+local datamanager = require "prosody.core.storagemanager".olddm;
+local array = require "prosody.util.array";
+local datetime = require "prosody.util.datetime";
+local st = require "prosody.util.stanza";
+local now = require "prosody.util.time".now;
+local id = require "prosody.util.id".medium;
+local jid_join = require "prosody.util.jid".join;
+local set = require "prosody.util.set";
+local it = require "prosody.util.iterators";
local host = module.host;
-local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000);
-local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
+local archive_item_limit = module:get_option_integer("storage_archive_item_limit", 10000, 0);
+local archive_item_count_cache = cache.new(module:get_option_integer("storage_archive_item_limit_cache_size", 1000, 1));
+
+local use_shift = module:get_option_boolean("storage_archive_experimental_fast_delete", false);
local driver = {};
@@ -121,100 +124,144 @@ function archive:append(username, key, value, when, with)
return key;
end
+local function binary_search(haystack, test, min, max)
+ if min == nil then
+ min = 1;
+ end
+ if max == nil then
+ max = #haystack;
+ end
+
+ local floor = math.floor;
+ while min < max do
+ local mid = floor((max + min) / 2);
+
+ local result = test(haystack[mid]);
+ if result < 0 then
+ max = mid;
+ elseif result > 0 then
+ min = mid + 1;
+ else
+ return mid, haystack[mid];
+ end
+ end
+
+ return min, nil;
+end
+
function archive:find(username, query)
- local items, err = datamanager.list_load(username, host, self.store);
- if not items then
+ local list, err = datamanager.list_open(username, host, self.store);
+ if not list then
if err then
- return items, err;
+ return list, err;
elseif query then
if query.before or query.after then
return nil, "item-not-found";
end
if query.total then
- return function () end, 0;
+ return function()
+ end, 0;
end
end
- return function () end;
+ return function()
+ end;
+ end
+
+ local i = 0;
+ local iter = function()
+ i = i + 1;
+ return list[i]
end
- local count = nil;
- local i, last_key = 0;
+
if query then
- items = array(items);
+ if query.reverse then
+ i = #list + 1
+ iter = function()
+ i = i - 1
+ return list[i]
+ end
+ query.before, query.after = query.after, query.before;
+ end
if query.key then
- items:filter(function (item)
+ iter = it.filter(function(item)
return item.key == query.key;
- end);
+ end, iter);
end
if query.ids then
local ids = set.new(query.ids);
- items:filter(function (item)
+ iter = it.filter(function(item)
return ids:contains(item.key);
- end);
+ end, iter);
end
if query.with then
- items:filter(function (item)
+ iter = it.filter(function(item)
return item.with == query.with;
- end);
+ end, iter);
end
if query.start then
- items:filter(function (item)
- local when = item.when or datetime.parse(item.attr.stamp);
- return when >= query.start;
- end);
+ if not query.reverse then
+ local wi = binary_search(list, function(item)
+ local when = item.when or datetime.parse(item.attr.stamp);
+ return query.start - when;
+ end);
+ i = wi - 1;
+ else
+ iter = it.filter(function(item)
+ local when = item.when or datetime.parse(item.attr.stamp);
+ return when >= query.start;
+ end, iter);
+ end
end
if query["end"] then
- items:filter(function (item)
- local when = item.when or datetime.parse(item.attr.stamp);
- return when <= query["end"];
- end);
- end
- if query.total then
- count = #items;
- end
- if query.reverse then
- items:reverse();
- if query.before then
- local found = false;
- for j = 1, #items do
- if (items[j].key or tostring(j)) == query.before then
- found = true;
- i = j;
- break;
- end
- end
- if not found then
- return nil, "item-not-found";
+ if query.reverse then
+ local wi = binary_search(list, function(item)
+ local when = item.when or datetime.parse(item.attr.stamp);
+ return query["end"] - when;
+ end);
+ if wi then
+ i = wi + 1;
end
+ else
+ iter = it.filter(function(item)
+ local when = item.when or datetime.parse(item.attr.stamp);
+ return when <= query["end"];
+ end, iter);
end
- last_key = query.after;
- elseif query.after then
+ end
+ if query.after then
local found = false;
- for j = 1, #items do
- if (items[j].key or tostring(j)) == query.after then
- found = true;
- i = j;
- break;
+ iter = it.filter(function(item)
+ local found_after = found;
+ if item.key == query.after then
+ found = true
end
- end
- if not found then
- return nil, "item-not-found";
- end
- last_key = query.before;
- elseif query.before then
- last_key = query.before;
+ return found_after;
+ end, iter);
end
- if query.limit and #items - i > query.limit then
- items[i+query.limit+1] = nil;
+ if query.before then
+ local found = false;
+ iter = it.filter(function(item)
+ if item.key == query.before then
+ found = true
+ end
+ return not found;
+ end, iter);
+ end
+ if query.limit then
+ iter = it.head(query.limit, iter);
end
end
- return function ()
- i = i + 1;
- local item = items[i];
- if not item or (last_key and item.key == last_key) then
- return;
+
+ return function()
+ local item = iter();
+ if item == nil then
+ if list.close then
+ list:close();
+ end
+ return
end
- local key = item.key or tostring(i);
- local when = item.when or datetime.parse(item.attr.stamp);
+ local key = item.key;
+ local when = item.when or item.attr and datetime.parse(item.attr.stamp);
local with = item.with;
item.key, item.when, item.with = nil, nil, nil;
item.attr.stamp = nil;
@@ -222,7 +269,7 @@ function archive:find(username, query)
item.attr.stamp_legacy = nil;
item = st.deserialize(item);
return key, item, when, with;
- end, count;
+ end
end
function archive:get(username, wanted_key)
@@ -297,12 +344,53 @@ function archive:users()
return datamanager.users(host, self.store, "list");
end
+function archive:trim(username, to_when)
+ local cache_key = jid_join(username, host, self.store);
+ local list, err = datamanager.list_open(username, host, self.store);
+ if not list then
+ if err == nil then
+ module:log("debug", "store already empty, can't trim");
+ return 0;
+ end
+ return list, err;
+ end
+
+ -- shortcut: check if the last item should be trimmed, if so, drop the whole archive
+ local last = list[#list].when or datetime.parse(list[#list].attr.stamp);
+ if last <= to_when then
+ if list.close then
+ list:close()
+ end
+ return datamanager.list_store(username, host, self.store, nil);
+ end
+
+ -- luacheck: ignore 211/exact
+ local i, exact = binary_search(list, function(item)
+ local when = item.when or datetime.parse(item.attr.stamp);
+ return to_when - when;
+ end);
+ if list.close then
+ list:close()
+ end
+ -- TODO if exact then ... off by one?
+ if i == 1 then return 0; end
+ local ok, err = datamanager.list_shift(username, host, self.store, i);
+ if not ok then return ok, err; end
+ archive_item_count_cache:set(cache_key, nil); -- TODO calculate how many items are left
+ return i-1;
+end
+
function archive:delete(username, query)
local cache_key = jid_join(username, host, self.store);
if not query or next(query) == nil then
- archive_item_count_cache:set(cache_key, nil);
+ archive_item_count_cache:set(cache_key, nil); -- nil because we don't check if the following succeeds
return datamanager.list_store(username, host, self.store, nil);
end
+
+ if use_shift and next(query) == "end" and next(query, "end") == nil then
+ return self:trim(username, query["end"]);
+ end
+
local items, err = datamanager.list_load(username, host, self.store);
if not items then
if err then
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-26 17:45:03 +0000