From cf7c286cfbc879121126248d21bef70dfec2e5bc Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 24 Nov 2015 19:00:43 +0100 Subject: util.session: What is a session? --- util/session.lua | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 util/session.lua diff --git a/util/session.lua b/util/session.lua new file mode 100644 index 00000000..735b6165 --- /dev/null +++ b/util/session.lua @@ -0,0 +1,11 @@ + +local function new_session(typ) + local session = { + type = typ .. "_unauthed"; + }; + return session; +end + +return { + new = new_session; +} -- cgit v1.2.3 From 3bf4a8eef76681cf3ef9dd4421a045c69cbfe860 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 24 Nov 2015 19:09:51 +0100 Subject: util.session: What is the identity of a session? --- util/session.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/util/session.lua b/util/session.lua index 735b6165..6993842c 100644 --- a/util/session.lua +++ b/util/session.lua @@ -6,6 +6,13 @@ local function new_session(typ) return session; end +local function set_id(session) + local id = typ .. tostring(session):match("%x+$"):lower(); + session.id = id; + return session; +end + return { new = new_session; + set_id = set_id; } -- cgit v1.2.3 From cfc3a5ad7be12ee67e1d14df04dc03acc73394f3 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 24 Nov 2015 19:04:54 +0100 Subject: util.session: What does the session say? --- util/session.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/util/session.lua b/util/session.lua index 6993842c..7703c265 100644 --- a/util/session.lua +++ b/util/session.lua @@ -1,3 +1,4 @@ +local logger = require "util.logger"; local function new_session(typ) local session = { @@ -12,7 +13,14 @@ local function set_id(session) return session; end +local function set_logger(session) + local log = logger.init(id); + session.log = log; + return session; +end + return { new = new_session; set_id = set_id; + set_logger = set_logger; } -- cgit v1.2.3 From 40f592ecf0d56591183a1b18ba7d7a0ba5fa87cf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 24 Nov 2015 19:26:27 +0100 Subject: util.session: How does a session relate do a connection? --- util/session.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/util/session.lua b/util/session.lua index 7703c265..03ddcf47 100644 --- a/util/session.lua +++ b/util/session.lua @@ -19,8 +19,15 @@ local function set_logger(session) return session; end +local function set_conn(session, conn) + session.conn = conn; + session.ip = conn:ip(); + return session; +end + return { new = new_session; set_id = set_id; set_logger = set_logger; + set_conn = set_conn; } -- cgit v1.2.3 From 273a126b4b19876418f091ad462897576325040b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 24 Nov 2015 19:31:37 +0100 Subject: util.session: How would you even send anything to a session? --- util/session.lua | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/util/session.lua b/util/session.lua index 03ddcf47..fda7fccd 100644 --- a/util/session.lua +++ b/util/session.lua @@ -1,3 +1,4 @@ +local initialize_filters = require "util.filters".initialize; local logger = require "util.logger"; local function new_session(typ) @@ -25,9 +26,40 @@ local function set_conn(session, conn) return session; end +local function set_send(session) + local conn = session.conn; + if not conn then + function session.send(data) + log("debug", "Discarding data sent to unconnected session: %s", tostring(data)); + return false; + end + return session; + end + local filter = initialize_filters(session); + local w = conn.write; + session.send = function (t) + if t.name then + t = filter("stanzas/out", t); + end + if t then + t = filter("bytes/out", tostring(t)); + if t then + local ret, err = w(conn, t); + if not ret then + session.log("debug", "Error writing to connection: %s", tostring(err)); + return false, err; + end + end + end + return true; + end + return session; +end + return { new = new_session; set_id = set_id; set_logger = set_logger; set_conn = set_conn; + set_send = set_send; } -- cgit v1.2.3 From c8454e918047a5de57dcdaeb3ffe94e43bb9bb9b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 25 Nov 2015 20:43:29 +0100 Subject: util.pposix: Expose the uname GNU extension field 'domainname' --- util-src/pposix.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/util-src/pposix.c b/util-src/pposix.c index 02ea0087..1b69852d 100644 --- a/util-src/pposix.c +++ b/util-src/pposix.c @@ -642,6 +642,10 @@ int lc_uname(lua_State* L) { lua_setfield(L, -2, "version"); lua_pushstring(L, uname_info.machine); lua_setfield(L, -2, "machine"); +#ifdef _GNU_SOURCE + lua_pushstring(L, uname_info.domainname); + lua_setfield(L, -2, "domainname"); +#endif return 1; } -- cgit v1.2.3 From dd9e60beba3cdea9b29aff668a4af411f03918b9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 25 Nov 2015 20:49:41 +0100 Subject: util.cache: Make sure cache size is specified as an integer --- util/cache.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/cache.lua b/util/cache.lua index 9453697c..32670751 100644 --- a/util/cache.lua +++ b/util/cache.lua @@ -2,6 +2,8 @@ local cache_methods = {}; local cache_mt = { __index = cache_methods }; local function new(size) + size = assert(tonumber(size), "cache size must be a number"); + size = math.floor(size); assert(size > 0, "cache size must be greater than zero"); local data = {}; return setmetatable({ data = data, count = 0, size = size, head = nil, tail = nil }, cache_mt); -- cgit v1.2.3 From 13d34d918fa517490a16ad08ea1485eb68f21cca Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 25 Nov 2015 21:06:01 +0100 Subject: mod_blocklist: Use util.cache to manage how many users blocklists are kept in memory --- plugins/mod_blocklist.lua | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/plugins/mod_blocklist.lua b/plugins/mod_blocklist.lua index 8589c1e9..65fc70b7 100644 --- a/plugins/mod_blocklist.lua +++ b/plugins/mod_blocklist.lua @@ -19,12 +19,13 @@ local jid_split = require"util.jid".split; local storage = module:open_store(); local sessions = prosody.hosts[module.host].sessions; --- Cache of blocklists used since module was loaded -local cache = {}; -if module:get_option_boolean("blocklist_weak_cache") then - -- Lower memory usage, more IO and latency - setmetatable(cache, { __mode = "v" }); -end +-- Cache of blocklists by username may randomly expire at any time +local cache = setmetatable({}, { __mode = "v" }); + +-- Second level of caching, keeps a fixed number of items, +-- also anchors items in the above cache +local cache_size = module:get_option_number("blocklist_cache_size", 64); +local cache2 = require"util.cache".new(cache_size); local null_blocklist = {}; @@ -36,6 +37,7 @@ local function set_blocklist(username, blocklist) return ok, err; end -- Successful save, update the cache + cache2:set(username, blocklist); cache[username] = blocklist; return true; end @@ -71,6 +73,9 @@ end local function get_blocklist(username) local blocklist = cache[username]; + if not blocklist then + blocklist = cache2:get(username); + end if not blocklist then if not user_exists(username, module.host) then return null_blocklist; @@ -79,8 +84,9 @@ local function get_blocklist(username) if not blocklist then blocklist = migrate_privacy_list(username); end - cache[username] = blocklist; + cache2:set(username, blocklist); end + cache[username] = blocklist; return blocklist; end -- cgit v1.2.3 From c81eea619aabb5d3322bb9baf89f48764f92802b Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 26 Nov 2015 00:07:48 +0000 Subject: util.cache: Small update to prefix private fields with an underscore, add a :count() method (same as util.queue) and add an optional on_evict callback --- util/cache.lua | 75 +++++++++++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/util/cache.lua b/util/cache.lua index 32670751..72b74351 100644 --- a/util/cache.lua +++ b/util/cache.lua @@ -1,13 +1,3 @@ -local cache_methods = {}; -local cache_mt = { __index = cache_methods }; - -local function new(size) - size = assert(tonumber(size), "cache size must be a number"); - size = math.floor(size); - assert(size > 0, "cache size must be greater than zero"); - local data = {}; - return setmetatable({ data = data, count = 0, size = size, head = nil, tail = nil }, cache_mt); -end local function _remove(list, m) if m.prev then @@ -16,29 +6,32 @@ local function _remove(list, m) if m.next then m.next.prev = m.prev; end - if list.tail == m then - list.tail = m.prev; + if list._tail == m then + list._tail = m.prev; end - if list.head == m then - list.head = m.next; + if list._head == m then + list._head = m.next; end - list.count = list.count - 1; + list._count = list._count - 1; end local function _insert(list, m) - if list.head then - list.head.prev = m; + if list._head then + list._head.prev = m; end - m.prev, m.next = nil, list.head; - list.head = m; - if not list.tail then - list.tail = m; + m.prev, m.next = nil, list._head; + list._head = m; + if not list._tail then + list._tail = m; end - list.count = list.count + 1; + list._count = list._count + 1; end +local cache_methods = {}; +local cache_mt = { __index = cache_methods }; + function cache_methods:set(k, v) - local m = self.data[k]; + local m = self._data[k]; if m then -- Key already exists if v ~= nil then @@ -49,27 +42,33 @@ function cache_methods:set(k, v) else -- Remove from list _remove(self, m); - self.data[k] = nil; + self._data[k] = nil; end - return; + return true; end -- New key if v == nil then - return; + return true; end -- Check whether we need to remove oldest k/v - if self.count == self.size then - self.data[self.tail.key] = nil; - _remove(self, self.tail); + if self._count == self.size then + local tail = self._tail; + local on_evict = self._on_evict; + if on_evict then + on_evict(tail.key, tail.value); + end + _remove(self, tail); + self._data[tail.key] = nil; end m = { key = k, value = v, prev = nil, next = nil }; - self.data[k] = m; + self._data[k] = m; _insert(self, m); + return true; end function cache_methods:get(k) - local m = self.data[k]; + local m = self._data[k]; if m then return m.value; end @@ -77,7 +76,7 @@ function cache_methods:get(k) end function cache_methods:items() - local m = self.head; + local m = self._head; return function () if not m then return; @@ -88,6 +87,18 @@ function cache_methods:items() end end +function cache_methods:count() + return self._count; +end + +local function new(size, on_evict) + size = assert(tonumber(size), "cache size must be a number"); + size = math.floor(size); + assert(size > 0, "cache size must be greater than zero"); + local data = {}; + return setmetatable({ _data = data, _count = 0, size = size, _head = nil, _tail = nil, _on_evict = on_evict }, cache_mt); +end + return { new = new; } -- cgit v1.2.3 From 451180d8a42fd3869be9b7002142467147da3203 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 26 Nov 2015 00:12:51 +0000 Subject: tests: Add tests for util.cache --- tests/test.lua | 1 + tests/test_util_cache.lua | 174 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 tests/test_util_cache.lua diff --git a/tests/test.lua b/tests/test.lua index 36c2fd6e..f7fdee91 100644 --- a/tests/test.lua +++ b/tests/test.lua @@ -20,6 +20,7 @@ function run_all_tests() dotest "util.ip" dotest "util.stanza" dotest "util.sasl.scram" + dotest "util.cache" dosingletest("test_sasl.lua", "latin1toutf8"); dosingletest("test_utf8.lua", "valid"); diff --git a/tests/test_util_cache.lua b/tests/test_util_cache.lua new file mode 100644 index 00000000..a174eea7 --- /dev/null +++ b/tests/test_util_cache.lua @@ -0,0 +1,174 @@ + +function new(new) + local c = new(5); + + assert_equal(c:count(), 0); + + c:set("one", 1) + assert_equal(c:count(), 1); + c:set("two", 2) + c:set("three", 3) + c:set("four", 4) + c:set("five", 5); + assert_equal(c:count(), 5); + + c:set("foo", nil); + assert_equal(c:count(), 5); + + assert_equal(c:get("one"), 1); + assert_equal(c:get("two"), 2); + assert_equal(c:get("three"), 3); + assert_equal(c:get("four"), 4); + assert_equal(c:get("five"), 5); + + assert_equal(c:get("foo"), nil); + assert_equal(c:get("bar"), nil); + + c:set("six", 6); + assert_equal(c:count(), 5); + + assert_equal(c:get("one"), nil); + assert_equal(c:get("two"), 2); + assert_equal(c:get("three"), 3); + assert_equal(c:get("four"), 4); + assert_equal(c:get("five"), 5); + assert_equal(c:get("six"), 6); + + c:set("three", nil); + assert_equal(c:count(), 4); + + assert_equal(c:get("one"), nil); + assert_equal(c:get("two"), 2); + assert_equal(c:get("three"), nil); + assert_equal(c:get("four"), 4); + assert_equal(c:get("five"), 5); + assert_equal(c:get("six"), 6); + + c:set("seven", 7); + assert_equal(c:count(), 5); + + assert_equal(c:get("one"), nil); + assert_equal(c:get("two"), 2); + assert_equal(c:get("three"), nil); + assert_equal(c:get("four"), 4); + assert_equal(c:get("five"), 5); + assert_equal(c:get("six"), 6); + assert_equal(c:get("seven"), 7); + + c:set("eight", 8); + assert_equal(c:count(), 5); + + assert_equal(c:get("one"), nil); + assert_equal(c:get("two"), nil); + assert_equal(c:get("three"), nil); + assert_equal(c:get("four"), 4); + assert_equal(c:get("five"), 5); + assert_equal(c:get("six"), 6); + assert_equal(c:get("seven"), 7); + assert_equal(c:get("eight"), 8); + + c:set("four", 4); + assert_equal(c:count(), 5); + + assert_equal(c:get("one"), nil); + assert_equal(c:get("two"), nil); + assert_equal(c:get("three"), nil); + assert_equal(c:get("four"), 4); + assert_equal(c:get("five"), 5); + assert_equal(c:get("six"), 6); + assert_equal(c:get("seven"), 7); + assert_equal(c:get("eight"), 8); + + c:set("nine", 9); + assert_equal(c:count(), 5); + + assert_equal(c:get("one"), nil); + assert_equal(c:get("two"), nil); + assert_equal(c:get("three"), nil); + assert_equal(c:get("four"), 4); + assert_equal(c:get("five"), nil); + assert_equal(c:get("six"), 6); + assert_equal(c:get("seven"), 7); + assert_equal(c:get("eight"), 8); + assert_equal(c:get("nine"), 9); + + local keys = { "nine", "four", "eight", "seven", "six" }; + local values = { 9, 4, 8, 7, 6 }; + local i = 0; + for k, v in c:items() do + i = i + 1; + assert_equal(k, keys[i]); + assert_equal(v, values[i]); + end + assert_equal(i, 5); + + c:set("four", "2+2"); + assert_equal(c:count(), 5); + + assert_equal(c:get("one"), nil); + assert_equal(c:get("two"), nil); + assert_equal(c:get("three"), nil); + assert_equal(c:get("four"), "2+2"); + assert_equal(c:get("five"), nil); + assert_equal(c:get("six"), 6); + assert_equal(c:get("seven"), 7); + assert_equal(c:get("eight"), 8); + assert_equal(c:get("nine"), 9); + + local keys = { "four", "nine", "eight", "seven", "six" }; + local values = { "2+2", 9, 8, 7, 6 }; + local i = 0; + for k, v in c:items() do + i = i + 1; + assert_equal(k, keys[i]); + assert_equal(v, values[i]); + end + assert_equal(i, 5); + + c:set("foo", nil); + assert_equal(c:count(), 5); + + assert_equal(c:get("one"), nil); + assert_equal(c:get("two"), nil); + assert_equal(c:get("three"), nil); + assert_equal(c:get("four"), "2+2"); + assert_equal(c:get("five"), nil); + assert_equal(c:get("six"), 6); + assert_equal(c:get("seven"), 7); + assert_equal(c:get("eight"), 8); + assert_equal(c:get("nine"), 9); + + local keys = { "four", "nine", "eight", "seven", "six" }; + local values = { "2+2", 9, 8, 7, 6 }; + local i = 0; + for k, v in c:items() do + i = i + 1; + assert_equal(k, keys[i]); + assert_equal(v, values[i]); + end + assert_equal(i, 5); + + c:set("four", nil); + + assert_equal(c:get("one"), nil); + assert_equal(c:get("two"), nil); + assert_equal(c:get("three"), nil); + assert_equal(c:get("four"), nil); + assert_equal(c:get("five"), nil); + assert_equal(c:get("six"), 6); + assert_equal(c:get("seven"), 7); + assert_equal(c:get("eight"), 8); + assert_equal(c:get("nine"), 9); + + local keys = { "nine", "eight", "seven", "six" }; + local values = { 9, 8, 7, 6 }; + local i = 0; + for k, v in c:items() do + i = i + 1; + assert_equal(k, keys[i]); + assert_equal(v, values[i]); + end + assert_equal(i, 4); + + +end -- cgit v1.2.3