From 3391b3c8909285808e4c0760a991049b384ecf4a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 28 Oct 2018 16:49:28 +0100 Subject: net.server_select: Deprecate connection:lock_read() method --- net/server_select.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/server_select.lua b/net/server_select.lua index bc86742c..1c016633 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -456,8 +456,8 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport maxreadlen = readlen or maxreadlen return bufferlen, maxreadlen, maxsendlen end - --TODO: Deprecate handler.lock_read = function (self, switch) + out_error( "server.lua, lock_read() is deprecated, use pause() and resume()" ) if switch == true then local tmp = _readlistlen _readlistlen = removesocket( _readlist, socket, _readlistlen ) -- cgit v1.2.3 From ef4bb01f0da61e8512c0c6f9dffb2c1822f42c05 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 28 Oct 2018 16:51:58 +0100 Subject: net.server_event: Deprecate :lock_read here too --- net/server_event.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/net/server_event.lua b/net/server_event.lua index 11bd6a29..ca80c3f2 100644 --- a/net/server_event.lua +++ b/net/server_event.lua @@ -253,6 +253,7 @@ end --TODO: Deprecate function interface_mt:lock_read(switch) + log("warn", ":lock_read is deprecated, use :pasue() and :resume()"); if switch then return self:pause(); else -- cgit v1.2.3 From b593a8c54f2c37aefae0503b130242589e42dc79 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 28 Oct 2018 16:53:10 +0100 Subject: net.server_select: Move code from :lock_read into :pause and :resume --- net/server_select.lua | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/net/server_select.lua b/net/server_select.lua index 1c016633..51a74c94 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -459,26 +459,28 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport handler.lock_read = function (self, switch) out_error( "server.lua, lock_read() is deprecated, use pause() and resume()" ) if switch == true then - local tmp = _readlistlen - _readlistlen = removesocket( _readlist, socket, _readlistlen ) - _readtimes[ handler ] = nil - if _readlistlen ~= tmp then - noread = true - end + return self:pause() elseif switch == false then - if noread then - noread = false - _readlistlen = addsocket(_readlist, socket, _readlistlen) - _readtimes[ handler ] = _currenttime - end + return self:resume() end return noread end handler.pause = function (self) - return self:lock_read(true); + local tmp = _readlistlen + _readlistlen = removesocket( _readlist, socket, _readlistlen ) + _readtimes[ handler ] = nil + if _readlistlen ~= tmp then + noread = true + end + return noread; end handler.resume = function (self) - return self:lock_read(false); + if noread then + noread = false + _readlistlen = addsocket(_readlist, socket, _readlistlen) + _readtimes[ handler ] = _currenttime + end + return noread; end handler.lock = function( self, switch ) handler.lock_read (switch) -- cgit v1.2.3 From d67b08d9c48ac3bef4a495b7cd700b956c7e7c61 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 28 Oct 2018 16:54:08 +0100 Subject: server_select: Fix :lock method This always unlocks reading. I don't believe this is used anywhere. server_event does not implement this. --- net/server_select.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/server_select.lua b/net/server_select.lua index 51a74c94..d74da130 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -483,7 +483,7 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport return noread; end handler.lock = function( self, switch ) - handler.lock_read (switch) + handler.lock_read (self, switch) if switch == true then handler.write = idfalse local tmp = _sendlistlen -- cgit v1.2.3 From 193eebddaf36f0df296b70d858c97d66d5ad20bb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 28 Oct 2018 16:55:21 +0100 Subject: net.server_select: Deprecate :lock method Exists only in server_select and I found nothing using it --- net/server_select.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/net/server_select.lua b/net/server_select.lua index d74da130..e30ac8fe 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -483,6 +483,7 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport return noread; end handler.lock = function( self, switch ) + out_error( "server.lua, lock() is deprecated" ) handler.lock_read (self, switch) if switch == true then handler.write = idfalse -- cgit v1.2.3 From e21ccabfac08dfb94fda4236a3b7bc842dd54b80 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 28 Oct 2018 17:08:50 +0100 Subject: net.server_select: Replace use of deprecated :lock_read in server.link --- net/server_select.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/server_select.lua b/net/server_select.lua index e30ac8fe..475f05b8 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -719,7 +719,7 @@ local function link(sender, receiver, buffersize) function receiver.sendbuffer() _sendbuffer(); if sender_locked and receiver.bufferlen() < buffersize then - sender:lock_read(false); -- Unlock now + sender:resume(); -- Unlock now sender_locked = nil; end end @@ -729,7 +729,7 @@ local function link(sender, receiver, buffersize) _readbuffer(); if not sender_locked and receiver.bufferlen() >= buffersize then sender_locked = true; - sender:lock_read(true); + sender:pause(); end end sender:set_mode("*a"); -- cgit v1.2.3 From e1846ef9a35e2e91a3c844ef489be780811aa1b0 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 28 Oct 2018 17:11:18 +0100 Subject: net.server_select: Still allow buffering outgoing data on write-locked connections --- net/server_select.lua | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/net/server_select.lua b/net/server_select.lua index 475f05b8..745e1f49 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -424,9 +424,8 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport bufferlen = bufferlen + #data if bufferlen > maxsendlen then _closelist[ handler ] = "send buffer exceeded" -- cannot close the client at the moment, have to wait to the end of the cycle - handler.write = idfalse -- don't write anymore return false - elseif socket and not _sendlist[ socket ] then + elseif not nosend and socket and not _sendlist[ socket ] then _sendlistlen = addsocket(_sendlist, socket, _sendlistlen) end bufferqueuelen = bufferqueuelen + 1 @@ -486,7 +485,6 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport out_error( "server.lua, lock() is deprecated" ) handler.lock_read (self, switch) if switch == true then - handler.write = idfalse local tmp = _sendlistlen _sendlistlen = removesocket( _sendlist, socket, _sendlistlen ) _writetimes[ handler ] = nil @@ -494,7 +492,6 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport nosend = true end elseif switch == false then - handler.write = write if nosend then nosend = false write( "" ) -- cgit v1.2.3 From 031dc2e2a0450b25e16bbc534b29522e0ee52793 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 25 Oct 2018 15:12:59 +0200 Subject: net.server: Add an API for holding writes of outgoing data --- net/server_epoll.lua | 20 ++++++++++++++++++-- net/server_event.lua | 13 +++++++++++++ net/server_select.lua | 31 +++++++++++++++++++------------ 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index 4b40c7d5..cdf3e8fe 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -409,8 +409,10 @@ function interface:write(data) else self.writebuffer = { data }; end - self:setwritetimeout(); - self:set(nil, true); + if not self._write_lock then + self:setwritetimeout(); + self:set(nil, true); + end return #data; end interface.send = interface.write; @@ -590,6 +592,20 @@ function interface:pausefor(t) end); end +function interface:pause_writes() + self._write_lock = true; + self:setwritetimeout(false); + self:set(nil, false); +end + +function interface:resume_writes() + self._write_lock = nil; + if self.writebuffer[1] then + self:setwritetimeout(); + self:set(nil, true); + end +end + -- Connected! function interface:onconnect() if self.conn and not self.peername and self.conn.getpeername then diff --git a/net/server_event.lua b/net/server_event.lua index ca80c3f2..70757e03 100644 --- a/net/server_event.lua +++ b/net/server_event.lua @@ -273,6 +273,19 @@ function interface_mt:resume() end end +function interface_mt:pause_writes() + return self:_lock(self.nointerface, self.noreading, true); +end + +function interface_mt:resume_writes() + self:_lock(self.nointerface, self.noreading, false); + if self.writecallback and not self.eventwrite then + self.eventwrite = addevent( base, self.conn, EV_WRITE, self.writecallback, cfg.WRITE_TIMEOUT ); -- register callback + return true; + end +end + + function interface_mt:counter(c) if c then self._connections = self._connections + c diff --git a/net/server_select.lua b/net/server_select.lua index 745e1f49..693cee5e 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -485,20 +485,27 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport out_error( "server.lua, lock() is deprecated" ) handler.lock_read (self, switch) if switch == true then - local tmp = _sendlistlen - _sendlistlen = removesocket( _sendlist, socket, _sendlistlen ) - _writetimes[ handler ] = nil - if _sendlistlen ~= tmp then - nosend = true - end + handler.pause_writes (self) elseif switch == false then - if nosend then - nosend = false - write( "" ) - end + handler.resume_writes (self) end return noread, nosend end + handler.pause_writes = function (self) + local tmp = _sendlistlen + _sendlistlen = removesocket( _sendlist, socket, _sendlistlen ) + _writetimes[ handler ] = nil + if _sendlistlen ~= tmp then + nosend = true + end + end + handler.resume_writes = function (self) + if nosend then + nosend = false + write( "" ) + end + end + local _readbuffer = function( ) -- this function reads data local buffer, err, part = receive( socket, pattern ) -- receive buffer with "pattern" if not err or (err == "wantread" or err == "timeout") then -- received something @@ -716,7 +723,7 @@ local function link(sender, receiver, buffersize) function receiver.sendbuffer() _sendbuffer(); if sender_locked and receiver.bufferlen() < buffersize then - sender:resume(); -- Unlock now + sender:lock_read(false); -- Unlock now sender_locked = nil; end end @@ -726,7 +733,7 @@ local function link(sender, receiver, buffersize) _readbuffer(); if not sender_locked and receiver.bufferlen() >= buffersize then sender_locked = true; - sender:pause(); + sender:lock_read(true); end end sender:set_mode("*a"); -- cgit v1.2.3 From 81a8dca01e7df9f7dbf7478ccac4ada001a88b6d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 28 Oct 2018 18:22:17 +0100 Subject: net.server_epoll: Reschedule delayed timers relative to current time This should normally never happen, but can be reproduced by suspending the process a while. --- net/server_epoll.lua | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index cdf3e8fe..ce8996a8 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -106,9 +106,13 @@ local function runtimers(next_delay, min_wait) end local new_timeout = f(now); if new_timeout then - -- Schedule for 'delay' from the time actually scheduled, - -- not from now, in order to prevent timer drift. - timer[1] = t + new_timeout; + -- Schedule for 'delay' from the time actually scheduled, not from now, + -- in order to prevent timer drift, unless it already drifted way out of sync. + if (t + new_timeout) > ( now - new_timeout ) then + timer[1] = t + new_timeout; + else + timer[1] = now + new_timeout; + end resort_timers = true; else t_remove(timers, i); -- cgit v1.2.3 From 9270a17d2089c16c1cb37040f55638867b5adfbe Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Oct 2018 02:13:09 +0100 Subject: net.server_epoll: Use method to update peername on connect --- net/server_epoll.lua | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index ce8996a8..f7e5ae49 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -612,9 +612,7 @@ end -- Connected! function interface:onconnect() - if self.conn and not self.peername and self.conn.getpeername then - self.peername, self.peerport = self.conn:getpeername(); - end + self:updatenames(); self.onconnect = noop; self:on("connect"); end -- cgit v1.2.3 From 48162c5fca869aa8adefc5dc7e8a58bf1ce40c20 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 1 Nov 2018 23:58:41 +0100 Subject: mod_pep: Remove incorrect features advertised on the bare host --- plugins/mod_pep.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/mod_pep.lua b/plugins/mod_pep.lua index 1d8c55bf..cb775fb5 100644 --- a/plugins/mod_pep.lua +++ b/plugins/mod_pep.lua @@ -250,9 +250,6 @@ end module:hook("iq/bare/"..xmlns_pubsub..":pubsub", handle_pubsub_iq); module:hook("iq/bare/"..xmlns_pubsub_owner..":pubsub", handle_pubsub_iq); -module:add_identity("pubsub", "pep", module:get_option_string("name", "Prosody")); -module:add_feature("http://jabber.org/protocol/pubsub#publish"); - local function get_caps_hash_from_presence(stanza, current) local t = stanza.attr.type; if not t then -- cgit v1.2.3 From 1682a8437de2afe9e01d5e4daaa87f56d7af7f6b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 10 Nov 2018 15:50:32 +0100 Subject: MUC: Fix spelling in comments --- plugins/muc/muc.lib.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua index 4060535a..d9fa37f5 100644 --- a/plugins/muc/muc.lib.lua +++ b/plugins/muc/muc.lib.lua @@ -39,7 +39,7 @@ function room_mt:__tostring() end function room_mt.save() - -- overriden by mod_muc.lua + -- overridden by mod_muc.lua end function room_mt:get_occupant_jid(real_jid) @@ -279,7 +279,7 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason) self_p = st.clone(base_presence):add_child(self_x); end - -- General populance + -- General populace for occupant_nick, n_occupant in self:each_occupant() do if occupant_nick ~= occupant.nick then local pr; @@ -609,7 +609,7 @@ function room_mt:handle_normal_presence(origin, stanza) x:tag("status", {code = "303";}):up(); x:tag("status", {code = "110";}):up(); self:route_stanza(generated_unavail:add_child(x)); - dest_nick = nil; -- set dest_nick to nil; so general populance doesn't see it for whole orig_occupant + dest_nick = nil; -- set dest_nick to nil; so general populace doesn't see it for whole orig_occupant end end @@ -966,7 +966,7 @@ function room_mt:handle_admin_query_get_command(origin, stanza) local _aff_rank = valid_affiliations[_aff or "none"]; local _rol = item.attr.role; if _aff and _aff_rank and not _rol then - -- You need to be at least an admin, and be requesting info about your affifiliation or lower + -- You need to be at least an admin, and be requesting info about your affiliation or lower -- e.g. an admin can't ask for a list of owners local affiliation_rank = valid_affiliations[affiliation or "none"]; if (affiliation_rank >= valid_affiliations.admin and affiliation_rank >= _aff_rank) @@ -1291,7 +1291,7 @@ function room_mt:set_affiliation(actor, jid, affiliation, reason, data) -- Outcast can be by host. is_host_only and affiliation == "outcast" and select(2, jid_split(occupant.bare_jid)) == host ) then - -- need to publcize in all cases; as affiliation in has changed. + -- need to publicize in all cases; as affiliation in has changed. occupants_updated[occupant] = occupant.role; if occupant.role ~= role and ( is_downgrade or -- cgit v1.2.3 From c309d32629c0b0f103f32dda5506528e2e1f468b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 27 Nov 2018 17:01:47 +0100 Subject: MUC: Move check for explicit room join earlier in room creation flow --- plugins/muc/mod_muc.lua | 2 +- plugins/muc/muc.lib.lua | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/plugins/muc/mod_muc.lua b/plugins/muc/mod_muc.lua index 954bae92..89e67744 100644 --- a/plugins/muc/mod_muc.lua +++ b/plugins/muc/mod_muc.lua @@ -453,7 +453,7 @@ for event_name, method in pairs { if room == nil then -- Watch presence to create rooms - if stanza.attr.type == nil and stanza.name == "presence" then + if stanza.attr.type == nil and stanza.name == "presence" and stanza:get_child("x", "http://jabber.org/protocol/muc") then room = muclib.new_room(room_jid); return room:handle_first_presence(origin, stanza); elseif stanza.attr.type ~= "error" then diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua index 96f58023..0009e9b2 100644 --- a/plugins/muc/muc.lib.lua +++ b/plugins/muc/muc.lib.lua @@ -428,13 +428,6 @@ module:hook("muc-occupant-pre-change", function(event) end, 1); function room_mt:handle_first_presence(origin, stanza) - if not stanza:get_child("x", "http://jabber.org/protocol/muc") then - module:log("debug", "Room creation without , possibly desynced"); - - origin.send(st.error_reply(stanza, "cancel", "item-not-found")); - return true; - end - local real_jid = stanza.attr.from; local dest_jid = stanza.attr.to; local bare_jid = jid_bare(real_jid); -- cgit v1.2.3 From 83ab43ca8ff3d3e93042f76b91194f2ec3c6e36f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 28 Nov 2018 20:36:53 +0100 Subject: util.format: Tweak how nil values are handled Because [] seems exsessive --- spec/util_format_spec.lua | 2 ++ util/format.lua | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/spec/util_format_spec.lua b/spec/util_format_spec.lua index 7e6a0c6e..8a2e9312 100644 --- a/spec/util_format_spec.lua +++ b/spec/util_format_spec.lua @@ -5,6 +5,8 @@ describe("util.format", function() it("should work", function() assert.equal("hello", format("%s", "hello")); assert.equal("", format("%s")); + assert.equal("", format("%d")); + assert.equal("", format("%q")); assert.equal(" []", format("", nil)); assert.equal("true", format("%s", true)); assert.equal("[true]", format("%d", true)); diff --git a/util/format.lua b/util/format.lua index c5e513fa..16c57bc6 100644 --- a/util/format.lua +++ b/util/format.lua @@ -28,13 +28,12 @@ local function format(formatstring, ...) if spec ~= "%%" then i = i + 1; local arg = args[i]; - if arg == nil then -- special handling for nil - arg = "" - args[i] = ""; - end local option = spec:sub(-1); - if option == "q" or option == "s" then -- arg should be string + if arg == nil then + args[i] = "nil"; + spec = "<%s>"; + elseif option == "q" or option == "s" then -- arg should be string args[i] = tostring(arg); elseif type(arg) ~= "number" then -- arg isn't number as expected? args[i] = tostring(arg); -- cgit v1.2.3 From 6188d89e486917bf5b4e1c6cafae1deeb7f0d132 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 29 Nov 2018 16:16:09 +0100 Subject: configure: Split list of possible suffixes into a line per Lua version --- configure | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure b/configure index 4307997c..92f078f4 100755 --- a/configure +++ b/configure @@ -404,7 +404,9 @@ then then suffixes="5.3 53 -5.3 -53" else - suffixes="5.1 51 -5.1 -51 5.2 52 -5.2 -52 5.3 53 -5.3 -53" + suffixes="5.1 51 -5.1 -51" + suffixes="$suffixes 5.2 52 -5.2 -52" + suffixes="$suffixes 5.3 53 -5.3 -53" fi for suffix in "" $suffixes do -- cgit v1.2.3 From 16ddfbbe443acfbb5657b77978b7e774faf34753 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 29 Nov 2018 16:19:39 +0100 Subject: configure: Recognise 5.4 as a valid Lua version --- configure | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 92f078f4..c484a280 100755 --- a/configure +++ b/configure @@ -237,7 +237,7 @@ do --lua-version|--with-lua-version) [ -n "$value" ] || die "Missing value in flag $key." LUA_VERSION="$value" - [ "$LUA_VERSION" = "5.1" ] || [ "$LUA_VERSION" = "5.2" ] || [ "$LUA_VERSION" = "5.3" ] || die "Invalid Lua version in flag $key." + [ "$LUA_VERSION" = "5.1" ] || [ "$LUA_VERSION" = "5.2" ] || [ "$LUA_VERSION" = "5.3" ] || [ "$LUA_VERSION" = "5.4" ] || die "Invalid Lua version in flag $key." LUA_VERSION_SET=yes ;; --with-lua) @@ -340,7 +340,7 @@ then fi detect_lua_version() { - detected_lua=$("$1" -e 'print(_VERSION:match(" (5%.[123])$"))' 2> /dev/null) + detected_lua=$("$1" -e 'print(_VERSION:match(" (5%.[1234])$"))' 2> /dev/null) if [ "$detected_lua" != "nil" ] then if [ "$LUA_VERSION_SET" != "yes" ] @@ -403,10 +403,14 @@ then elif [ "$LUA_VERSION_SET" = "yes" ] && [ "$LUA_VERSION" = "5.3" ] then suffixes="5.3 53 -5.3 -53" + elif [ "$LUA_VERSION_SET" = "yes" ] && [ "$LUA_VERSION" = "5.4" ] + then + suffixes="5.4 54 -5.4 -54" else suffixes="5.1 51 -5.1 -51" suffixes="$suffixes 5.2 52 -5.2 -52" suffixes="$suffixes 5.3 53 -5.3 -53" + suffixes="$suffixes 5.4 54 -5.4 -54" fi for suffix in "" $suffixes do -- cgit v1.2.3 From 2e3e5db959672f2175223d670d106f1202e95f49 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 29 Nov 2018 16:53:22 +0100 Subject: net.websocket.frames: Prefer Lua 5.2 built-in bit module over LuaJIT version When running on Lua 5.2 this makes sense since bit32 is usually already loaded. It's sensible to prefer this going forward in case of incompatibilities between the two variants. --- net/websocket/frames.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/websocket/frames.lua b/net/websocket/frames.lua index ba25d261..b5aebb40 100644 --- a/net/websocket/frames.lua +++ b/net/websocket/frames.lua @@ -9,7 +9,7 @@ local softreq = require "util.dependencies".softreq; local random_bytes = require "util.random".bytes; -local bit = assert(softreq"bit" or softreq"bit32", +local bit = assert(softreq"bit32" or softreq"bit", "No bit module found. See https://prosody.im/doc/depends#bitop"); local band = bit.band; local bor = bit.bor; -- cgit v1.2.3 From 452611bf58da7b159ba69d3a36fa9624915eac57 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 30 Nov 2018 23:58:55 +0100 Subject: tests: Add scansion test for #689 about keeping the full subscription request stanza --- spec/scansion/keep_full_sub_req.scs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 spec/scansion/keep_full_sub_req.scs diff --git a/spec/scansion/keep_full_sub_req.scs b/spec/scansion/keep_full_sub_req.scs new file mode 100644 index 00000000..318bbb60 --- /dev/null +++ b/spec/scansion/keep_full_sub_req.scs @@ -0,0 +1,37 @@ +# server MUST keep a record of the complete presence stanza comprising the subscription request (#689) + +[Client] Alice + jid: pars-a@localhost + password: password + +[Client] Bob + jid: pars-b@localhost + password: password + +--------- + +Alice connects + +Alice sends: + + + + +Alice disconnects + +Bob connects + +Bob sends: + + +Bob receives: + + + +Bob receives: + + + + +Bob disconnects + -- cgit v1.2.3 From 5dff3b198343a7804811126efa30bedaad092d93 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 1 Dec 2018 18:02:58 +0100 Subject: spec/keep_full_sub_req: Add missing type attribute --- spec/scansion/keep_full_sub_req.scs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/scansion/keep_full_sub_req.scs b/spec/scansion/keep_full_sub_req.scs index 318bbb60..3b80b8b2 100644 --- a/spec/scansion/keep_full_sub_req.scs +++ b/spec/scansion/keep_full_sub_req.scs @@ -29,7 +29,7 @@ Bob receives: Bob receives: - + -- cgit v1.2.3 From 5adc46bde533b5ce92029eb33b2ddc2899dbd737 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 1 Dec 2018 18:07:56 +0100 Subject: spec/keep_full_sub_req: Verify that the presence subscription stays the same after a reconnect --- spec/scansion/keep_full_sub_req.scs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/spec/scansion/keep_full_sub_req.scs b/spec/scansion/keep_full_sub_req.scs index 3b80b8b2..5a5e1fdf 100644 --- a/spec/scansion/keep_full_sub_req.scs +++ b/spec/scansion/keep_full_sub_req.scs @@ -27,6 +27,23 @@ Bob sends: Bob receives: +Bob receives: + + + + +Bob disconnects + +# Works if they reconnect too + +Bob connects + +Bob sends: + + +Bob receives: + + Bob receives: -- cgit v1.2.3 From c219636c741e61d57ef8315480e6a596e6e4ff7a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 1 Dec 2018 18:12:01 +0100 Subject: spec/keep_full_sub_req: Make the second connect a differenct device (workaround for scansion issue) scansion threw an error when a client connected again --- spec/scansion/keep_full_sub_req.scs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/spec/scansion/keep_full_sub_req.scs b/spec/scansion/keep_full_sub_req.scs index 5a5e1fdf..244c1d55 100644 --- a/spec/scansion/keep_full_sub_req.scs +++ b/spec/scansion/keep_full_sub_req.scs @@ -8,6 +8,10 @@ jid: pars-b@localhost password: password +[Client] Bob's phone + jid: pars-b@localhost/phone + password: password + --------- Alice connects @@ -36,19 +40,19 @@ Bob disconnects # Works if they reconnect too -Bob connects +Bob's phone connects -Bob sends: +Bob's phone sends: -Bob receives: - +Bob's phone receives: + -Bob receives: +Bob's phone receives: -Bob disconnects +Bob's phone disconnects -- cgit v1.2.3 From 38b328ad1be357a028a818eaad7c6c3541ac6759 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Sat, 1 Dec 2018 22:13:24 +0000 Subject: rostermanager, mod_presence: Store stanza for incoming subscription requests (fixes #689) (thanks Zash, Ge0rG) --- core/rostermanager.lua | 6 +++--- plugins/mod_presence.lua | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/rostermanager.lua b/core/rostermanager.lua index 61b08002..2d616e4b 100644 --- a/core/rostermanager.lua +++ b/core/rostermanager.lua @@ -263,15 +263,15 @@ end function is_contact_pending_in(username, host, jid) local roster = load_roster(username, host); - return roster[false].pending[jid]; + return roster[false].pending[jid] ~= nil; end -local function set_contact_pending_in(username, host, jid) +local function set_contact_pending_in(username, host, jid, stanza) local roster = load_roster(username, host); local item = roster[jid]; if item and (item.subscription == "from" or item.subscription == "both") then return; -- false end - roster[false].pending[jid] = true; + roster[false].pending[jid] = st.is_stanza(stanza) and st.preserialize(stanza) or true; return save_roster(username, host, roster, jid); end function is_contact_pending_out(username, host, jid) diff --git a/plugins/mod_presence.lua b/plugins/mod_presence.lua index 5056a3a3..51254c63 100644 --- a/plugins/mod_presence.lua +++ b/plugins/mod_presence.lua @@ -80,8 +80,10 @@ function handle_normal_presence(origin, stanza) res.presence.attr.to = nil; end end - for jid in pairs(roster[false].pending) do -- resend incoming subscription requests - origin.send(st.presence({type="subscribe", from=jid})); -- TODO add to attribute? Use original? + for jid, pending_request in pairs(roster[false].pending) do -- resend incoming subscription requests + local subscribe = st.clone(st.deserialize(pending_request)); + subscribe.attr.type, subscribe.attr.from = "subscribe", jid; + origin.send(subscribe); end local request = st.presence({type="subscribe", from=origin.username.."@"..origin.host}); for jid, item in pairs(roster) do -- resend outgoing subscription requests @@ -225,7 +227,7 @@ function handle_inbound_presence_subscriptions_and_probes(origin, stanza, from_b else core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="unavailable"}), true); -- acknowledging receipt if not rostermanager.is_contact_pending_in(node, host, from_bare) then - if rostermanager.set_contact_pending_in(node, host, from_bare) then + if rostermanager.set_contact_pending_in(node, host, from_bare, stanza) then sessionmanager.send_to_available_resources(node, host, stanza); end -- TODO else return error, unable to save end -- cgit v1.2.3 From e984432ca4a85a2e9abef99b667731d8541ed662 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 2 Dec 2018 17:20:44 +0100 Subject: mod_presence: Remove unnecessary stanza clone call --- plugins/mod_presence.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_presence.lua b/plugins/mod_presence.lua index 51254c63..1ea837e8 100644 --- a/plugins/mod_presence.lua +++ b/plugins/mod_presence.lua @@ -81,7 +81,7 @@ function handle_normal_presence(origin, stanza) end end for jid, pending_request in pairs(roster[false].pending) do -- resend incoming subscription requests - local subscribe = st.clone(st.deserialize(pending_request)); + local subscribe = st.deserialize(pending_request); subscribe.attr.type, subscribe.attr.from = "subscribe", jid; origin.send(subscribe); end -- cgit v1.2.3 From aa3cfe1ea0241eeb60d09ad11bd2727d28887a7d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 2 Dec 2018 17:22:26 +0100 Subject: mod_presence: Handle older boolean subscription request data (thanks Martin) --- plugins/mod_presence.lua | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/mod_presence.lua b/plugins/mod_presence.lua index 1ea837e8..5aed5854 100644 --- a/plugins/mod_presence.lua +++ b/plugins/mod_presence.lua @@ -81,9 +81,13 @@ function handle_normal_presence(origin, stanza) end end for jid, pending_request in pairs(roster[false].pending) do -- resend incoming subscription requests - local subscribe = st.deserialize(pending_request); - subscribe.attr.type, subscribe.attr.from = "subscribe", jid; - origin.send(subscribe); + if type(pending_request) == "table" then + local subscribe = st.deserialize(pending_request); + subscribe.attr.type, subscribe.attr.from = "subscribe", jid; + origin.send(subscribe); + else + origin.send(st.presence({type="subscribe", from=jid})); + end end local request = st.presence({type="subscribe", from=origin.username.."@"..origin.host}); for jid, item in pairs(roster) do -- resend outgoing subscription requests -- cgit v1.2.3 From f6cb6c1fed62059341d1b10aa9bb8b9720833fde Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 3 Dec 2018 19:38:19 +0000 Subject: configure: Also look for lua.h in a directory with the same suffix as the interpreter (FreeBSD-friendly) --- configure | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/configure b/configure index c484a280..4a816364 100755 --- a/configure +++ b/configure @@ -469,24 +469,37 @@ if [ -f "$lua_h" ] then echo "lua.h found in $lua_h" else - v_dir="$LUA_INCDIR/lua/$LUA_VERSION" - lua_h="$v_dir/lua.h" - if [ -f "$lua_h" ] - then + for postfix in "$LUA_VERSION" "$LUA_SUFFIX"; do + if ! [ "$postfix" = "" ]; then + v_dir="$LUA_INCDIR/lua/$postfix"; + else + v_dir="$LUA_INCDIR/lua"; + fi + lua_h="$v_dir/lua.h" + if [ -f "$lua_h" ] + then echo "lua.h found in $lua_h" LUA_INCDIR="$v_dir" - else - d_dir="$LUA_INCDIR/lua$LUA_VERSION" + break; + else + d_dir="$LUA_INCDIR/lua$postfix" lua_h="$d_dir/lua.h" if [ -f "$lua_h" ] then - echo "lua.h found in $lua_h (Debian/Ubuntu)" - LUA_INCDIR="$d_dir" + echo "lua.h found in $lua_h" + LUA_INCDIR="$d_dir" + break; else - echo "lua.h not found (looked in $LUA_INCDIR, $v_dir, $d_dir)" - die "You may want to use the flag --with-lua or --with-lua-include. See --help." + lua_h_search="$lua_h_search\n $v_dir/lua.h\n $d_dir/lua.h" fi - fi + fi + done + if [ ! -f "$lua_h" ]; then + echo "lua.h not found. Looked for:" + echo "$lua_h_search" | uniq + echo + die "You may want to use the flag --with-lua or --with-lua-include. See --help." + fi fi if [ "$lua_interp_found" = "yes" ] -- cgit v1.2.3 From 3e2ef7ac97fbb6b9c544f7f7f6937e395ef58a8d Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 3 Dec 2018 23:06:41 +0000 Subject: configure: Refactor header search to make it more portable --- configure | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/configure b/configure index 4a816364..dec7b60d 100755 --- a/configure +++ b/configure @@ -463,12 +463,13 @@ then LUA_LIBDIR="$LUA_DIR/lib" fi -echo_n "Checking Lua includes... " lua_h="$LUA_INCDIR/lua.h" +echo_n "Looking for lua.h at $lua_h..." if [ -f "$lua_h" ] then - echo "lua.h found in $lua_h" + echo found else + echo "not found" for postfix in "$LUA_VERSION" "$LUA_SUFFIX"; do if ! [ "$postfix" = "" ]; then v_dir="$LUA_INCDIR/lua/$postfix"; @@ -476,27 +477,29 @@ else v_dir="$LUA_INCDIR/lua"; fi lua_h="$v_dir/lua.h" + echo_n "Looking for lua.h at $lua_h..." if [ -f "$lua_h" ] then - echo "lua.h found in $lua_h" LUA_INCDIR="$v_dir" + echo found break; else + echo "not found" d_dir="$LUA_INCDIR/lua$postfix" lua_h="$d_dir/lua.h" + echo_n "Looking for lua.h at $lua_h..." if [ -f "$lua_h" ] then - echo "lua.h found in $lua_h" + echo found LUA_INCDIR="$d_dir" break; else - lua_h_search="$lua_h_search\n $v_dir/lua.h\n $d_dir/lua.h" + echo "not found" fi fi done if [ ! -f "$lua_h" ]; then - echo "lua.h not found. Looked for:" - echo "$lua_h_search" | uniq + echo "lua.h not found." echo die "You may want to use the flag --with-lua or --with-lua-include. See --help." fi -- cgit v1.2.3 From 4d02538a3dd73f8f7f7c21d21aa3ffafecfd7a0c Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 4 Dec 2018 12:11:15 +0000 Subject: util.time: Bump POSIX_C_SOURCE to ensure visibility of CLOCK_MONOTONIC on FreeBSD (fixes #1253) --- util-src/time.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-src/time.c b/util-src/time.c index bfad52ee..bc6b5b1c 100644 --- a/util-src/time.c +++ b/util-src/time.c @@ -1,5 +1,5 @@ #ifndef _POSIX_C_SOURCE -#define _POSIX_C_SOURCE 199309L +#define _POSIX_C_SOURCE 200809L #endif #include -- cgit v1.2.3 From 6c6c7bd07f3bcd44d8bb8294180510b4f29a0fef Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 4 Dec 2018 12:11:58 +0000 Subject: util.pposix: Don't define POSIX_C_SOURCE on FreeBSD to ensure visibility of initgroups() --- util-src/pposix.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/util-src/pposix.c b/util-src/pposix.c index 5c926603..169343b8 100644 --- a/util-src/pposix.c +++ b/util-src/pposix.c @@ -25,14 +25,18 @@ #define _DEFAULT_SOURCE #endif #endif + #if defined(__APPLE__) #ifndef _DARWIN_C_SOURCE #define _DARWIN_C_SOURCE #endif #endif + +#if ! defined(__FreeBSD__) #ifndef _POSIX_C_SOURCE #define _POSIX_C_SOURCE 200809L #endif +#endif #include #include -- cgit v1.2.3 From f82561a4992773fc76c214ead3e442e6ba5f1af0 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 4 Dec 2018 16:19:08 +0000 Subject: makefile: Add lint target (to match GNUMakefile) --- makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/makefile b/makefile index d19ec24d..1b8e99b5 100644 --- a/makefile +++ b/makefile @@ -19,6 +19,8 @@ INSTALL_EXEC=$(INSTALL) -m755 MKDIR=install -d MKDIR_PRIVATE=$(MKDIR) -m750 +LUACHECK=luacheck + .PHONY: all test clean install all: prosody.install prosodyctl.install prosody.cfg.lua.install prosody.version @@ -68,6 +70,11 @@ clean: rm -f prosody.version $(MAKE) clean -C util-src +lint: + $(LUACHECK) -q $$(HGPLAIN= hg files -I '**.lua') prosody prosodyctl + @echo $$(sed -n '/^\tlocal exclude_files/,/^}/p;' .luacheckrc | sed '1d;$d' | wc -l) files ignored + shellcheck configure + test: busted --lua=$(RUNWITH) -- cgit v1.2.3 From 256b2aff1bc66f653fbbaa3ea35586db5e61141f Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 4 Dec 2018 16:19:58 +0000 Subject: makefile: Allow configuring path to busted (to match GNUMakefile) --- makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/makefile b/makefile index 1b8e99b5..0b1c8788 100644 --- a/makefile +++ b/makefile @@ -20,6 +20,7 @@ MKDIR=install -d MKDIR_PRIVATE=$(MKDIR) -m750 LUACHECK=luacheck +BUSTED=busted .PHONY: all test clean install @@ -76,7 +77,7 @@ lint: shellcheck configure test: - busted --lua=$(RUNWITH) + $(BUSTED) --lua=$(RUNWITH) prosody.install: prosody -- cgit v1.2.3 From 27083633e2ef371a834b30c654fefb6fe665bc8a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 4 Dec 2018 19:49:31 +0100 Subject: MUC/subject: Don't consider messages with or (fixes #667) --- plugins/muc/subject.lib.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/muc/subject.lib.lua b/plugins/muc/subject.lib.lua index 938abf61..c8b99cc7 100644 --- a/plugins/muc/subject.lib.lua +++ b/plugins/muc/subject.lib.lua @@ -94,6 +94,12 @@ module:hook("muc-occupant-groupchat", function(event) local stanza = event.stanza; local subject = stanza:get_child("subject"); if subject then + if stanza:get_child("body") or stanza:get_child("thread") then + -- Note: A message with a and a or a and + -- a is a legitimate message, but it SHALL NOT be interpreted + -- as a subject change. + return; + end local room = event.room; local occupant = event.occupant; -- Role check for subject changes -- cgit v1.2.3 From 7d5ceb5754d44db6255a5f5325a737682c0a3178 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 6 Dec 2018 17:54:50 +0100 Subject: MUC: Add test case for #667 --- spec/scansion/muc_subject_issue_667.scs | 79 +++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 spec/scansion/muc_subject_issue_667.scs diff --git a/spec/scansion/muc_subject_issue_667.scs b/spec/scansion/muc_subject_issue_667.scs new file mode 100644 index 00000000..68f4c17a --- /dev/null +++ b/spec/scansion/muc_subject_issue_667.scs @@ -0,0 +1,79 @@ +# #667 MUC message with subject and body SHALL NOT be interpreted as a subject change + +[Client] Romeo + password: password + jid: romeo@localhost + +----- + +Romeo connects + +# and creates a room +Romeo sends: + + + + +Romeo receives: + + + + + + + + +Romeo receives: + + + + +Romeo sends: + + Greetings + Hello everyone + + +Romeo receives: + + Greetings + Hello everyone + + +Romeo sends: + + Something to talk about + + +Romeo receives: + + Something to talk about + + +Romeo sends: + + + + +Romeo receives: + + + + + + + +# These have delay tags but we ignore those for now +Romeo receives: + + Greetings + Hello everyone + + +Romeo receives: + + Something to talk about + + +Romeo disconnects + -- cgit v1.2.3 From 9ca48298d7444c3fa207487419e06c76716cb795 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 16:35:00 +0100 Subject: moduleapi: Use pack from util.table --- core/moduleapi.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/moduleapi.lua b/core/moduleapi.lua index 10f9f04d..d2aa1e8c 100644 --- a/core/moduleapi.lua +++ b/core/moduleapi.lua @@ -20,7 +20,7 @@ local error, setmetatable, type = error, setmetatable, type; local ipairs, pairs, select = ipairs, pairs, select; local tonumber, tostring = tonumber, tostring; local require = require; -local pack = table.pack or function(...) return {n=select("#",...), ...}; end -- table.pack is only in 5.2 +local pack = table.pack or require "util.table".pack; -- table.pack is only in 5.2 local unpack = table.unpack or unpack; --luacheck: ignore 113 -- renamed in 5.2 local prosody = prosody; -- cgit v1.2.3 From 755f1d8050c373af7abf578ac30067f1d141c93f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 16:35:39 +0100 Subject: util.format: Use pack from util.table --- util/format.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util/format.lua b/util/format.lua index 16c57bc6..6c46384a 100644 --- a/util/format.lua +++ b/util/format.lua @@ -3,12 +3,13 @@ -- local tostring = tostring; -local select = select; local unpack = table.unpack or unpack; -- luacheck: ignore 113/unpack +local pack = require "util.table".pack; -- TODO table.pack in 5.2+ local type = type; local function format(formatstring, ...) - local args, args_length = { ... }, select('#', ...); + local args = pack(...); + local args_length = args.n; -- format specifier spec: -- 1. Start: '%%' -- cgit v1.2.3 From 08cb0002476600be61921a3a2a15bc9626e052e5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 16:36:05 +0100 Subject: util.iterators: Use pack from table.pack --- util/iterators.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/iterators.lua b/util/iterators.lua index 302cca36..c03c2fd6 100644 --- a/util/iterators.lua +++ b/util/iterators.lua @@ -11,9 +11,9 @@ local it = {}; local t_insert = table.insert; -local select, next = select, next; -local unpack = table.unpack or unpack; --luacheck: ignore 113 143 -local pack = table.pack or function (...) return { n = select("#", ...), ... }; end -- luacheck: ignore 143 +local next = next; +local unpack = table.unpack or unpack; --luacheck: ignore 113 +local pack = table.pack or require "util.table".pack; local type = type; local table, setmetatable = table, setmetatable; -- cgit v1.2.3 From d67408b4d87c4556371c15082097ac00f4edcb6c Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 17:00:30 +0100 Subject: luacheckrc: Set Lua standard to 5.3 with 5.2 compat enabled --- .luacheckrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.luacheckrc b/.luacheckrc index ce3d377b..3192768c 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -2,6 +2,7 @@ cache = true codes = true ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", "143/table", "113/unpack" } +std = "lua53c" max_line_length = 150 read_globals = { -- cgit v1.2.3 From e22dada0b27f0793bd88b4882fc73fcdc7936b6a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 17:07:28 +0100 Subject: lint: No longer ignore access to the deprecated global 'unpack' _G.unpack is deprecated in Lua 5.2 --- .luacheckrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.luacheckrc b/.luacheckrc index 3192768c..96882c73 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -1,6 +1,6 @@ cache = true codes = true -ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", "143/table", "113/unpack" } +ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", "143/table", } std = "lua53c" max_line_length = 150 -- cgit v1.2.3 From 6dcf53f7b284f265ce3f2215e33c6f0c28be5ce9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 17:09:55 +0100 Subject: various: Don't rely on _G.unpack existing --- net/resolvers/basic.lua | 1 + net/resolvers/manual.lua | 1 + net/resolvers/service.lua | 1 + net/websocket/frames.lua | 1 + plugins/mod_admin_telnet.lua | 1 + plugins/mod_pep_simple.lua | 1 + plugins/mod_storage_sql.lua | 2 +- spec/core_storagemanager_spec.lua | 2 +- 8 files changed, 8 insertions(+), 2 deletions(-) diff --git a/net/resolvers/basic.lua b/net/resolvers/basic.lua index 9a3c9952..56f9c77d 100644 --- a/net/resolvers/basic.lua +++ b/net/resolvers/basic.lua @@ -1,5 +1,6 @@ local adns = require "net.adns"; local inet_pton = require "util.net".pton; +local unpack = table.unpack or unpack; -- luacheck: ignore 113 local methods = {}; local resolver_mt = { __index = methods }; diff --git a/net/resolvers/manual.lua b/net/resolvers/manual.lua index c0d4e5d5..dbc40256 100644 --- a/net/resolvers/manual.lua +++ b/net/resolvers/manual.lua @@ -1,5 +1,6 @@ local methods = {}; local resolver_mt = { __index = methods }; +local unpack = table.unpack or unpack; -- luacheck: ignore 113 -- Find the next target to connect to, and -- pass it to cb() diff --git a/net/resolvers/service.lua b/net/resolvers/service.lua index b5a2d821..d1b8556c 100644 --- a/net/resolvers/service.lua +++ b/net/resolvers/service.lua @@ -1,5 +1,6 @@ local adns = require "net.adns"; local basic = require "net.resolvers.basic"; +local unpack = table.unpack or unpack; -- luacheck: ignore 113 local methods = {}; local resolver_mt = { __index = methods }; diff --git a/net/websocket/frames.lua b/net/websocket/frames.lua index b5aebb40..c3333020 100644 --- a/net/websocket/frames.lua +++ b/net/websocket/frames.lua @@ -16,6 +16,7 @@ local bor = bit.bor; local bxor = bit.bxor; local lshift = bit.lshift; local rshift = bit.rshift; +local unpack = table.unpack or unpack; -- luacheck: ignore 113 local t_concat = table.concat; local s_byte = string.byte; diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 1cbe27a4..8a3508db 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -22,6 +22,7 @@ local prosody = _G.prosody; local console_listener = { default_port = 5582; default_mode = "*a"; interface = "127.0.0.1" }; +local unpack = table.unpack or unpack; -- luacheck: ignore 113 local iterators = require "util.iterators"; local keys, values = iterators.keys, iterators.values; local jid_bare, jid_split, jid_join = import("util.jid", "bare", "prepped_split", "join"); diff --git a/plugins/mod_pep_simple.lua b/plugins/mod_pep_simple.lua index f0b5d7ef..f91e5448 100644 --- a/plugins/mod_pep_simple.lua +++ b/plugins/mod_pep_simple.lua @@ -14,6 +14,7 @@ local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed local pairs = pairs; local next = next; local type = type; +local unpack = table.unpack or unpack; -- luacheck: ignore 113 local calculate_hash = require "util.caps".calculate_hash; local core_post_stanza = prosody.core_post_stanza; local bare_sessions = prosody.bare_sessions; diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 56cef569..5c0c0208 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -11,7 +11,7 @@ local is_stanza = require"util.stanza".is_stanza; local t_concat = table.concat; local noop = function() end -local unpack = table.unpack or unpack; +local unpack = table.unpack or unpack; -- luacheck: ignore 113 local function iterator(result) return function(result_) local row = result_(); diff --git a/spec/core_storagemanager_spec.lua b/spec/core_storagemanager_spec.lua index a0a8b5ef..fd2f8742 100644 --- a/spec/core_storagemanager_spec.lua +++ b/spec/core_storagemanager_spec.lua @@ -1,4 +1,4 @@ -local unpack = table.unpack or unpack; +local unpack = table.unpack or unpack; -- luacheck: ignore 113 local server = require "net.server_select"; package.loaded["net.server"] = server; -- cgit v1.2.3 From a8b096516b2ef1791bc348e18baa7e7f5399de8f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 17:10:51 +0100 Subject: lint: Remove use of the 143 error code Does not appear to be invoked by anything --- net/websocket/frames.lua | 4 ++-- util/import.lua | 2 +- util/multitable.lua | 2 +- util/serialization.lua | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/net/websocket/frames.lua b/net/websocket/frames.lua index c3333020..86752109 100644 --- a/net/websocket/frames.lua +++ b/net/websocket/frames.lua @@ -22,8 +22,8 @@ local t_concat = table.concat; local s_byte = string.byte; local s_char= string.char; local s_sub = string.sub; -local s_pack = string.pack; -- luacheck: ignore 143 -local s_unpack = string.unpack; -- luacheck: ignore 143 +local s_pack = string.pack; +local s_unpack = string.unpack; if not s_pack and softreq"struct" then s_pack = softreq"struct".pack; diff --git a/util/import.lua b/util/import.lua index 8ecfe43c..1007bc0a 100644 --- a/util/import.lua +++ b/util/import.lua @@ -8,7 +8,7 @@ -local unpack = table.unpack or unpack; --luacheck: ignore 113 143 +local unpack = table.unpack or unpack; --luacheck: ignore 113 local t_insert = table.insert; function _G.import(module, ...) local m = package.loaded[module] or require(module); diff --git a/util/multitable.lua b/util/multitable.lua index 8d32ed8a..4f2cd972 100644 --- a/util/multitable.lua +++ b/util/multitable.lua @@ -9,7 +9,7 @@ local select = select; local t_insert = table.insert; local pairs, next, type = pairs, next, type; -local unpack = table.unpack or unpack; --luacheck: ignore 113 143 +local unpack = table.unpack or unpack; --luacheck: ignore 113 local _ENV = nil; -- luacheck: std none diff --git a/util/serialization.lua b/util/serialization.lua index dd6a2a2b..7ae77a3a 100644 --- a/util/serialization.lua +++ b/util/serialization.lua @@ -20,7 +20,6 @@ local pcall = pcall; local envload = require"util.envload".envload; local pos_inf, neg_inf = math.huge, -math.huge; --- luacheck: ignore 143/math local m_type = math.type or function (n) return n % 1 == 0 and n <= 9007199254740992 and n >= -9007199254740992 and "integer" or "float"; end; -- cgit v1.2.3 From c0b8a6ef63746675fbe1441bceed47f6fb2bee88 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 12 Oct 2018 01:29:34 +0200 Subject: util.format: Serialize values for the %q format Improves eg debug logs --- spec/util_format_spec.lua | 1 + util/format.lua | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/spec/util_format_spec.lua b/spec/util_format_spec.lua index 8a2e9312..b9652d19 100644 --- a/spec/util_format_spec.lua +++ b/spec/util_format_spec.lua @@ -11,6 +11,7 @@ describe("util.format", function() assert.equal("true", format("%s", true)); assert.equal("[true]", format("%d", true)); assert.equal("% [true]", format("%%", true)); + assert.equal("{ }", format("%q", { })); end); end); end); diff --git a/util/format.lua b/util/format.lua index 6c46384a..c31f599f 100644 --- a/util/format.lua +++ b/util/format.lua @@ -6,6 +6,7 @@ local tostring = tostring; local unpack = table.unpack or unpack; -- luacheck: ignore 113/unpack local pack = require "util.table".pack; -- TODO table.pack in 5.2+ local type = type; +local dump = require "util.serialization".new("debug"); local function format(formatstring, ...) local args = pack(...); @@ -34,7 +35,10 @@ local function format(formatstring, ...) if arg == nil then args[i] = "nil"; spec = "<%s>"; - elseif option == "q" or option == "s" then -- arg should be string + elseif option == "q" then + args[i] = dump(arg); + spec = "%s"; + elseif option == "s" then args[i] = tostring(arg); elseif type(arg) ~= "number" then -- arg isn't number as expected? args[i] = tostring(arg); -- cgit v1.2.3 From 109ee6261982614ea367de4b2716b0ea6f375168 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 17:13:39 +0100 Subject: luacheckrc: No longer ignore access to undefined fields on table lib --- .luacheckrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.luacheckrc b/.luacheckrc index 96882c73..3e3fb2b5 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -1,6 +1,6 @@ cache = true codes = true -ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", "143/table", } +ignore = { "411/err", "421/err", "411/ok", "421/ok", "211/_ENV", "431/log", } std = "lua53c" max_line_length = 150 -- cgit v1.2.3 From a575159cb0172220dd3657ed8a3eff5ca22d9532 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 21:17:39 +0100 Subject: net.server_epoll: Call onconnect right after accept()ing a new client --- net/server_epoll.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index 13c8315a..3088b55b 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -577,6 +577,8 @@ function interface:onacceptable() client:init(); if self.tls_direct then client:starttls(self.tls_ctx); + else + client:onconnect(); end end -- cgit v1.2.3 From 05886ceece8c11a31ca9f414332bb9fa99c15473 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 21:28:48 +0100 Subject: net.server_epoll: Bail on callback error An error calling a callback would be considered a truthy return value, which is not right. --- net/server_epoll.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index 3088b55b..b2165b1d 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -180,6 +180,7 @@ function interface:on(what, ...) local ok, err = pcall(listener, self, ...); if not ok then log("error", "Error calling on%s: %s", what, err); + return; end return err; end -- cgit v1.2.3 From 4a60144a3f67a83ccab4f1599548f292e20dc24b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 9 Dec 2018 20:53:33 +0100 Subject: net.connlisteners: Remove deprecated stub module This was deprecated in 0.9.x Removing so auto-completion chooses net/connect.lua instead of net/conn --- net/connlisteners.lua | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 net/connlisteners.lua diff --git a/net/connlisteners.lua b/net/connlisteners.lua deleted file mode 100644 index 9b8f88c3..00000000 --- a/net/connlisteners.lua +++ /dev/null @@ -1,18 +0,0 @@ --- COMPAT w/pre-0.9 -local log = require "util.logger".init("net.connlisteners"); -local traceback = debug.traceback; - -local _ENV = nil; --- luacheck: std none - -local function fail() - log("error", "Attempt to use legacy connlisteners API. For more info see https://prosody.im/doc/developers/network"); - log("error", "Legacy connlisteners API usage, %s", traceback("", 2)); -end - -return { - register = fail; - get = fail; - start = fail; - -- epic fail -}; -- cgit v1.2.3 From 15113f0bb16515caf35d100d4aedf4894bdbda6b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 11 Dec 2018 23:24:14 +0100 Subject: spec/scansion/prosody.cfg.lua: Update a comment from prosody.cfg.lua.dist for easier comparisons --- spec/scansion/prosody.cfg.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/scansion/prosody.cfg.lua b/spec/scansion/prosody.cfg.lua index 94861449..170371e1 100644 --- a/spec/scansion/prosody.cfg.lua +++ b/spec/scansion/prosody.cfg.lua @@ -14,7 +14,7 @@ modules_enabled = { -- Not essential, but recommended "carbons"; -- Keep multiple clients in sync - "pep"; -- Enables users to publish their mood, activity, playing music and more + "pep"; -- Enables users to publish their avatar, mood, activity, playing music and more "private"; -- Private XML storage (for room bookmarks, etc.) "blocklist"; -- Allow users to block communications with other users "vcard"; -- Allow users to set vCards -- cgit v1.2.3 From 743c5bd552deeb935adcf94057cd015e10a70745 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 11 Dec 2018 23:25:16 +0100 Subject: spec/scansion/prosody.cfg.lua: Replace mod_vcard with mod_vcard4 and mod_vcard_legacy as in default config --- spec/scansion/prosody.cfg.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/scansion/prosody.cfg.lua b/spec/scansion/prosody.cfg.lua index 170371e1..e8872bff 100644 --- a/spec/scansion/prosody.cfg.lua +++ b/spec/scansion/prosody.cfg.lua @@ -17,7 +17,8 @@ modules_enabled = { "pep"; -- Enables users to publish their avatar, mood, activity, playing music and more "private"; -- Private XML storage (for room bookmarks, etc.) "blocklist"; -- Allow users to block communications with other users - "vcard"; -- Allow users to set vCards + "vcard4"; -- User profiles (stored in PEP) + "vcard_legacy"; -- Conversion between legacy vCard and PEP Avatar, vcard -- Nice to have "version"; -- Replies to server version requests -- cgit v1.2.3 From abbf8a6b78264e44bb7d634dbf1bcb66649e0248 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 11 Dec 2018 23:26:16 +0100 Subject: spec/scansion/prosody.cfg.lua: Add remaining modules listened in prosody.cfg.lua.dist for easier comparisons --- spec/scansion/prosody.cfg.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/scansion/prosody.cfg.lua b/spec/scansion/prosody.cfg.lua index e8872bff..8d6e7c0a 100644 --- a/spec/scansion/prosody.cfg.lua +++ b/spec/scansion/prosody.cfg.lua @@ -27,6 +27,11 @@ modules_enabled = { "ping"; -- Replies to XMPP pings with pongs "register"; -- Allow users to register on this server using a client and change passwords --"mam"; -- Store messages in an archive and allow users to access it + --"csi_simple"; -- Simple Mobile optimizations + + -- Admin interfaces + --"admin_adhoc"; -- Allows administration via an XMPP client that supports ad-hoc commands + --"admin_telnet"; -- Opens telnet console interface on localhost port 5582 -- HTTP modules --"bosh"; -- Enable BOSH clients, aka "Jabber over HTTP" -- cgit v1.2.3 From a454069cfa2a593d304d0fb49997b93eee66bef9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 16 Dec 2018 02:56:11 +0100 Subject: core.rostermanager: Cache rosters of offline users for faster access (fixes #1233) --- core/rostermanager.lua | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/core/rostermanager.lua b/core/rostermanager.lua index 2d616e4b..d551a1b1 100644 --- a/core/rostermanager.lua +++ b/core/rostermanager.lua @@ -12,6 +12,7 @@ local log = require "util.logger".init("rostermanager"); local new_id = require "util.id".short; +local new_cache = require "util.cache".new; local pairs = pairs; local tostring = tostring; @@ -111,6 +112,23 @@ local function load_roster(username, host) else -- Attempt to load roster for non-loaded user log("debug", "load_roster: loading for offline user: %s", jid); end + local roster_cache = hosts[host] and hosts[host].roster_cache; + if not roster_cache then + if hosts[host] then + roster_cache = new_cache(1024); + hosts[host].roster_cache = roster_cache; + end + else + roster = roster_cache:get(jid); + if roster then + log("debug", "load_roster: cache hit"); + roster_cache:set(jid, roster); + if user then user.roster = roster; end + return roster; + else + log("debug", "load_roster: cache miss, loading from storage"); + end + end local roster_store = storagemanager.open(host, "roster", "keyval"); local data, err = roster_store:get(username); roster = data or {}; @@ -134,6 +152,10 @@ local function load_roster(username, host) if not err then hosts[host].events.fire_event("roster-load", { username = username, host = host, roster = roster }); end + if roster_cache and not user then + log("debug", "load_roster: caching loaded roster"); + roster_cache:set(jid, roster); + end return roster, err; end -- cgit v1.2.3 From 6e6914b3cab39ff86f39869879a284e97c1c11df Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 16 Dec 2018 22:49:58 +0100 Subject: MUC: Add another message to #667 test --- spec/scansion/muc_subject_issue_667.scs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/spec/scansion/muc_subject_issue_667.scs b/spec/scansion/muc_subject_issue_667.scs index 68f4c17a..859a0bfd 100644 --- a/spec/scansion/muc_subject_issue_667.scs +++ b/spec/scansion/muc_subject_issue_667.scs @@ -50,6 +50,16 @@ Romeo receives: Something to talk about +Romeo sends: + + Lorem ipsum dolor sit amet + + +Romeo receives: + + Lorem ipsum dolor sit amet + + Romeo sends: @@ -70,6 +80,11 @@ Romeo receives: Hello everyone +Romeo receives: + + Lorem ipsum dolor sit amet + + Romeo receives: Something to talk about -- cgit v1.2.3 From 77a51af33dbafc0f301bce2f3b09122645f4e406 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 16 Dec 2018 22:53:56 +0100 Subject: MUC: Add descriptive comments to #667 test --- spec/scansion/muc_subject_issue_667.scs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/spec/scansion/muc_subject_issue_667.scs b/spec/scansion/muc_subject_issue_667.scs index 859a0bfd..417f957a 100644 --- a/spec/scansion/muc_subject_issue_667.scs +++ b/spec/scansion/muc_subject_issue_667.scs @@ -23,11 +23,13 @@ Romeo receives: +# the default (empty) subject Romeo receives: +# this should be treated as a normal message Romeo sends: Greetings @@ -40,6 +42,7 @@ Romeo receives: Hello everyone +# this is a subject change Romeo sends: Something to talk about @@ -50,6 +53,7 @@ Romeo receives: Something to talk about +# a message without Romeo sends: Lorem ipsum dolor sit amet @@ -60,11 +64,13 @@ Romeo receives: Lorem ipsum dolor sit amet +# Resync Romeo sends: +# Presences Romeo receives: @@ -73,6 +79,7 @@ Romeo receives: +# History # These have delay tags but we ignore those for now Romeo receives: @@ -85,6 +92,7 @@ Romeo receives: Lorem ipsum dolor sit amet +# Finally, the topic Romeo receives: Something to talk about -- cgit v1.2.3 From 9aee66a6c5ee1754afd603b14c7109c2b13db542 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 16 Dec 2018 22:59:14 +0100 Subject: MUC: Test that subject is still empty after sending a non-subject change message with a subject (#667) --- spec/scansion/muc_subject_issue_667.scs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/spec/scansion/muc_subject_issue_667.scs b/spec/scansion/muc_subject_issue_667.scs index 417f957a..74980073 100644 --- a/spec/scansion/muc_subject_issue_667.scs +++ b/spec/scansion/muc_subject_issue_667.scs @@ -42,6 +42,33 @@ Romeo receives: Hello everyone +# Resync +Romeo sends: + + + + +# Presences +Romeo receives: + + + + + + + +Romeo receives: + + Greetings + Hello everyone + + +# the still empty subject +Romeo receives: + + + + # this is a subject change Romeo sends: -- cgit v1.2.3 From 91e23f83983af91b65287c209caa54f9027a6487 Mon Sep 17 00:00:00 2001 From: Maxime ?pep? Buquet Date: Tue, 18 Dec 2018 20:23:33 +0000 Subject: admin_telnet: show when bidi is used on s2s --- plugins/mod_admin_telnet.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 8a3508db..63136d63 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -521,6 +521,9 @@ local function session_flags(session, line) if session.remote then line[#line+1] = "(remote)"; end + if session.is_bidi then + line[#line+1] = "(bidi)"; + end return table.concat(line, " "); end -- cgit v1.2.3 From ed48077ce4b5ac8310270edb93c7d68a88cf0602 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 23 Dec 2018 02:50:22 +0100 Subject: mod_pubsub: Add semicolon (code style) --- plugins/mod_pubsub/mod_pubsub.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_pubsub/mod_pubsub.lua b/plugins/mod_pubsub/mod_pubsub.lua index 40adcafe..1edc721b 100644 --- a/plugins/mod_pubsub/mod_pubsub.lua +++ b/plugins/mod_pubsub/mod_pubsub.lua @@ -73,7 +73,7 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj) local msg_type = node_obj and node_obj.config.message_type or "headline"; local message = st.message({ from = module.host, type = msg_type, id = id }) :tag("event", { xmlns = xmlns_pubsub_event }) - :tag(kind, { node = node }) + :tag(kind, { node = node }); if item then message:add_child(item); -- cgit v1.2.3 From 4cbf600434638bcb53c8defb767a076135ab8f24 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 23 Dec 2018 02:53:12 +0100 Subject: mod_pep: Move broadcaster code around to be more like in mod_pubsub This eases comparing and contrasting these two modules. --- plugins/mod_pep.lua | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/mod_pep.lua b/plugins/mod_pep.lua index cb775fb5..40385616 100644 --- a/plugins/mod_pep.lua +++ b/plugins/mod_pep.lua @@ -136,9 +136,6 @@ local function get_broadcaster(username) if kind == "retract" then kind = "items"; -- XEP-0060 signals retraction in an container end - local message = st.message({ from = user_bare, type = "headline" }) - :tag("event", { xmlns = xmlns_pubsub_event }) - :tag(kind, { node = node }); if item then item = st.clone(item); item.attr.xmlns = nil; -- Clear the pubsub namespace @@ -147,6 +144,12 @@ local function get_broadcaster(username) item:maptags(function () return nil; end); end end + end + + local message = st.message({ from = user_bare, type = "headline" }) + :tag("event", { xmlns = xmlns_pubsub_event }) + :tag(kind, { node = node }); + if item then message:add_child(item); end for jid in pairs(jids) do -- cgit v1.2.3 From 36d31b94bf54bac423fdeac12c4978d22ca1b6f8 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 23 Dec 2018 02:54:39 +0100 Subject: mod_pep: Add some spacing between blocks in broadcaster to improve readability --- plugins/mod_pep.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/mod_pep.lua b/plugins/mod_pep.lua index 40385616..6275d47c 100644 --- a/plugins/mod_pep.lua +++ b/plugins/mod_pep.lua @@ -136,6 +136,7 @@ local function get_broadcaster(username) if kind == "retract" then kind = "items"; -- XEP-0060 signals retraction in an container end + if item then item = st.clone(item); item.attr.xmlns = nil; -- Clear the pubsub namespace @@ -149,9 +150,11 @@ local function get_broadcaster(username) local message = st.message({ from = user_bare, type = "headline" }) :tag("event", { xmlns = xmlns_pubsub_event }) :tag(kind, { node = node }); + if item then message:add_child(item); end + for jid in pairs(jids) do module:log("debug", "Sending notification to %s from %s: %s", jid, user_bare, tostring(item)); message.attr.to = jid; -- cgit v1.2.3 From b4e3b4ea25135148113b1fce6207b28889a4eb7e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 23 Dec 2018 02:56:10 +0100 Subject: mod_pep: Set an 'id' on notifications mod_pubsub got this in f2d35eee69c9 --- plugins/mod_pep.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/mod_pep.lua b/plugins/mod_pep.lua index 6275d47c..5e3f43f2 100644 --- a/plugins/mod_pep.lua +++ b/plugins/mod_pep.lua @@ -8,6 +8,7 @@ local calculate_hash = require "util.caps".calculate_hash; local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed; local cache = require "util.cache"; local set = require "util.set"; +local new_id = require "util.id".medium; local xmlns_pubsub = "http://jabber.org/protocol/pubsub"; local xmlns_pubsub_event = "http://jabber.org/protocol/pubsub#event"; @@ -147,7 +148,8 @@ local function get_broadcaster(username) end end - local message = st.message({ from = user_bare, type = "headline" }) + local id = new_id(); + local message = st.message({ from = user_bare, type = "headline", id = id }) :tag("event", { xmlns = xmlns_pubsub_event }) :tag(kind, { node = node }); -- cgit v1.2.3 From 6c573bacf091c7fe4e15591833639003b2519a3b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 23 Dec 2018 03:05:58 +0100 Subject: mod_pubsub: Change order of luacheck directives to match arguments they apply to --- plugins/mod_pubsub/mod_pubsub.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_pubsub/mod_pubsub.lua b/plugins/mod_pubsub/mod_pubsub.lua index 1edc721b..abc4fee8 100644 --- a/plugins/mod_pubsub/mod_pubsub.lua +++ b/plugins/mod_pubsub/mod_pubsub.lua @@ -99,7 +99,7 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj) end local max_max_items = module:get_option_number("pubsub_max_items", 256); -function check_node_config(node, actor, new_config) -- luacheck: ignore 212/actor 212/node +function check_node_config(node, actor, new_config) -- luacheck: ignore 212/node 212/actor if (new_config["max_items"] or 1) > max_max_items then return false; end -- cgit v1.2.3 From d39cf4f42e2ec7afff198437a7ccb82f8b7664ab Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 23 Dec 2018 03:06:35 +0100 Subject: mod_pubsub: Split line in config check to improve readability Also makes it easier to compare with mod_pep --- plugins/mod_pubsub/mod_pubsub.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/mod_pubsub/mod_pubsub.lua b/plugins/mod_pubsub/mod_pubsub.lua index abc4fee8..0036b48f 100644 --- a/plugins/mod_pubsub/mod_pubsub.lua +++ b/plugins/mod_pubsub/mod_pubsub.lua @@ -103,7 +103,8 @@ function check_node_config(node, actor, new_config) -- luacheck: ignore 212/node if (new_config["max_items"] or 1) > max_max_items then return false; end - if new_config["access_model"] ~= "whitelist" and new_config["access_model"] ~= "open" then + if new_config["access_model"] ~= "whitelist" + and new_config["access_model"] ~= "open" then return false; end return true; -- cgit v1.2.3 From f4b4239e3763ac2f27f1f7d0244b5b2c7d666821 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 23 Dec 2018 14:52:52 +0100 Subject: util.table: Add test for pack() --- spec/util_table_spec.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 spec/util_table_spec.lua diff --git a/spec/util_table_spec.lua b/spec/util_table_spec.lua new file mode 100644 index 00000000..97266fe2 --- /dev/null +++ b/spec/util_table_spec.lua @@ -0,0 +1,10 @@ +local u_table = require "util.table"; +describe("util.table", function () + describe("pack()", function () + it("works", function () + assert.same({ "lorem", "ipsum", "dolor", "sit", "amet", n = 5 }, u_table.pack("lorem", "ipsum", "dolor", "sit", "amet")); + end); + end); +end); + + -- cgit v1.2.3 From 0a11d53f81d33de026024e624f062cf73f6fd006 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 23 Dec 2018 15:01:37 +0100 Subject: util.table: Add test for create() --- spec/util_table_spec.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spec/util_table_spec.lua b/spec/util_table_spec.lua index 97266fe2..76f54b69 100644 --- a/spec/util_table_spec.lua +++ b/spec/util_table_spec.lua @@ -1,5 +1,12 @@ local u_table = require "util.table"; describe("util.table", function () + describe("create()", function () + it("works", function () + -- Can't test the allocated sizes of the table, so what you gonna do? + assert.is.table(u_table.create(1,1)); + end); + end); + describe("pack()", function () it("works", function () assert.same({ "lorem", "ipsum", "dolor", "sit", "amet", n = 5 }, u_table.pack("lorem", "ipsum", "dolor", "sit", "amet")); -- cgit v1.2.3 From 321a68c41995de92e3002c48c7df8571343dfd54 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 24 Dec 2018 03:00:27 +0100 Subject: net.adns: Silence individual luacheck warnings instead of ignoring entire file --- .luacheckrc | 1 - net/adns.lua | 16 +++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.luacheckrc b/.luacheckrc index 3e3fb2b5..5035f446 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -132,7 +132,6 @@ if os.getenv("PROSODY_STRICT_LINT") ~= "1" then "fallbacks/bit.lua"; "fallbacks/lxp.lua"; - "net/adns.lua"; "net/cqueues.lua"; "net/dns.lua"; "net/server_select.lua"; diff --git a/net/adns.lua b/net/adns.lua index 560e4b53..4fa01f8a 100644 --- a/net/adns.lua +++ b/net/adns.lua @@ -14,7 +14,7 @@ local log = require "util.logger".init("adns"); local coroutine, tostring, pcall = coroutine, tostring, pcall; local setmetatable = setmetatable; -local function dummy_send(sock, data, i, j) return (j-i)+1; end +local function dummy_send(sock, data, i, j) return (j-i)+1; end -- luacheck: ignore 212 local _ENV = nil; -- luacheck: std none @@ -29,8 +29,7 @@ local function new_async_socket(sock, resolver) local peername = ""; local listener = {}; local handler = {}; - local err; - function listener.onincoming(conn, data) + function listener.onincoming(conn, data) -- luacheck: ignore 212/conn if data then resolver:feed(handler, data); end @@ -46,9 +45,12 @@ local function new_async_socket(sock, resolver) resolver:servfail(conn); -- Let the magic commence end end - handler, err = server.wrapclient(sock, "dns", 53, listener); - if not handler then - return nil, err; + do + local err; + handler, err = server.wrapclient(sock, "dns", 53, listener); + if not handler then + return nil, err; + end end handler.settimeout = function () end @@ -89,7 +91,7 @@ function async_resolver_methods:lookup(handler, qname, qtype, qclass) end)(resolver:peek(qname, qtype, qclass)); end -function query_methods:cancel(call_handler, reason) +function query_methods:cancel(call_handler, reason) -- luacheck: ignore 212/reason log("warn", "Cancelling DNS lookup for %s", tostring(self[4])); self[1].cancel(self[2], self[3], self[4], self[5], call_handler); end -- cgit v1.2.3 From b250ff0d70b614a5ee0025fc68f175428bfca4b5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 28 Dec 2018 20:49:01 +0100 Subject: util.stanza: Require a type attribute for iq stanzas --- spec/util_stanza_spec.lua | 19 +++++++++++++++---- util/stanza.lua | 8 +++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/spec/util_stanza_spec.lua b/spec/util_stanza_spec.lua index 6fbae41a..18e39554 100644 --- a/spec/util_stanza_spec.lua +++ b/spec/util_stanza_spec.lua @@ -95,20 +95,31 @@ describe("util.stanza", function() describe("#iq()", function() it("should create an iq stanza", function() - local i = st.iq({ id = "foo" }); + local i = st.iq({ type = "get", id = "foo" }); assert.are.equal("iq", i.name); assert.are.equal("foo", i.attr.id); + assert.are.equal("get", i.attr.type); end); - it("should reject stanzas with no id", function () + it("should reject stanzas with no attributes", function () assert.has.error_match(function () st.iq(); - end, "id attribute"); + end, "attributes"); + end); + + it("should reject stanzas with no id", function () assert.has.error_match(function () - st.iq({ foo = "bar" }); + st.iq({ type = "get" }); end, "id attribute"); end); + + it("should reject stanzas with no type", function () + assert.has.error_match(function () + st.iq({ id = "foo" }); + end, "type attribute"); + + end); end); describe("#presence()", function () diff --git a/util/stanza.lua b/util/stanza.lua index a90d56b3..e9847ca6 100644 --- a/util/stanza.lua +++ b/util/stanza.lua @@ -423,9 +423,15 @@ local function message(attr, body) end end local function iq(attr) - if not (attr and attr.id) then + if not attr then + error("iq stanzas require id and type attributes"); + end + if not attr.id then error("iq stanzas require an id attribute"); end + if not attr.type then + error("iq stanzas require a type attribute"); + end return new_stanza("iq", attr); end -- cgit v1.2.3 From 5806d67e82705701d6b778d4b6399724a035c528 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 28 Dec 2018 20:51:31 +0100 Subject: core.moduleapi: Add a promise-based API for tracking IQ stanzas (fixes #714) --- core/moduleapi.lua | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/core/moduleapi.lua b/core/moduleapi.lua index d2aa1e8c..f7aa7216 100644 --- a/core/moduleapi.lua +++ b/core/moduleapi.lua @@ -361,6 +361,71 @@ function api:send(stanza, origin) return core_post_stanza(origin or hosts[self.host], stanza); end +function api:send_iq(stanza, origin, timeout) + local iq_cache = self._iq_cache; + if not iq_cache then + iq_cache = require "util.cache".new(256, function (_, iq) + iq.reject("evicted"); + self:unhook(iq.result_event, iq.result_handler); + self:unhook(iq.error_event, iq.error_handler); + end); + self._iq_cache = iq_cache; + end + return require "util.promise".new(function (resolve, reject) + local event_type; + if stanza.attr.from == self.host then + event_type = "host"; + else -- assume bare since we can't hook full jids + event_type = "bare"; + end + local result_event = "iq-result/"..event_type.."/"..stanza.attr.id; + local error_event = "iq-error/"..event_type.."/"..stanza.attr.id; + local cache_key = event_type.."/"..stanza.attr.id; + + local function result_handler(event) + if event.stanza.attr.from == stanza.attr.to then + resolve(event); + return true; + end + end + + local function error_handler(event) + if event.stanza.attr.from == stanza.attr.to then + reject(event); + return true; + end + end + + if iq_cache:get(cache_key) then + error("choose another iq stanza id attribute") + end + + self:hook(result_event, result_handler); + self:hook(error_event, error_handler); + + local timeout_handle = self:add_timer(timeout or 120, function () + reject("timeout"); + self:unhook(result_event, result_handler); + self:unhook(error_event, error_handler); + iq_cache:set(cache_key, nil); + end); + + local ok = iq_cache:set(cache_key, { + reject = reject, resolve = resolve, + timeout_handle = timeout_handle, + result_event = result_event, error_event = error_event, + result_handler = result_handler, error_handler = error_handler; + }); + + if not ok then + reject("cache insertion failure"); + return; + end + + self:send(stanza, origin); + end); +end + function api:broadcast(jids, stanza, iter) for jid in (iter or it.values)(jids) do local new_stanza = st.clone(stanza); -- cgit v1.2.3 From 07ab17120c03070c138137dd30d3fb601716a8bc Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 28 Dec 2018 20:56:01 +0100 Subject: mod_admin_telnet: Invert host existence check Simplifies and reduces indentation --- plugins/mod_admin_telnet.lua | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 63136d63..5ba88b84 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -1067,13 +1067,12 @@ def_env.xmpp = {}; local st = require "util.stanza"; function def_env.xmpp:ping(localhost, remotehost) - if prosody.hosts[localhost] then - module:send(st.iq{ from=localhost, to=remotehost, type="get", id="ping" } - :tag("ping", {xmlns="urn:xmpp:ping"}), prosody.hosts[localhost]); - return true, "Sent ping"; - else + if not prosody.hosts[localhost] then return nil, "No such host"; end + module:send(st.iq{ from=localhost, to=remotehost, type="get", id="ping" } + :tag("ping", {xmlns="urn:xmpp:ping"}), prosody.hosts[localhost]); + return true, "Sent ping"; end def_env.dns = {}; -- cgit v1.2.3 From b1b7f0261f054bf83a955e073dbdeb9631518097 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 27 Dec 2018 02:53:34 +0100 Subject: mod_admin_telnet: Enable async processing using util.async --- plugins/mod_admin_telnet.lua | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 5ba88b84..bb97a09b 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -31,6 +31,7 @@ local cert_verify_identity = require "util.x509".verify_identity; local envload = require "util.envload".envload; local envloadfile = require "util.envload".envloadfile; local has_pposix, pposix = pcall(require, "util.pposix"); +local async = require "util.async"; local commands = module:shared("commands") local def_env = module:shared("env"); @@ -48,6 +49,21 @@ end console = {}; +local runner_callbacks = {}; + +function runner_callbacks:ready() + self.data.conn:resume(); +end + +function runner_callbacks:waiting() + self.data.conn:pause(); +end + +function runner_callbacks:error(err) + module:log("error", "Traceback[telnet]: %s", err); +end + + function console:new_session(conn) local w = function(s) conn:write(s:gsub("\n", "\r\n")); end; local session = { conn = conn; @@ -63,6 +79,11 @@ function console:new_session(conn) }; session.env = setmetatable({}, default_env_mt); + session.thread = async.runner(function (line) + console:process_line(session, line); + session.send(string.char(0)); + end, runner_callbacks, session); + -- Load up environment with helper objects for name, t in pairs(def_env) do if type(t) == "table" then @@ -151,8 +172,7 @@ function console_listener.onincoming(conn, data) for line in data:gmatch("[^\n]*[\n\004]") do if session.closed then return end - console:process_line(session, line); - session.send(string.char(0)); + session.thread:run(line); end session.partial_data = data:match("[^\n]+$"); end -- cgit v1.2.3 From 01878444937fe8b7ea149ef35cd76453517bf798 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 28 Dec 2018 20:59:10 +0100 Subject: mod_admin_telnet: Make xmpp:ping command wait and report the reply --- plugins/mod_admin_telnet.lua | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index bb97a09b..ee6a4176 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -1086,13 +1086,28 @@ end def_env.xmpp = {}; local st = require "util.stanza"; -function def_env.xmpp:ping(localhost, remotehost) +local new_id = require "util.id".medium; +function def_env.xmpp:ping(localhost, remotehost, timeout) if not prosody.hosts[localhost] then return nil, "No such host"; end - module:send(st.iq{ from=localhost, to=remotehost, type="get", id="ping" } - :tag("ping", {xmlns="urn:xmpp:ping"}), prosody.hosts[localhost]); - return true, "Sent ping"; + local iq = st.iq{ from=localhost, to=remotehost, type="get", id=new_id()} + :tag("ping", {xmlns="urn:xmpp:ping"}); + local ret, err; + local wait, done = async.waiter(); + module:context(localhost):send_iq(iq, nil, timeout) + :next(function (ret_) ret = ret_; end, + function (err_) err = err_; end) + :finally(done); + wait(); + if ret then + return true, "pong from " .. ret.stanza.attr.from; + elseif type(err) == "table" and st.is_stanza(err.stanza) then + local t, cond, text = err.stanza:get_error(); + return false, text or cond or t; + else + return false, tostring(err); + end end def_env.dns = {}; -- cgit v1.2.3 From ba5a1d71a0b8b34e04d1684b2037306d6131d526 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 10 Mar 2018 19:58:41 +0100 Subject: spec: Stub tests for util.interpolation --- spec/util_interpolation_spec.lua | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 spec/util_interpolation_spec.lua diff --git a/spec/util_interpolation_spec.lua b/spec/util_interpolation_spec.lua new file mode 100644 index 00000000..88d9f844 --- /dev/null +++ b/spec/util_interpolation_spec.lua @@ -0,0 +1,17 @@ +local template = [[ +{greet!}, {name?world}! +]]; +local expect1 = [[ +Hello, WORLD! +]]; +local expect2 = [[ +Hello, world! +]]; + +describe("util.interpolation", function () + it("renders", function () + local render = require "util.interpolation".new("%b{}", string.upper); + assert.equal(expect1, render(template, { greet = "Hello", name = "world" })); + assert.equal(expect2, render(template, { greet = "Hello" })); + end); +end); -- cgit v1.2.3 From f8bc92174749c64b3804b8ad46f47dfceb107312 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 24 Nov 2018 02:24:48 +0100 Subject: mod_saslauth: Improve log message when no SASL mechanisms offered (thanks hexa) --- plugins/mod_saslauth.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/mod_saslauth.lua b/plugins/mod_saslauth.lua index fba84ef8..ba30b9e6 100644 --- a/plugins/mod_saslauth.lua +++ b/plugins/mod_saslauth.lua @@ -275,7 +275,8 @@ module:hook("stream-features", function(event) if mechanisms[1] then features:add_child(mechanisms); elseif not next(sasl_mechanisms) then - log("warn", "No available SASL mechanisms, verify that the configured authentication module is working"); + local authmod = module:get_option_string("authentication", "internal_plain"); + log("error", "No available SASL mechanisms, verify that the configured authentication module '%s' is loaded and configured correctly", authmod); else log("warn", "All available authentication mechanisms are either disabled or not suitable for an insecure connection"); end -- cgit v1.2.3 From da5ec59b36188e325ed6dc9fa6ebf8187d492b33 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 28 Dec 2018 00:13:03 +0100 Subject: mod_c2s: Improve log message in case there are no stream features on offer (thanks hexa) --- plugins/mod_c2s.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/mod_c2s.lua b/plugins/mod_c2s.lua index 8e31a968..36e6a152 100644 --- a/plugins/mod_c2s.lua +++ b/plugins/mod_c2s.lua @@ -106,7 +106,13 @@ function stream_callbacks.streamopened(session, attr) if features.tags[1] or session.full_jid then send(features); else - (session.log or log)("warn", "No stream features to offer"); + if session.secure then + -- Normally STARTTLS would be offered + (session.log or log)("warn", "No stream features to offer on secure session. Check authentication settings."); + else + -- Here SASL should be offered + (session.log or log)("warn", "No stream features to offer on insecure session. Check encryption and security settings."); + end session:close{ condition = "undefined-condition", text = "No stream features to proceed with" }; end end -- cgit v1.2.3 From 3f9505a0d5ce144beb79748bc783e06862693129 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 28 Dec 2018 00:04:26 +0100 Subject: mod_tls: Keep TLS context errors and repeat them again for each session --- plugins/mod_tls.lua | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/plugins/mod_tls.lua b/plugins/mod_tls.lua index 029ddd1d..4ead60dc 100644 --- a/plugins/mod_tls.lua +++ b/plugins/mod_tls.lua @@ -35,9 +35,10 @@ local host = hosts[module.host]; local ssl_ctx_c2s, ssl_ctx_s2sout, ssl_ctx_s2sin; local ssl_cfg_c2s, ssl_cfg_s2sout, ssl_cfg_s2sin; +local err_c2s, err_s2sin, err_s2sout; function module.load() - local NULL, err = {}; + local NULL = {}; local modhost = module.host; local parent = modhost:match("%.(.*)$"); @@ -52,14 +53,14 @@ function module.load() local parent_s2s = rawgetopt(parent, "s2s_ssl") or NULL; local host_s2s = rawgetopt(modhost, "s2s_ssl") or parent_s2s; - ssl_ctx_c2s, err, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections - if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err); end + ssl_ctx_c2s, err_c2s, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections + if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err_c2s); end - ssl_ctx_s2sout, err, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s); -- for outgoing server connections - if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err); end + ssl_ctx_s2sout, err_s2sout, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s); -- for outgoing server connections + if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err_s2sout); end - ssl_ctx_s2sin, err, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s); -- for incoming server connections - if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err); end + ssl_ctx_s2sin, err_s2sin, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s); -- for incoming server connections + if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err_s2sin); end end module:hook_global("config-reloaded", module.load); @@ -74,12 +75,21 @@ local function can_do_tls(session) return session.ssl_ctx; end if session.type == "c2s_unauthed" then + if not ssl_ctx_c2s and c2s_require_encryption then + session.log("error", "No TLS context available for c2s. Earlier error was: %s", err_c2s); + end session.ssl_ctx = ssl_ctx_c2s; session.ssl_cfg = ssl_cfg_c2s; elseif session.type == "s2sin_unauthed" and allow_s2s_tls then + if not ssl_ctx_s2sin and s2s_require_encryption then + session.log("error", "No TLS context available for s2sin. Earlier error was: %s", err_s2sin); + end session.ssl_ctx = ssl_ctx_s2sin; session.ssl_cfg = ssl_cfg_s2sin; elseif session.direction == "outgoing" and allow_s2s_tls then + if not ssl_ctx_s2sout and s2s_require_encryption then + session.log("error", "No TLS context available for s2sout. Earlier error was: %s", err_s2sout); + end session.ssl_ctx = ssl_ctx_s2sout; session.ssl_cfg = ssl_cfg_s2sout; else -- cgit v1.2.3 From 708dc874279d00235cb729b61f7438e753bd8008 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 29 Dec 2018 03:21:13 +0100 Subject: mod_admin_telnet: Validate hostnames in xmpp:ping command Attempt to ping some invalid hostnames cause weird behavior --- plugins/mod_admin_telnet.lua | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index ee6a4176..f3731c8a 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -1088,8 +1088,17 @@ def_env.xmpp = {}; local st = require "util.stanza"; local new_id = require "util.id".medium; function def_env.xmpp:ping(localhost, remotehost, timeout) - if not prosody.hosts[localhost] then - return nil, "No such host"; + localhost = select(2, jid_split(localhost)); + remotehost = select(2, jid_split(remotehost)); + if not localhost then + return nil, "Invalid sender hostname"; + elseif not prosody.hosts[localhost] then + return nil, "No such local host"; + end + if not remotehost then + return nil, "Invalid destination hostname"; + elseif prosody.hosts[remotehost] then + return nil, "Both hosts are local"; end local iq = st.iq{ from=localhost, to=remotehost, type="get", id=new_id()} :tag("ping", {xmlns="urn:xmpp:ping"}); -- cgit v1.2.3 From 26e3090de67f9fc5152c764122b7a62d3d9534ac Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 30 Dec 2018 03:20:37 +0100 Subject: luacheckrc: Teach luacheck about the new module:send_iq() API --- .luacheckrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.luacheckrc b/.luacheckrc index 5035f446..b2fa7cdb 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -80,6 +80,7 @@ files["plugins/"] = { "module.remove_item", "module.require", "module.send", + "module.send_iq", "module.set_global", "module.shared", "module.unhook", -- cgit v1.2.3 From 93c5ebb743337c82103ea94166065106a9f5b641 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 30 Dec 2018 03:24:54 +0100 Subject: util.promise: Remove references to callbacks after settling promise This is to help the garbage collector. --- util/promise.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/promise.lua b/util/promise.lua index 07c9c4dc..0b182b54 100644 --- a/util/promise.lua +++ b/util/promise.lua @@ -49,6 +49,9 @@ local function promise_settle(promise, new_state, new_next, cbs, value) for _, cb in ipairs(cbs) do cb(value); end + -- No need to keep references to callbacks + promise._pending_on_fulfilled = nil; + promise._pending_on_rejected = nil; return true; end -- cgit v1.2.3 From 38462e35420ca9e93af018210be940353b32d508 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Sun, 30 Dec 2018 12:55:58 +0000 Subject: util.error: Add new util library for structured errors --- util/error.lua | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 util/error.lua diff --git a/util/error.lua b/util/error.lua new file mode 100644 index 00000000..ed61793f --- /dev/null +++ b/util/error.lua @@ -0,0 +1,40 @@ +local error_mt = { __name = "error" }; + +function error_mt:__tostring() + return ("error<%s:%s:%s>"):format(self.type, self.condition, self.text); +end + +local function is_err(e) + return getmetatable(e) == error_mt; +end + +local function new(e, context, registry) + local template = (registry and registry[e]) or e or {}; + return setmetatable({ + type = template.type or "cancel"; + condition = template.condition or "undefined-condition"; + text = template.text; + + context = context or template.context or { _error_id = e }; + }, error_mt); +end + +local function coerce(ok, err, ...) + if ok or is_err(err) then + return ok, err, ...; + end + + local new_err = setmetatable({ + native = err; + + type = "cancel"; + condition = "undefined-condition"; + }, error_mt); + return ok, new_err, ...; +end + +return { + new = new; + coerce = coerce; + is_err = is_err; +} -- cgit v1.2.3 From 7799a8a7bf9f29c6d34bdcd3ded42fc6387c0fe5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 30 Dec 2018 14:26:58 +0100 Subject: core.moduleapi: Move util imports to top --- core/moduleapi.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/moduleapi.lua b/core/moduleapi.lua index f7aa7216..c7fff11f 100644 --- a/core/moduleapi.lua +++ b/core/moduleapi.lua @@ -14,6 +14,8 @@ local pluginloader = require "util.pluginloader"; local timer = require "util.timer"; local resolve_relative_path = require"util.paths".resolve_relative_path; local st = require "util.stanza"; +local cache = require "util.cache"; +local promise = require "util.promise"; local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat; local error, setmetatable, type = error, setmetatable, type; @@ -364,14 +366,14 @@ end function api:send_iq(stanza, origin, timeout) local iq_cache = self._iq_cache; if not iq_cache then - iq_cache = require "util.cache".new(256, function (_, iq) + iq_cache = cache.new(256, function (_, iq) iq.reject("evicted"); self:unhook(iq.result_event, iq.result_handler); self:unhook(iq.error_event, iq.error_handler); end); self._iq_cache = iq_cache; end - return require "util.promise".new(function (resolve, reject) + return promise.new(function (resolve, reject) local event_type; if stanza.attr.from == self.host then event_type = "host"; -- cgit v1.2.3 From ad3e5fd42bd1dddd478fffd412a842e98c735e9a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 30 Dec 2018 16:03:15 +0100 Subject: core.moduleapi: Use util.error for :send_iq errors --- core/moduleapi.lua | 26 +++++++++++++++++++++----- plugins/mod_admin_telnet.lua | 3 --- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/core/moduleapi.lua b/core/moduleapi.lua index c7fff11f..57aa4e9f 100644 --- a/core/moduleapi.lua +++ b/core/moduleapi.lua @@ -15,6 +15,7 @@ local timer = require "util.timer"; local resolve_relative_path = require"util.paths".resolve_relative_path; local st = require "util.stanza"; local cache = require "util.cache"; +local errutil = require "util.error"; local promise = require "util.promise"; local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat; @@ -367,7 +368,10 @@ function api:send_iq(stanza, origin, timeout) local iq_cache = self._iq_cache; if not iq_cache then iq_cache = cache.new(256, function (_, iq) - iq.reject("evicted"); + iq.reject(errutil.new({ + type = "wait", condition = "resource-constraint", + text = "evicted from iq tracking cache" + })); self:unhook(iq.result_event, iq.result_handler); self:unhook(iq.error_event, iq.error_handler); end); @@ -393,20 +397,29 @@ function api:send_iq(stanza, origin, timeout) local function error_handler(event) if event.stanza.attr.from == stanza.attr.to then - reject(event); + local error_type, condition, text = event.stanza:get_error(); + local err = errutil.new({ type = error_type, condition = condition, text = text }, event); + reject(err); return true; end end if iq_cache:get(cache_key) then - error("choose another iq stanza id attribute") + reject(errutil.new({ + type = "modify", condition = "conflict", + text = "iq stanza id attribute already used", + })); + return; end self:hook(result_event, result_handler); self:hook(error_event, error_handler); local timeout_handle = self:add_timer(timeout or 120, function () - reject("timeout"); + reject(errutil.new({ + type = "wait", condition = "remote-server-timeout", + text = "IQ stanza timed out", + })); self:unhook(result_event, result_handler); self:unhook(error_event, error_handler); iq_cache:set(cache_key, nil); @@ -420,7 +433,10 @@ function api:send_iq(stanza, origin, timeout) }); if not ok then - reject("cache insertion failure"); + reject(errutil.new({ + type = "wait", condition = "internal-server-error", + text = "Could not store IQ tracking data" + })); return; end diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index f3731c8a..5bb9361e 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -1111,9 +1111,6 @@ function def_env.xmpp:ping(localhost, remotehost, timeout) wait(); if ret then return true, "pong from " .. ret.stanza.attr.from; - elseif type(err) == "table" and st.is_stanza(err.stanza) then - local t, cond, text = err.stanza:get_error(); - return false, text or cond or t; else return false, tostring(err); end -- cgit v1.2.3 From ac2d2297b62ba2973c49d7d5bee3927cef64ea97 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 30 Dec 2018 20:30:59 +0100 Subject: util.error: Add a function for creating an error object from an error stanza --- util/error.lua | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/util/error.lua b/util/error.lua index ed61793f..344dd274 100644 --- a/util/error.lua +++ b/util/error.lua @@ -33,8 +33,20 @@ local function coerce(ok, err, ...) return ok, new_err, ...; end +local function from_stanza(stanza, context) + local error_type, condition, text = stanza:get_error(); + return setmetatable({ + type = error_type or "cancel"; + condition = condition or "undefined-condition"; + text = text; + + context = context or { stanza = stanza }; + }, error_mt); +end + return { new = new; coerce = coerce; is_err = is_err; + from_stanza = from_stanza; } -- cgit v1.2.3 From a8d08ff3ba20bd2a64d8f59930019d460488c6e5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 30 Dec 2018 20:35:20 +0100 Subject: core.moduleapi: Use convenience function for creating error object from stanza --- core/moduleapi.lua | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/moduleapi.lua b/core/moduleapi.lua index 57aa4e9f..c6193cfd 100644 --- a/core/moduleapi.lua +++ b/core/moduleapi.lua @@ -397,9 +397,7 @@ function api:send_iq(stanza, origin, timeout) local function error_handler(event) if event.stanza.attr.from == stanza.attr.to then - local error_type, condition, text = event.stanza:get_error(); - local err = errutil.new({ type = error_type, condition = condition, text = text }, event); - reject(err); + reject(errutil.from_stanza(event.stanza), event); return true; end end -- cgit v1.2.3 From e4087a66e42e86ea4aae8d6cba1bc7dbb66ecb87 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 3 Jan 2019 17:25:43 +0100 Subject: mod_mam: Perform message expiry based on building an index by date For each day, store a set of users that have new messages. To expire messages, we collect the union of sets of users from dates that fall outside the cleanup range. The previous algoritm did not work well with many users, especially with the default settings. --- plugins/mod_mam/mod_mam.lua | 70 +++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 94bedbb1..18f84752 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -33,7 +33,7 @@ local is_stanza = st.is_stanza; local tostring = tostring; local time_now = os.time; local m_min = math.min; -local timestamp, timestamp_parse = require "util.datetime".datetime, require "util.datetime".parse; +local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date"); local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50); local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://jabber.org/protocol/chatstates" }); @@ -46,13 +46,8 @@ if not archive.find then end local use_total = module:get_option_boolean("mam_include_total", true); -local cleanup; - -local function schedule_cleanup(username) - if cleanup and not cleanup[username] then - table.insert(cleanup, username); - cleanup[username] = true; - end +function schedule_cleanup() + -- replaced by non-noop later if cleanup is enabled end -- Handle prefs. @@ -96,7 +91,6 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event) local qid = query.attr.queryid; get_prefs(origin.username, true); - schedule_cleanup(origin.username); -- Search query parameters local qwith, qstart, qend; @@ -212,6 +206,7 @@ end local function shall_store(user, who) -- TODO Cache this? if not um.user_exists(user, host) then + module:log("debug", "%s@%s does not exist", user, host) return false; end local prefs = get_prefs(user); @@ -329,6 +324,9 @@ module:hook("pre-message/full", strip_stanza_id_after_other_events, -1); local cleanup_after = module:get_option_string("archive_expires_after", "1w"); local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60); if cleanup_after ~= "never" then + local cleanup_storage = module:open_store("archive_cleanup"); + local cleanup_map = module:open_store("archive_cleanup", "map"); + local day = 86400; local multipliers = { d = day, w = day * 7, m = 31 * day, y = 365.2425 * day }; local n, m = cleanup_after:lower():match("(%d+)%s*([dwmy]?)"); @@ -346,33 +344,43 @@ if cleanup_after ~= "never" then return false; end - -- Set of known users to do message expiry for - -- Populated either below or when new messages are added - cleanup = {}; + -- For each day, store a set of users that have new messages. To expire + -- messages, we collect the union of sets of users from dates that fall + -- outside the cleanup range. - -- Iterating over users is not supported by all authentication modules - -- Catch and ignore error if not supported - pcall(function () - -- If this works, then we schedule cleanup for all known users on startup - for user in um.users(module.host) do - schedule_cleanup(user); - end - end); + function schedule_cleanup(username, date) + cleanup_map:set(date or datestamp(), username, true); + end - -- At odd intervals, delete old messages for one user - module:add_timer(math.random(10, 60), function() - local user = table.remove(cleanup, 1); - if user then - module:log("debug", "Removing old messages for user %q", user); + cleanup_runner = require "util.async".runner(function () + local users = {}; + local cut_off = datestamp(os.time() - cleanup_after); + for date in cleanup_storage:users() do + if date < cut_off then + module:log("debug", "Messages from %q should be expired", date); + local messages_this_day = cleanup_storage:get(date); + if messages_this_day then + for user in pairs(messages_this_day) do + users[user] = true; + end + cleanup_storage:set(date, nil); + end + end + end + local sum, num_users = 0, 0; + for user in pairs(users) do local ok, err = archive:delete(user, { ["end"] = os.time() - cleanup_after; }) - if not ok then - module:log("warn", "Could not expire archives for user %s: %s", user, err); - elseif type(ok) == "number" then - module:log("debug", "Removed %d messages", ok); + if ok then + num_users = num_users + 1; + sum = sum + tonumber(ok) or 0; end - cleanup[user] = nil; end - return math.random(cleanup_interval, cleanup_interval * 2); + module:log("info", "Deleted expired %d messages for %d users", sum, num_users); + end); + + cleanup_task = module:add_timer(1, function () + cleanup_runner:run(true); + return cleanup_interval; end); else module:log("debug", "Archive expiry disabled"); -- cgit v1.2.3 From 76f85eb776b39feeb37c8dd276fcd9eaf6c722c7 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 4 Jan 2019 10:14:55 +0100 Subject: mod_mam: Fix word order in log message --- plugins/mod_mam/mod_mam.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 18f84752..35a4b9a0 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -375,7 +375,7 @@ if cleanup_after ~= "never" then sum = sum + tonumber(ok) or 0; end end - module:log("info", "Deleted expired %d messages for %d users", sum, num_users); + module:log("info", "Deleted %d expired messages for %d users", sum, num_users); end); cleanup_task = module:add_timer(1, function () -- cgit v1.2.3 From 1271c21c89d544ad3d34526f9317c952092c0658 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 4 Jan 2019 13:38:30 +0100 Subject: mod_admin_telnet: Remove the long gone 'section' argument in the undocumented config:get command --- plugins/mod_admin_telnet.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 5bb9361e..4c049b95 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -495,9 +495,9 @@ function def_env.config:load(filename, format) return true, "Config loaded"; end -function def_env.config:get(host, section, key) +function def_env.config:get(host, key) local config_get = require "core.configmanager".get - return true, tostring(config_get(host, section, key)); + return true, tostring(config_get(host, key)); end function def_env.config:reload() -- cgit v1.2.3 From ce8379079e7e0fae9b292a7a8ce6ddd91a4f6e46 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 4 Jan 2019 13:39:13 +0100 Subject: mod_admin_telnet: config:get: Assume the global section if only one argument is given --- plugins/mod_admin_telnet.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 4c049b95..c6b67b95 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -496,6 +496,9 @@ function def_env.config:load(filename, format) end function def_env.config:get(host, key) + if key == nil then + host, key = "*", host; + end local config_get = require "core.configmanager".get return true, tostring(config_get(host, key)); end -- cgit v1.2.3 From f902e414f63ae70f74c414a57c0c3cfe26ba693d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 4 Jan 2019 13:41:39 +0100 Subject: mod_admin_telnet: Serialize config values (table: 0x123abc isn't useful) --- plugins/mod_admin_telnet.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index c6b67b95..cd9f8078 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -32,6 +32,7 @@ local envload = require "util.envload".envload; local envloadfile = require "util.envload".envloadfile; local has_pposix, pposix = pcall(require, "util.pposix"); local async = require "util.async"; +local serialize = require "util.serialization".new({ fatal = false, unquoted = true}); local commands = module:shared("commands") local def_env = module:shared("env"); @@ -500,7 +501,7 @@ function def_env.config:get(host, key) host, key = "*", host; end local config_get = require "core.configmanager".get - return true, tostring(config_get(host, key)); + return true, serialize(config_get(host, key)); end function def_env.config:reload() -- cgit v1.2.3 From 9e01c3ff642670c8b10a20686f7a6e100ad08fe9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 4 Jan 2019 15:13:52 +0100 Subject: mod_admin_telnet: Sort stats by name --- plugins/mod_admin_telnet.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index cd9f8078..5ce504f8 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -1543,7 +1543,7 @@ function def_env.stats:show(filter) local stats, changed, extra = require "core.statsmanager".get_stats(); local available, displayed = 0, 0; local displayed_stats = new_stats_context(self); - for name, value in pairs(stats) do + for name, value in iterators.sorted_pairs(stats) do available = available + 1; if not filter or name:match(filter) then displayed = displayed + 1; -- cgit v1.2.3 From fa09b8372d173408df3cb468d9e1b5098c783f31 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 6 Jan 2019 09:34:59 +0100 Subject: mod_mam: Measure how long it takes to run the message expiry job job --- plugins/mod_mam/mod_mam.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 35a4b9a0..5ba04d68 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -351,8 +351,10 @@ if cleanup_after ~= "never" then function schedule_cleanup(username, date) cleanup_map:set(date or datestamp(), username, true); end + local cleanup_time = module:measure("cleanup", "times"); cleanup_runner = require "util.async".runner(function () + local cleanup_done = cleanup_time(); local users = {}; local cut_off = datestamp(os.time() - cleanup_after); for date in cleanup_storage:users() do @@ -376,6 +378,7 @@ if cleanup_after ~= "never" then end end module:log("info", "Deleted %d expired messages for %d users", sum, num_users); + cleanup_done(); end); cleanup_task = module:add_timer(1, function () -- cgit v1.2.3 From d9e14b0e994443bb64eaae05b080a44929b8d03d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 6 Jan 2019 09:44:55 +0100 Subject: mod_mam: Handle expiry of messages that expire in the middle of the cut-off day --- plugins/mod_mam/mod_mam.lua | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 5ba04d68..d2ca709b 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -358,14 +358,18 @@ if cleanup_after ~= "never" then local users = {}; local cut_off = datestamp(os.time() - cleanup_after); for date in cleanup_storage:users() do - if date < cut_off then + if date <= cut_off then module:log("debug", "Messages from %q should be expired", date); local messages_this_day = cleanup_storage:get(date); if messages_this_day then for user in pairs(messages_this_day) do users[user] = true; end - cleanup_storage:set(date, nil); + if date < cut_off then + -- Messages from the same day as the cut-off might not have expired yet, + -- but all earlier will have, so clear storage for those days. + cleanup_storage:set(date, nil); + end end end end -- cgit v1.2.3 From 60213520c2a27bf0472a0ef5a2cf16640134a1a8 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 6 Jan 2019 10:39:33 +0100 Subject: util.http: Pre-generate urlencoding mappings (optimization) Function calls are more expensive than table lookups --- util/http.lua | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/util/http.lua b/util/http.lua index cfb89193..1730d4d4 100644 --- a/util/http.lua +++ b/util/http.lua @@ -6,24 +6,25 @@ -- local format, char = string.format, string.char; -local pairs, ipairs, tonumber = pairs, ipairs, tonumber; +local pairs, ipairs = pairs, ipairs; local t_insert, t_concat = table.insert, table.concat; +local url_codes = {}; +for i = 0, 255 do + local c = char(i); + local u = format("%%%02x", i); + url_codes[c] = u; + url_codes[u] = c; +end local function urlencode(s) - return s and (s:gsub("[^a-zA-Z0-9.~_-]", function (c) return format("%%%02x", c:byte()); end)); + return s and (s:gsub("[^a-zA-Z0-9.~_-]", url_codes)); end local function urldecode(s) - return s and (s:gsub("%%(%x%x)", function (c) return char(tonumber(c,16)); end)); + return s and (s:gsub("%%%x%x", url_codes)); end local function _formencodepart(s) - return s and (s:gsub("%W", function (c) - if c ~= " " then - return format("%%%02x", c:byte()); - else - return "+"; - end - end)); + return s and (urlencode(s):gsub("%%20", "+")); end local function formencode(form) -- cgit v1.2.3 From 17be1c595df0e62ff28d9e115b83ace74c800db0 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 6 Jan 2019 10:42:45 +0100 Subject: mod_http_errors: Normalize CSS --- plugins/mod_http_errors.lua | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/plugins/mod_http_errors.lua b/plugins/mod_http_errors.lua index 13473219..2bb13298 100644 --- a/plugins/mod_http_errors.lua +++ b/plugins/mod_http_errors.lua @@ -26,21 +26,24 @@ local html = [[ {title} -- cgit v1.2.3 From f61b36e2d32c4e5ea7ed7eb3d800bcc7ef5afee2 Mon Sep 17 00:00:00 2001 From: Jonas Wielicki Date: Sun, 6 Jan 2019 11:28:54 +0100 Subject: MUC: add ID to message if no ID is present --- plugins/muc/muc.lib.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua index bb79cda6..a34e912b 100644 --- a/plugins/muc/muc.lib.lua +++ b/plugins/muc/muc.lib.lua @@ -23,6 +23,7 @@ local resourceprep = require "util.encodings".stringprep.resourceprep; local st = require "util.stanza"; local base64 = require "util.encodings".base64; local md5 = require "util.hashes".md5; +local id = require "util.id"; local log = module._log; @@ -1037,6 +1038,9 @@ end function room_mt:handle_groupchat_to_room(origin, stanza) local from = stanza.attr.from; local occupant = self:get_occupant_by_real_jid(from); + if not stanza.attr.id then + stanza.attr.id = id.medium() + end if module:fire_event("muc-occupant-groupchat", { room = self; origin = origin; stanza = stanza; from = from; occupant = occupant; }) then return true; end -- cgit v1.2.3 From 12cea7094d339f68a661d00f096e8db9a51417f5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 6 Jan 2019 12:20:57 +0100 Subject: MUC: Rename import to avoid name clash [luacheck] --- plugins/muc/muc.lib.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua index a34e912b..7259dde2 100644 --- a/plugins/muc/muc.lib.lua +++ b/plugins/muc/muc.lib.lua @@ -23,7 +23,7 @@ local resourceprep = require "util.encodings".stringprep.resourceprep; local st = require "util.stanza"; local base64 = require "util.encodings".base64; local md5 = require "util.hashes".md5; -local id = require "util.id"; +local new_id = require "util.id".medium; local log = module._log; @@ -1039,7 +1039,7 @@ function room_mt:handle_groupchat_to_room(origin, stanza) local from = stanza.attr.from; local occupant = self:get_occupant_by_real_jid(from); if not stanza.attr.id then - stanza.attr.id = id.medium() + stanza.attr.id = new_id() end if module:fire_event("muc-occupant-groupchat", { room = self; origin = origin; stanza = stanza; from = from; occupant = occupant; -- cgit v1.2.3 From d9677a50b51b8e40ef6cac11052f7a2477dff51e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 10 Jan 2019 14:27:01 +0100 Subject: GNUmakefile: Add target for running scansion --- GNUmakefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/GNUmakefile b/GNUmakefile index 6c134679..dd199997 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -21,6 +21,7 @@ MKDIR_PRIVATE=$(MKDIR) -m750 LUACHECK=luacheck BUSTED=busted +SCANSION=scansion .PHONY: all test coverage clean install @@ -71,6 +72,11 @@ clean: test: $(BUSTED) --lua=$(RUNWITH) +integration-test: all + $(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua start + $(SCANSION) -d ./spec/scansion + $(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua stop + coverage: -rm -- luacov.* $(BUSTED) --lua=$(RUNWITH) -c -- cgit v1.2.3 From caecb38b5b01262452ac26e15aece8a9c1870518 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 10 Jan 2019 14:54:34 +0100 Subject: prosodyctl: Pass the original argv table to subcommands (with first argument removed) This preserves eg arg[-1] where you might find the path to the Lua executable, which can be useful. --- prosodyctl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prosodyctl b/prosodyctl index 76de45a2..ecf00baa 100755 --- a/prosodyctl +++ b/prosodyctl @@ -83,7 +83,7 @@ local jid_split = require "util.jid".prepped_split; local prosodyctl_timeout = (configmanager.get("*", "prosodyctl_timeout") or 5) * 2; ----------------------- local commands = {}; -local command = arg[1]; +local command = table.remove(arg, 1); function commands.adduser(arg) if not arg[1] or arg[1] == "--help" then @@ -1365,7 +1365,7 @@ local command_runner = async.runner(function () os.exit(0); end - os.exit(commands[command]({ select(2, unpack(arg)) })); + os.exit(commands[command](arg)); end, watchers); command_runner:run(true); -- cgit v1.2.3 From 1a3edc0ab921f131c7304e92d7402ef88ff465ef Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 10 Jan 2019 14:57:26 +0100 Subject: util.prosodyctl: Allow passing path to Lua runtime to the start() function By default the shebang is used. Being able to override it is useful in cases where the shebang does not match the configured runtime. --- util/prosodyctl.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/util/prosodyctl.lua b/util/prosodyctl.lua index 5f0c4d12..9b627bde 100644 --- a/util/prosodyctl.lua +++ b/util/prosodyctl.lua @@ -229,7 +229,8 @@ local function isrunning() return true, signal.kill(pid, 0) == 0; end -local function start(source_dir) +local function start(source_dir, lua) + lua = lua and lua .. " " or ""; local ok, ret = isrunning(); if not ok then return ok, ret; @@ -238,9 +239,9 @@ local function start(source_dir) return false, "already-running"; end if not source_dir then - os.execute("./prosody"); + os.execute(lua .. "./prosody"); else - os.execute(source_dir.."/../../bin/prosody"); + os.execute(lua .. source_dir.."/../../bin/prosody"); end return true; end -- cgit v1.2.3 From 08d0a26c8c614aeafbaa990f55b3d3372d88dbb5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 10 Jan 2019 15:25:38 +0100 Subject: prosodyctl: Use the same runtime for starting prosody Improves the experience with the `make integration-test` command --- prosodyctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prosodyctl b/prosodyctl index ecf00baa..0ff03a7b 100755 --- a/prosodyctl +++ b/prosodyctl @@ -222,7 +222,7 @@ function commands.start(arg) end --luacheck: ignore 411/ret - local ok, ret = prosodyctl.start(prosody.paths.source); + local ok, ret = prosodyctl.start(prosody.paths.source, arg[-1]); if ok then local daemonize = configmanager.get("*", "daemonize"); if daemonize == nil then -- cgit v1.2.3 From 4a1dec0392e8a0a076c91ce6ff47f938ee9b2641 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 14 Jan 2019 00:17:02 +0100 Subject: mod_storage_memory: Implement :user iteration API --- plugins/mod_storage_memory.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua index ed04a5fb..3a9de1cc 100644 --- a/plugins/mod_storage_memory.lua +++ b/plugins/mod_storage_memory.lua @@ -22,6 +22,10 @@ local function _purge_store(self, username) return true; end +local function _users(self) + return next, self.store, nil; +end + local keyval_store = {}; keyval_store.__index = keyval_store; @@ -39,9 +43,13 @@ end keyval_store.purge = _purge_store; +keyval_store.users = _users; + local archive_store = {}; archive_store.__index = archive_store; +archive_store.users = _users; + function archive_store:append(username, key, value, when, with) if is_stanza(value) then value = st.preserialize(value); -- cgit v1.2.3 From fe9e33d8c22b774d5a1f4cbc0b1145f54201c096 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 15 Jan 2019 20:08:30 +0100 Subject: mod_c2s, mod_s2s, mod_component: Log invalid XML escaped (fixes #734) See 6ed0d6224d64 --- plugins/mod_c2s.lua | 2 +- plugins/mod_component.lua | 2 +- plugins/mod_s2s/mod_s2s.lua | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/mod_c2s.lua b/plugins/mod_c2s.lua index 36e6a152..8d7b92fe 100644 --- a/plugins/mod_c2s.lua +++ b/plugins/mod_c2s.lua @@ -290,7 +290,7 @@ function listener.onconnect(conn) if data then local ok, err = stream:feed(data); if not ok then - log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_")); + log("debug", "Received invalid XML (%s) %d bytes: %q", tostring(err), #data, data:sub(1, 300)); session:close("not-well-formed"); end end diff --git a/plugins/mod_component.lua b/plugins/mod_component.lua index b41204a2..b8c87dee 100644 --- a/plugins/mod_component.lua +++ b/plugins/mod_component.lua @@ -310,7 +310,7 @@ function listener.onconnect(conn) function session.data(_, data) local ok, err = stream:feed(data); if ok then return; end - module:log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_")); + log("debug", "Received invalid XML (%s) %d bytes: %q", tostring(err), #data, data:sub(1, 300)); session:close("not-well-formed"); end diff --git a/plugins/mod_s2s/mod_s2s.lua b/plugins/mod_s2s/mod_s2s.lua index aae37b7f..79308847 100644 --- a/plugins/mod_s2s/mod_s2s.lua +++ b/plugins/mod_s2s/mod_s2s.lua @@ -595,8 +595,7 @@ local function initialize_session(session) if data then local ok, err = stream:feed(data); if ok then return; end - log("warn", "Received invalid XML: %s", data); - log("warn", "Problem was: %s", err); + log("debug", "Received invalid XML (%s) %d bytes: %q", tostring(err), #data, data:sub(1, 300)); session:close("not-well-formed"); end end -- cgit v1.2.3 From 7ae9fe492d2e616d94b77c280a2f5e0f2c744075 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 16 Jan 2019 13:53:04 +0100 Subject: util.http: Fix decoding of uppercase URL encoded chars Broken in 1af5106a2c34 --- spec/util_http_spec.lua | 5 +++++ util/http.lua | 1 + 2 files changed, 6 insertions(+) diff --git a/spec/util_http_spec.lua b/spec/util_http_spec.lua index 0f51a86c..d38645f7 100644 --- a/spec/util_http_spec.lua +++ b/spec/util_http_spec.lua @@ -28,6 +28,11 @@ describe("util.http", function() it("should decode important URL characters", function() assert.are.equal("This & that = something", http.urldecode("This%20%26%20that%20%3d%20something"), "Important URL chars escaped"); end); + + it("should decode both lower and uppercase", function () + assert.are.equal("This & that = {something}.", http.urldecode("This%20%26%20that%20%3D%20%7Bsomething%7D%2E"), "Important URL chars escaped"); + end); + end); describe("#formencode()", function() diff --git a/util/http.lua b/util/http.lua index 1730d4d4..3852f91c 100644 --- a/util/http.lua +++ b/util/http.lua @@ -15,6 +15,7 @@ for i = 0, 255 do local u = format("%%%02x", i); url_codes[c] = u; url_codes[u] = c; + url_codes[u:upper()] = c; end local function urlencode(s) return s and (s:gsub("[^a-zA-Z0-9.~_-]", url_codes)); -- cgit v1.2.3 From 0c19a99587bd6f0f590498dd1639c52af662c9ee Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 16 Jan 2019 14:20:16 +0100 Subject: mod_admin_telnet: sttas:show: Use format option that allows float numbers string.format("%d", 0.5) causes an error on Lua 5.3 --- plugins/mod_admin_telnet.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 5ce504f8..7fae8983 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -1255,7 +1255,7 @@ local function format_stat(type, value, ref_value) --do return tostring(value) end if type == "duration" then if ref_value < 0.001 then - return ("%d µs"):format(value*1000000); + return ("%g µs"):format(value*1000000); elseif ref_value < 0.9 then return ("%0.2f ms"):format(value*1000); end -- cgit v1.2.3 From 2d229c341839f408fd64dd02be2524bb7f121eda Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 16 Jan 2019 20:01:38 +0100 Subject: core.s2smanager: Add stub reset_stream method to destroyed sessions Fixes traceback if connection is closed from the 's2s-authenticated' event --- core/s2smanager.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/s2smanager.lua b/core/s2smanager.lua index 58269c49..0ba5e7c6 100644 --- a/core/s2smanager.lua +++ b/core/s2smanager.lua @@ -50,6 +50,9 @@ local resting_session = { -- Resting, not dead close = function (session) session.log("debug", "Attempt to close already-closed session"); end; + reset_stream = function (session) + session.log("debug", "Attempt to reset stream of already-closed session"); + end; filter = function (type, data) return data; end; --luacheck: ignore 212/type }; resting_session.__index = resting_session; -- cgit v1.2.3 From 0cc513679d55d2f1b67cbf4b5c133f86eaeeaaca Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 4 Oct 2018 12:22:12 +0200 Subject: mod_http: Solve CORS problems once and for all This blindly allows any cross-site requests. Future work should add an API to allow each HTTP app some influence over this for each HTTP path --- plugins/mod_http.lua | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/plugins/mod_http.lua b/plugins/mod_http.lua index a1d409bd..07d1094b 100644 --- a/plugins/mod_http.lua +++ b/plugins/mod_http.lua @@ -22,6 +22,11 @@ server.set_default_host(module:get_option_string("http_default_host")); server.set_option("body_size_limit", module:get_option_number("http_max_content_size")); server.set_option("buffer_size_limit", module:get_option_number("http_max_buffer_size")); +-- CORS settigs +local opt_methods = module:get_option_set("access_control_allow_methods", { "GET", "POST", "PUT", "OPTIONS" }); +local opt_headers = module:get_option_set("access_control_allow_headers", { "Content-Type" }); +local opt_max_age = module:get_option_number("access_control_max_age", 2 * 60 * 60); + local function get_http_event(host, app_path, key) local method, path = key:match("^(%S+)%s+(.+)$"); if not method then -- No path specified, default to "" (base path) @@ -83,6 +88,13 @@ function moduleapi.http_url(module, app_name, default_path) return "http://disabled.invalid/"; end +local function apply_cors_headers(response, methods, headers, max_age, origin) + response.headers.access_control_allow_methods = tostring(methods); + response.headers.access_control_allow_headers = tostring(headers); + response.headers.access_control_max_age = tostring(max_age) + response.headers.access_control_allow_origin = origin or "*"; +end + function module.add_host(module) local host = module.host; if host ~= "*" then @@ -101,6 +113,12 @@ function module.add_host(module) end apps[app_name] = apps[app_name] or {}; local app_handlers = apps[app_name]; + + local function cors_handler(event_data) + local request, response = event_data.request, event_data.response; + apply_cors_headers(response, opt_methods, opt_headers, opt_max_age, request.headers.origin); + end + for key, handler in pairs(event.item.route or {}) do local event_name = get_http_event(host, app_path, key); if event_name then @@ -121,6 +139,7 @@ function module.add_host(module) if not app_handlers[event_name] then app_handlers[event_name] = handler; module:hook_object_event(server, event_name, handler); + module:hook_object_event(server, event_name, cors_handler, 1); else module:log("warn", "App %s added handler twice for '%s', ignoring", app_name, event_name); end -- cgit v1.2.3 From 2cbbc9fe2bb407230e89c1903f8470ba6ff7ecb0 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 4 Oct 2018 12:23:06 +0200 Subject: mod_bosh: Drop CORS code in favor of than in mod_http This deprecates the cross_domain_bosh setting. As a compat measure, if it is set, mod_http_crossdomain is loaded. --- plugins/mod_bosh.lua | 30 +++++------------------------- prosodyctl | 2 +- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/plugins/mod_bosh.lua b/plugins/mod_bosh.lua index d4701148..82615161 100644 --- a/plugins/mod_bosh.lua +++ b/plugins/mod_bosh.lua @@ -44,10 +44,12 @@ local bosh_max_polling = module:get_option_number("bosh_max_polling", 5); local bosh_max_wait = module:get_option_number("bosh_max_wait", 120); local consider_bosh_secure = module:get_option_boolean("consider_bosh_secure"); -local cross_domain = module:get_option("cross_domain_bosh", false); +local cross_domain = module:get_option("cross_domain_bosh"); -if cross_domain == true then cross_domain = "*"; end -if type(cross_domain) == "table" then cross_domain = table.concat(cross_domain, ", "); end +if cross_domain ~= nil then + module:log("info", "The 'cross_domain_bosh' option has been deprecated"); + module:depends("http_crossdomain"); +end local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat; @@ -91,22 +93,6 @@ function check_inactive(now, session, context, reason) -- luacheck: ignore 212/n end end -local function set_cross_domain_headers(response) - local headers = response.headers; - headers.access_control_allow_methods = "GET, POST, OPTIONS"; - headers.access_control_allow_headers = "Content-Type"; - headers.access_control_max_age = "7200"; - headers.access_control_allow_origin = cross_domain; - return response; -end - -function handle_OPTIONS(event) - if cross_domain and event.request.headers.origin then - set_cross_domain_headers(event.response); - end - return ""; -end - function handle_POST(event) log("debug", "Handling new request %s: %s\n----------", tostring(event.request), tostring(event.request.body)); @@ -121,10 +107,6 @@ function handle_POST(event) local headers = response.headers; headers.content_type = "text/xml; charset=utf-8"; - if cross_domain and request.headers.origin then - set_cross_domain_headers(response); - end - -- stream:feed() calls the stream_callbacks, so all stanzas in -- the body are processed in this next line before it returns. -- In particular, the streamopened() stream callback is where @@ -511,8 +493,6 @@ module:provides("http", { route = { ["GET"] = GET_response; ["GET /"] = GET_response; - ["OPTIONS"] = handle_OPTIONS; - ["OPTIONS /"] = handle_OPTIONS; ["POST"] = handle_POST; ["POST /"] = handle_POST; }; diff --git a/prosodyctl b/prosodyctl index 0ff03a7b..4d91cfe5 100755 --- a/prosodyctl +++ b/prosodyctl @@ -806,7 +806,7 @@ function commands.check(arg) print("Checking config..."); local deprecated = set.new({ "bosh_ports", "disallow_s2s", "no_daemonize", "anonymous_login", "require_encryption", - "vcard_compatibility", + "vcard_compatibility", "cross_domain_bosh", }); local known_global_options = set.new({ "pidfile", "log", "plugin_paths", "prosody_user", "prosody_group", "daemonize", -- cgit v1.2.3 From 26542811eafd9c708a130272d7b7de77b92712de Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 4 Oct 2018 12:24:08 +0200 Subject: mod_websocket: Drop CORS code in favor of that in mod_http Like for mod_bosh, deprecates consider_websocket_secure and depend on mod_http_crossdomain if it is set. --- plugins/mod_websocket.lua | 38 ++++---------------------------------- prosodyctl | 2 +- 2 files changed, 5 insertions(+), 35 deletions(-) diff --git a/plugins/mod_websocket.lua b/plugins/mod_websocket.lua index a668b4fa..da0ce8a6 100644 --- a/plugins/mod_websocket.lua +++ b/plugins/mod_websocket.lua @@ -29,18 +29,11 @@ local t_concat = table.concat; local stream_close_timeout = module:get_option_number("c2s_close_timeout", 5); local consider_websocket_secure = module:get_option_boolean("consider_websocket_secure"); -local cross_domain = module:get_option_set("cross_domain_websocket", {}); -if cross_domain:contains("*") or cross_domain:contains(true) then - cross_domain = true; +local cross_domain = module:get_option("cross_domain_websocket"); +if cross_domain ~= nil then + module:log("info", "The 'cross_domain_websocket' option has been deprecated"); + module:depends("http_crossdomain"); end - -local function check_origin(origin) - if cross_domain == true then - return true; - end - return cross_domain:contains(origin); -end - local xmlns_framing = "urn:ietf:params:xml:ns:xmpp-framing"; local xmlns_streams = "http://etherx.jabber.org/streams"; local xmlns_client = "jabber:client"; @@ -158,11 +151,6 @@ function handle_request(event) return 501; end - if not check_origin(request.headers.origin or "") then - module:log("debug", "Origin %s is not allowed by 'cross_domain_websocket'", request.headers.origin or "(missing header)"); - return 403; - end - local function websocket_close(code, message) conn:write(build_close(code, message)); conn:close(); @@ -329,22 +317,4 @@ module:provides("http", { function module.add_host(module) module:hook("c2s-read-timeout", keepalive, -0.9); - - if cross_domain ~= true then - local url = require "socket.url"; - local ws_url = module:http_url("websocket", "xmpp-websocket"); - local url_components = url.parse(ws_url); - -- The 'Origin' consists of the base URL without path - url_components.path = nil; - local this_origin = url.build(url_components); - local local_cross_domain = module:get_option_set("cross_domain_websocket", { this_origin }); - -- Don't add / remove something added by another host - -- This might be weird with random load order - local_cross_domain:exclude(cross_domain); - cross_domain:include(local_cross_domain); - module:log("debug", "cross_domain = %s", tostring(cross_domain)); - function module.unload() - cross_domain:exclude(local_cross_domain); - end - end end diff --git a/prosodyctl b/prosodyctl index 4d91cfe5..efb98386 100755 --- a/prosodyctl +++ b/prosodyctl @@ -806,7 +806,7 @@ function commands.check(arg) print("Checking config..."); local deprecated = set.new({ "bosh_ports", "disallow_s2s", "no_daemonize", "anonymous_login", "require_encryption", - "vcard_compatibility", "cross_domain_bosh", + "vcard_compatibility", "cross_domain_bosh", "cross_domain_websocket" }); local known_global_options = set.new({ "pidfile", "log", "plugin_paths", "prosody_user", "prosody_group", "daemonize", -- cgit v1.2.3 From 4d5f1da475da390dab6e8835f8159d868fe8c5b3 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 18 Jan 2019 02:03:40 +0100 Subject: mod_http: Set up to handle OPTIONS Lower priority to allow http modules to handle it themselves, should they wish to --- plugins/mod_http.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/mod_http.lua b/plugins/mod_http.lua index 07d1094b..01f20f76 100644 --- a/plugins/mod_http.lua +++ b/plugins/mod_http.lua @@ -119,9 +119,15 @@ function module.add_host(module) apply_cors_headers(response, opt_methods, opt_headers, opt_max_age, request.headers.origin); end + local function options_handler(event_data) + cors_handler(event_data); + return ""; + end + for key, handler in pairs(event.item.route or {}) do local event_name = get_http_event(host, app_path, key); if event_name then + local options_event_name = event_name:gsub("^%S+", "OPTIONS"); if type(handler) ~= "function" then local data = handler; handler = function () return data; end @@ -140,6 +146,7 @@ function module.add_host(module) app_handlers[event_name] = handler; module:hook_object_event(server, event_name, handler); module:hook_object_event(server, event_name, cors_handler, 1); + module:hook_object_event(server, options_event_name, options_handler, -1); else module:log("warn", "App %s added handler twice for '%s', ignoring", app_name, event_name); end -- cgit v1.2.3 From 13ce41d3f969bc039e3e1fcedcc566da86ba9028 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 17 Jan 2019 20:42:38 +0100 Subject: mod_http: Determine CORS methods to whitelist from actual methods used --- plugins/mod_http.lua | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/mod_http.lua b/plugins/mod_http.lua index 01f20f76..829c2d02 100644 --- a/plugins/mod_http.lua +++ b/plugins/mod_http.lua @@ -14,6 +14,7 @@ local moduleapi = require "core.moduleapi"; local url_parse = require "socket.url".parse; local url_build = require "socket.url".build; local normalize_path = require "util.http".normalize_path; +local set = require "util.set"; local server = require "net.http.server"; @@ -23,7 +24,7 @@ server.set_option("body_size_limit", module:get_option_number("http_max_content_ server.set_option("buffer_size_limit", module:get_option_number("http_max_buffer_size")); -- CORS settigs -local opt_methods = module:get_option_set("access_control_allow_methods", { "GET", "POST", "PUT", "OPTIONS" }); +local opt_methods = module:get_option_set("access_control_allow_methods", { "GET", "OPTIONS" }); local opt_headers = module:get_option_set("access_control_allow_headers", { "Content-Type" }); local opt_max_age = module:get_option_number("access_control_max_age", 2 * 60 * 60); @@ -114,9 +115,11 @@ function module.add_host(module) apps[app_name] = apps[app_name] or {}; local app_handlers = apps[app_name]; + local app_methods = opt_methods; + local function cors_handler(event_data) local request, response = event_data.request, event_data.response; - apply_cors_headers(response, opt_methods, opt_headers, opt_max_age, request.headers.origin); + apply_cors_headers(response, app_methods, opt_headers, opt_max_age, request.headers.origin); end local function options_handler(event_data) @@ -127,6 +130,10 @@ function module.add_host(module) for key, handler in pairs(event.item.route or {}) do local event_name = get_http_event(host, app_path, key); if event_name then + local method = event_name:match("^%S+"); + if not app_methods:contains(method) then + app_methods = app_methods + set.new{ method }; + end local options_event_name = event_name:gsub("^%S+", "OPTIONS"); if type(handler) ~= "function" then local data = handler; -- cgit v1.2.3 From 22195430f16dcf56cb4728c00254d89964617255 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 18 Jan 2019 18:30:41 +0100 Subject: prosodyctl: Fix module.command invocation (thanks woffs) The first argument is already removed once since c7727c13260f --- prosodyctl | 2 -- 1 file changed, 2 deletions(-) diff --git a/prosodyctl b/prosodyctl index efb98386..a628ed2b 100755 --- a/prosodyctl +++ b/prosodyctl @@ -1300,8 +1300,6 @@ local command_runner = async.runner(function () end end - table.remove(arg, 1); - local module = modulemanager.get_module("*", module_name); if not module then show_message("Failed to load module '"..module_name.."': Unknown error"); -- cgit v1.2.3 From 814eebb672ee88a87a7c3effe5c437efd34f5f3e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 19 Jan 2019 20:03:04 +0100 Subject: mod_bosh, mod_websocket: Remove accidentally included dependency on non-existant module --- plugins/mod_bosh.lua | 1 - plugins/mod_websocket.lua | 1 - 2 files changed, 2 deletions(-) diff --git a/plugins/mod_bosh.lua b/plugins/mod_bosh.lua index 82615161..082ed961 100644 --- a/plugins/mod_bosh.lua +++ b/plugins/mod_bosh.lua @@ -48,7 +48,6 @@ local cross_domain = module:get_option("cross_domain_bosh"); if cross_domain ~= nil then module:log("info", "The 'cross_domain_bosh' option has been deprecated"); - module:depends("http_crossdomain"); end local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat; diff --git a/plugins/mod_websocket.lua b/plugins/mod_websocket.lua index da0ce8a6..008f6823 100644 --- a/plugins/mod_websocket.lua +++ b/plugins/mod_websocket.lua @@ -32,7 +32,6 @@ local consider_websocket_secure = module:get_option_boolean("consider_websocket_ local cross_domain = module:get_option("cross_domain_websocket"); if cross_domain ~= nil then module:log("info", "The 'cross_domain_websocket' option has been deprecated"); - module:depends("http_crossdomain"); end local xmlns_framing = "urn:ietf:params:xml:ns:xmpp-framing"; local xmlns_streams = "http://etherx.jabber.org/streams"; -- cgit v1.2.3 From 88568a7af092d21061020932dd3ed7cfb0113c15 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 20 Jan 2019 20:24:17 +0100 Subject: mod_mam: Fix operator precedence (thanks mimi89999) --- plugins/mod_mam/mod_mam.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index d2ca709b..67bf177e 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -378,7 +378,7 @@ if cleanup_after ~= "never" then local ok, err = archive:delete(user, { ["end"] = os.time() - cleanup_after; }) if ok then num_users = num_users + 1; - sum = sum + tonumber(ok) or 0; + sum = sum + (tonumber(ok) or 0); end end module:log("info", "Deleted %d expired messages for %d users", sum, num_users); -- cgit v1.2.3 From ffd845f929ab51d182ae10765b5b9730d235f63b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 21 Jan 2019 21:30:54 +0100 Subject: TODO: Remove statistics since this was done in 0.10 --- TODO | 1 - 1 file changed, 1 deletion(-) diff --git a/TODO b/TODO index d22d20f4..9ec820b2 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,4 @@ == 1.0 == - Roster providers -- Statistics - Clustering - World domination -- cgit v1.2.3 From 976c452a40c2711c5afc14097762515efa216023 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 22 Jan 2019 09:21:23 +0100 Subject: core.statsmanager: Do a final collection on shutdown --- core/statsmanager.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/core/statsmanager.lua b/core/statsmanager.lua index 237b1dd5..50798ad0 100644 --- a/core/statsmanager.lua +++ b/core/statsmanager.lua @@ -97,6 +97,7 @@ if stats then end timer.add_task(stats_interval, collect); prosody.events.add_handler("server-started", function () collect() end, -1); + prosody.events.add_handler("server-stopped", function () collect() end, -1); else log("debug", "Statistics enabled using %s provider, collection is disabled", stats_provider_name); end -- cgit v1.2.3 From 053a17c00a2808a9da7bea08e87e74b34a54e0c2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 24 Jan 2019 05:48:55 +0100 Subject: GNUmakefile: Stop Prosody in case of failure in integration-test Normally make skips the remaning steps in the rule if one fails. This collects the status code and re-returns it after stopping the running Prosody instance. --- GNUmakefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index dd199997..977e91c6 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -74,8 +74,9 @@ test: integration-test: all $(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua start - $(SCANSION) -d ./spec/scansion - $(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua stop + $(SCANSION) -d ./spec/scansion; R=$$? \ + $(RUNWITH) prosodyctl --config ./spec/scansion/prosody.cfg.lua stop \ + exit $$R coverage: -rm -- luacov.* -- cgit v1.2.3 From e5603e10c58e543ac1e4bf8223ea07fb648ba58e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 9 Feb 2019 20:34:00 +0100 Subject: net.server_epoll: Separate timeout for initial connection attempts server_event has this separation already --- net/server_epoll.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index 807e0b4c..a80b33a9 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -41,6 +41,9 @@ local default_config = { __index = { -- How long to wait for a socket to become writable after queuing data to send send_timeout = 60; + -- How long to wait for a socket to become writable after creation + connect_timeout = 20; + -- Some number possibly influencing how many pending connections can be accepted tcp_backlog = 128; @@ -585,7 +588,7 @@ end -- Initialization function interface:init() - self:setwritetimeout(); + self:setwritetimeout(cfg.connect_timeout); return self:add(true, true); end -- cgit v1.2.3 From b80261ccbf3fbb5e0f07ccfe8b7419b48ee9bcbf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 9 Feb 2019 20:54:35 +0100 Subject: net.server_epoll: Increase send_timeout to 3 minutes (to match server_event) The separate connect_timeout means we can afford a longer send_timeout --- net/server_epoll.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index a80b33a9..fdf006f6 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -39,7 +39,7 @@ local default_config = { __index = { read_timeout = 14 * 60; -- How long to wait for a socket to become writable after queuing data to send - send_timeout = 60; + send_timeout = 180; -- How long to wait for a socket to become writable after creation connect_timeout = 20; -- cgit v1.2.3 From eb245f3dd6fb501214fabe1536a64bb76cef9c48 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 8 Dec 2018 18:02:56 +0100 Subject: prosodyctl: about: Report the current operating system according to uname --- prosodyctl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/prosodyctl b/prosodyctl index a628ed2b..f6812e23 100755 --- a/prosodyctl +++ b/prosodyctl @@ -363,6 +363,13 @@ function commands.about(arg) .."\n "; end))); print(""); + local have_pposix, pposix = pcall(require, "util.pposix"); + if have_pposix and pposix.uname then + print("# Operating system"); + local uname, err = pposix.uname(); + print(uname and uname.sysname .. " " .. uname.release or "Unknown POSIX", err or ""); + print(""); + end print("# Lua environment"); print("Lua version: ", _G._VERSION); print(""); -- cgit v1.2.3 From 344b370b5e6367761a32da79587ad02a15604845 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Feb 2019 16:18:30 +0100 Subject: MUC: Factor out role change permission check into its own method I would like to invert this logic so that it checks if the role change is allowed instead of checking if it is not allowed as it does now, in order to make it easier to understand. --- plugins/muc/muc.lib.lua | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua index 9b168e93..2b6a7d76 100644 --- a/plugins/muc/muc.lib.lua +++ b/plugins/muc/muc.lib.lua @@ -1368,6 +1368,30 @@ function room_mt:get_role(nick) return occupant and occupant.role or nil; end +function room_mt:may_set_role(actor, occupant, role) + -- Can't do anything to other owners or admins + local occupant_affiliation = self:get_affiliation(occupant.bare_jid); + if occupant_affiliation == "owner" or occupant_affiliation == "admin" then + return nil, "cancel", "not-allowed"; + end + + -- If you are trying to give or take moderator role you need to be an owner or admin + if occupant.role == "moderator" or role == "moderator" then + local actor_affiliation = self:get_affiliation(actor); + if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then + return nil, "cancel", "not-allowed"; + end + end + + -- Need to be in the room and a moderator + local actor_occupant = self:get_occupant_by_real_jid(actor); + if not actor_occupant or actor_occupant.role ~= "moderator" then + return nil, "cancel", "not-allowed"; + end + + return true; +end + function room_mt:set_role(actor, occupant_jid, role, reason) if not actor then return nil, "modify", "not-acceptable"; end @@ -1382,24 +1406,9 @@ function room_mt:set_role(actor, occupant_jid, role, reason) if actor == true then actor = nil -- So we can pass it safely to 'publicise_occupant_status' below else - -- Can't do anything to other owners or admins - local occupant_affiliation = self:get_affiliation(occupant.bare_jid); - if occupant_affiliation == "owner" or occupant_affiliation == "admin" then - return nil, "cancel", "not-allowed"; - end - - -- If you are trying to give or take moderator role you need to be an owner or admin - if occupant.role == "moderator" or role == "moderator" then - local actor_affiliation = self:get_affiliation(actor); - if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then - return nil, "cancel", "not-allowed"; - end - end - - -- Need to be in the room and a moderator - local actor_occupant = self:get_occupant_by_real_jid(actor); - if not actor_occupant or actor_occupant.role ~= "moderator" then - return nil, "cancel", "not-allowed"; + local allowed, err, condition = self:may_set_role(actor, occupant, role) + if not allowed then + return allowed, err, condition; end end -- cgit v1.2.3 From 4b80adc9641fca202ab7e2488722280932b44ad2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Feb 2019 16:30:11 +0100 Subject: MUC: Fire an event to allow affecting decision of whether to allow a role change --- plugins/muc/muc.lib.lua | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua index 2b6a7d76..a8d3d790 100644 --- a/plugins/muc/muc.lib.lua +++ b/plugins/muc/muc.lib.lua @@ -1369,6 +1369,18 @@ function room_mt:get_role(nick) end function room_mt:may_set_role(actor, occupant, role) + local event = { + room = self, + actor = actor, + occupant = occupant, + role = role, + }; + + module:fire_event("muc-pre-set-role", event); + if event.allowed ~= nil then + return event.allowed, event.error, event.condition; + end + -- Can't do anything to other owners or admins local occupant_affiliation = self:get_affiliation(occupant.bare_jid); if occupant_affiliation == "owner" or occupant_affiliation == "admin" then -- cgit v1.2.3 From 9ee67a16d1441e74435d156af7b0c19bb8d28d4e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 13 Sep 2018 21:16:37 +0200 Subject: net.server: New API for creating server listeners server.listen(interface, port, listeners, options); --- net/server_epoll.lua | 18 ++++++++++++++---- net/server_event.lua | 22 ++++++++++++++++------ net/server_select.lua | 30 ++++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index fdf006f6..5609f058 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -637,7 +637,7 @@ function interface:onconnect() self:on("connect"); end -local function addserver(addr, port, listeners, read_size, tls_ctx) +local function listen(addr, port, listeners, config) local conn, err = socket.bind(addr, port, cfg.tcp_backlog); if not conn then return conn, err; end conn:settimeout(0); @@ -645,10 +645,10 @@ local function addserver(addr, port, listeners, read_size, tls_ctx) conn = conn; created = gettime(); listeners = listeners; - read_size = read_size; + read_size = config and config.read_size; onreadable = interface.onacceptable; - tls_ctx = tls_ctx; - tls_direct = tls_ctx and true or false; + tls_ctx = config and config.tls_ctx; + tls_direct = config and config.tls_direct; sockname = addr; sockport = port; }, interface_mt); @@ -656,6 +656,15 @@ local function addserver(addr, port, listeners, read_size, tls_ctx) return server; end +-- COMPAT +local function addserver(addr, port, listeners, read_size, tls_ctx) + return listen(addr, port, listeners, { + read_size = read_size; + tls_ctx = tls_ctx; + tls_direct = tls_ctx and true or false; + }); +end + -- COMPAT local function wrapclient(conn, addr, port, listeners, read_size, tls_ctx) local client = wrapsocket(conn, nil, read_size, listeners, tls_ctx); @@ -792,6 +801,7 @@ return { addserver = addserver; addclient = addclient; add_task = addtimer; + listen = listen; at = at; loop = loop; closeall = closeall; diff --git a/net/server_event.lua b/net/server_event.lua index 70757e03..b78bf412 100644 --- a/net/server_event.lua +++ b/net/server_event.lua @@ -649,7 +649,7 @@ local function handleclient( client, ip, port, server, pattern, listener, sslctx return interface end -local function handleserver( server, addr, port, pattern, listener, sslctx ) -- creates an server interface +local function handleserver( server, addr, port, pattern, listener, sslctx, startssl ) -- creates an server interface debug "creating server interface..." local interface = { _connections = 0; @@ -695,7 +695,7 @@ local function handleserver( server, addr, port, pattern, listener, sslctx ) -- interface._connections = interface._connections + 1 -- increase connection count local clientinterface = handleclient( client, client_ip, client_port, interface, pattern, listener, sslctx ) --vdebug( "client id:", clientinterface, "startssl:", startssl ) - if has_luasec and sslctx then + if has_luasec and startssl then clientinterface:starttls(sslctx, true) else clientinterface:_start_session( true ) @@ -714,9 +714,9 @@ local function handleserver( server, addr, port, pattern, listener, sslctx ) -- return interface end -local function addserver( addr, port, listener, pattern, sslctx, startssl ) -- TODO: check arguments - --vdebug( "creating new tcp server with following parameters:", addr or "nil", port or "nil", sslctx or "nil", startssl or "nil") - if sslctx and not has_luasec then +local function listen(addr, port, listener, config) + config = config or {} + if config.sslctx and not has_luasec then debug "fatal error: luasec not found" return nil, "luasec not found" end @@ -725,11 +725,20 @@ local function addserver( addr, port, listener, pattern, sslctx, startssl ) -- debug( "creating server socket on "..addr.." port "..port.." failed:", err ) return nil, err end - local interface = handleserver( server, addr, port, pattern, listener, sslctx, startssl ) -- new server handler + local interface = handleserver( server, addr, port, config.read_size, listener, config.tls_ctx, config.tls_direct) -- new server handler debug( "new server created with id:", tostring(interface)) return interface end +local function addserver( addr, port, listener, pattern, sslctx ) -- TODO: check arguments + --vdebug( "creating new tcp server with following parameters:", addr or "nil", port or "nil", sslctx or "nil", startssl or "nil") + return listen( addr, port, listener, { + read_size = pattern, + tls_ctx = sslctx, + tls_direct = not not sslctx, + }); +end + local function wrapclient( client, ip, port, listeners, pattern, sslctx ) local interface = handleclient( client, ip, port, nil, pattern, listeners, sslctx ) interface:_start_connection(sslctx) @@ -890,6 +899,7 @@ return { event_base = base, addevent = newevent, addserver = addserver, + listen = listen, addclient = addclient, wrapclient = wrapclient, setquitting = setquitting, diff --git a/net/server_select.lua b/net/server_select.lua index f616116e..d82936e6 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -68,6 +68,7 @@ local idfalse local closeall local addsocket local addserver +local listen local addtimer local getserver local wrapserver @@ -157,7 +158,7 @@ _maxsslhandshake = 30 -- max handshake round-trips ----------------------------------// PRIVATE //-- -wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- this function wraps a server -- FIXME Make sure FD < _maxfd +wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx, ssldirect ) -- this function wraps a server -- FIXME Make sure FD < _maxfd if socket:getfd() >= _maxfd then out_error("server.lua: Disallowed FD number: "..socket:getfd()) @@ -244,13 +245,13 @@ wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- t local client, err = accept( socket ) -- try to accept if client then local ip, clientport = client:getpeername( ) - local handler, client, err = wrapconnection( handler, listeners, client, ip, serverport, clientport, pattern, sslctx ) -- wrap new client socket + local handler, client, err = wrapconnection( handler, listeners, client, ip, serverport, clientport, pattern, sslctx, ssldirect ) -- wrap new client socket if err then -- error while wrapping ssl socket return false end connections = connections + 1 out_put( "server.lua: accepted new client connection from ", tostring(ip), ":", tostring(clientport), " to ", tostring(serverport)) - if dispatch and not sslctx then -- SSL connections will notify onconnect when handshake completes + if dispatch and not ssldirect then -- SSL connections will notify onconnect when handshake completes return dispatch( handler ); end return; @@ -264,7 +265,7 @@ wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx ) -- t return handler end -wrapconnection = function( server, listeners, socket, ip, serverport, clientport, pattern, sslctx ) -- this function wraps a client to a handler object +wrapconnection = function( server, listeners, socket, ip, serverport, clientport, pattern, sslctx, ssldirect ) -- this function wraps a client to a handler object if socket:getfd() >= _maxfd then out_error("server.lua: Disallowed FD number: "..socket:getfd()) -- PROTIP: Switch to libevent @@ -666,7 +667,7 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport _socketlist[ socket ] = handler _readlistlen = addsocket(_readlist, socket, _readlistlen) - if sslctx and has_luasec then + if sslctx and ssldirect and has_luasec then out_put "server.lua: auto-starting ssl negotiation..." handler.autostart_ssl = true; local ok, err = handler:starttls(sslctx); @@ -741,9 +742,13 @@ end ----------------------------------// PUBLIC //-- -addserver = function( addr, port, listeners, pattern, sslctx ) -- this function provides a way for other scripts to reg a server +listen = function ( addr, port, listeners, config ) addr = addr or "*" + config = config or {} local err + local sslctx = config.tls_ctx; + local ssldirect = config.tls_direct; + local pattern = config.read_size; if type( listeners ) ~= "table" then err = "invalid listener table" elseif type ( addr ) ~= "string" then @@ -764,7 +769,7 @@ addserver = function( addr, port, listeners, pattern, sslctx ) -- this function out_error( "server.lua, [", addr, "]:", port, ": ", err ) return nil, err end - local handler, err = wrapserver( listeners, server, addr, port, pattern, sslctx ) -- wrap new server socket + local handler, err = wrapserver( listeners, server, addr, port, pattern, sslctx, ssldirect ) -- wrap new server socket if not handler then server:close( ) return nil, err @@ -777,6 +782,14 @@ addserver = function( addr, port, listeners, pattern, sslctx ) -- this function return handler end +addserver = function( addr, port, listeners, pattern, sslctx ) -- this function provides a way for other scripts to reg a server + return listen(addr, port, listeners, { + read_size = pattern; + tls_ctx = sslctx; + tls_direct = sslctx and true or false; + }); +end + getserver = function ( addr, port ) return _server[ addr..":"..port ]; end @@ -985,7 +998,7 @@ end --// EXPERIMENTAL //-- local wrapclient = function( socket, ip, serverport, listeners, pattern, sslctx ) - local handler, socket, err = wrapconnection( nil, listeners, socket, ip, serverport, "clientport", pattern, sslctx ) + local handler, socket, err = wrapconnection( nil, listeners, socket, ip, serverport, "clientport", pattern, sslctx, sslctx) if not handler then return nil, err end _socketlist[ socket ] = handler if not sslctx then @@ -1121,6 +1134,7 @@ return { stats = stats, closeall = closeall, addserver = addserver, + listen = listen, getserver = getserver, setlogger = setlogger, getsettings = getsettings, -- cgit v1.2.3 From ddf2d725a6968d38b5819c2ef881a1479b5972a6 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 13 Sep 2018 21:17:37 +0200 Subject: core.portmanager: Use server.listen API --- core/portmanager.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/portmanager.lua b/core/portmanager.lua index 1ed37da0..cf836634 100644 --- a/core/portmanager.lua +++ b/core/portmanager.lua @@ -126,7 +126,11 @@ local function activate(service_name) end if not err then -- Start listening on interface+port - local handler, err = server.addserver(interface, port_number, listener, mode, ssl); + local handler, err = server.listen(interface, port_number, listener, { + read_size = mode, + tls_ctx = ssl, + tls_direct = service_info.encryption == "ssl"; + }); if not handler then log("error", "Failed to open server port %d on %s, %s", port_number, interface, error_to_friendly_message(service_name, port_number, err)); -- cgit v1.2.3 From 3d5cb716b4f2dd544f83ce7bd1fee55f3a1ef43f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 10 Oct 2018 17:22:08 +0200 Subject: core.portmanager: Reduce scope of variable Not sure why it was all the way out there, seems like there would have been unexpected behaviour from that --- core/portmanager.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/portmanager.lua b/core/portmanager.lua index cf836634..db41e1ea 100644 --- a/core/portmanager.lua +++ b/core/portmanager.lua @@ -95,7 +95,7 @@ local function activate(service_name) } bind_ports = set.new(type(bind_ports) ~= "table" and { bind_ports } or bind_ports ); - local mode, ssl = listener.default_mode or default_mode; + local mode = listener.default_mode or default_mode; local hooked_ports = {}; for interface in bind_interfaces do @@ -107,7 +107,7 @@ local function activate(service_name) log("error", "Multiple services configured to listen on the same port ([%s]:%d): %s, %s", interface, port, active_services:search(nil, interface, port)[1][1].service.name or "", service_name or ""); else - local err; + local ssl, err; -- Create SSL context for this service/port if service_info.encryption == "ssl" then local global_ssl_config = config.get("*", "ssl") or {}; -- cgit v1.2.3 From 37ba8188057989a530aa5a5e138b181d1ad855a1 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 4 Mar 2019 13:13:37 +0100 Subject: mod_muc_mam: Validate that the FORM_TYPE of a query is as expected --- plugins/mod_muc_mam.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/mod_muc_mam.lua b/plugins/mod_muc_mam.lua index 166a5c71..963e5255 100644 --- a/plugins/mod_muc_mam.lua +++ b/plugins/mod_muc_mam.lua @@ -21,6 +21,7 @@ local jid_bare = require "util.jid".bare; local jid_split = require "util.jid".split; local jid_prep = require "util.jid".prep; local dataform = require "util.dataforms".new; +local get_form_type = require "util.dataforms".get_type; local mod_muc = module:depends"muc"; local get_room_from_jid = mod_muc.get_room_from_jid; @@ -131,7 +132,11 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event) local qstart, qend; local form = query:get_child("x", "jabber:x:data"); if form then - local err; + local form_type, err = get_form_type(form); + if form_type ~= xmlns_mam then + origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'")); + return true; + end form, err = query_form:data(form); if err then origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err)))); -- cgit v1.2.3 From 7f1a7179dde0dc2569c4100b307a55c6c2a71fd2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 10 Mar 2019 18:05:08 +0100 Subject: doc/net.server: Document the new server.listen() API --- doc/net.server.lua | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/net.server.lua b/doc/net.server.lua index 7342c549..aa9a4a9d 100644 --- a/doc/net.server.lua +++ b/doc/net.server.lua @@ -160,6 +160,26 @@ Returns: local function addserver(address, port, listeners, pattern, sslctx) end +--[[ Binds and listens on the given address and port +Mostly the same as addserver but with all optional arguments in a table + +Arguments: + - address: address to bind to, may be "*" to bind all addresses. will be resolved if it is a string. + - port: port to bind (as number) + - listeners: a table of listeners + - config: table of extra settings + - read_size: the amount of bytes to read or a read pattern + - tls_ctx: is a valid luasec constructor + - tls_direct: boolean true for direct TLS, false (or nil) for starttls + +Returns: + - handle + - nil, "an error message": on failure (e.g. out of file descriptors) +]] +local function listen(address, port, listeners, config) +end + + --[[ Wraps a lua-socket socket client socket in a handle. The socket must be already connected to the remote end. If `sslctx` is given, a SSL session will be negotiated before listeners are called. @@ -255,4 +275,5 @@ return { closeall = closeall; hook_signal = hook_signal; watchfd = watchfd; + listen = listen; } -- cgit v1.2.3 From ac2aeb46621d033813e7ca0730302dfff2a6d4a6 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 10 Mar 2019 19:35:34 +0100 Subject: net.server_epoll: Add support for SNI (#409) --- net/server_epoll.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index 5609f058..3c8b2613 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -509,6 +509,13 @@ function interface:tlshandskake() end conn:settimeout(0); self.conn = conn; + if conn.sni then + if self.servername then + conn:sni(self.servername); + elseif self._server and self._server.hosts then + conn:sni(self._server.hosts, true); + end + end self:on("starttls"); self.ondrain = nil; self.onwritable = interface.tlshandskake; @@ -649,6 +656,7 @@ local function listen(addr, port, listeners, config) onreadable = interface.onacceptable; tls_ctx = config and config.tls_ctx; tls_direct = config and config.tls_direct; + hosts = config and config.sni_hosts; sockname = addr; sockport = port; }, interface_mt); -- cgit v1.2.3 From 7e568a68dc06ca5fda2786cdb16b86d7715ee309 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 10 Oct 2018 17:23:03 +0200 Subject: core.portmanager: Record TLS config for each port --- core/portmanager.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/portmanager.lua b/core/portmanager.lua index db41e1ea..17758a36 100644 --- a/core/portmanager.lua +++ b/core/portmanager.lua @@ -107,12 +107,12 @@ local function activate(service_name) log("error", "Multiple services configured to listen on the same port ([%s]:%d): %s, %s", interface, port, active_services:search(nil, interface, port)[1][1].service.name or "", service_name or ""); else - local ssl, err; + local ssl, cfg, err; -- Create SSL context for this service/port if service_info.encryption == "ssl" then local global_ssl_config = config.get("*", "ssl") or {}; local prefix_ssl_config = config.get("*", config_prefix.."ssl") or global_ssl_config; - ssl, err = certmanager.create_context(service_info.name.." port "..port, "server", + ssl, err, cfg = certmanager.create_context(service_info.name.." port "..port, "server", prefix_ssl_config[interface], prefix_ssl_config[port], prefix_ssl_config, @@ -130,6 +130,7 @@ local function activate(service_name) read_size = mode, tls_ctx = ssl, tls_direct = service_info.encryption == "ssl"; + sni_hosts = {}, }); if not handler then log("error", "Failed to open server port %d on %s, %s", port_number, interface, @@ -140,6 +141,7 @@ local function activate(service_name) active_services:add(service_name, interface, port_number, { server = handler; service = service_info; + tls_cfg = cfg; }); end end -- cgit v1.2.3 From dda988a3891717bc1ef5bc5fd3c6b9172ee00887 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 14 Sep 2018 01:30:56 +0200 Subject: core.portmanager: Collect per-host certificates for SNI --- core/portmanager.lua | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/core/portmanager.lua b/core/portmanager.lua index 17758a36..5aef07d7 100644 --- a/core/portmanager.lua +++ b/core/portmanager.lua @@ -10,6 +10,7 @@ local set = require "util.set"; local table = table; local setmetatable, rawset, rawget = setmetatable, rawset, rawget; local type, tonumber, tostring, ipairs = type, tonumber, tostring, ipairs; +local pairs = pairs; local prosody = prosody; local fire_event = prosody.events.fire_event; @@ -227,15 +228,55 @@ end -- Event handlers +local function add_sni_host(host, service) + -- local global_ssl_config = config.get(host, "ssl") or {}; + for name, interface, port, n, active_service --luacheck: ignore 213 + in active_services:iter(service, nil, nil, nil) do + if active_service.server.hosts and active_service.tls_cfg then + -- local config_prefix = (active_service.config_prefix or name).."_"; + -- if config_prefix == "_" then + -- config_prefix = ""; + -- end + -- local prefix_ssl_config = config.get(host, config_prefix.."ssl") or global_ssl_config; + -- FIXME only global 'ssl' settings are mixed in here + -- TODO per host and per service settings should be merged in, + -- without overriding the per-host certificate + local ssl, err, cfg = certmanager.create_context(host, "server"); + if ssl then + active_service.server.hosts[host] = ssl; + if not active_service.tls_cfg.certificate then + active_service.server.tls_ctx = ssl; + active_service.tls_cfg = cfg; + end + else + log("error", "err = %q", err); + end + end + end +end + prosody.events.add_handler("item-added/net-provider", function (event) local item = event.item; register_service(item.name, item); + for host in pairs(prosody.hosts) do + add_sni_host(host, item.name); + end end); prosody.events.add_handler("item-removed/net-provider", function (event) local item = event.item; unregister_service(item.name, item); end); +prosody.events.add_handler("host-activated", add_sni_host); +prosody.events.add_handler("host-deactivated", function (host) + for name, interface, port, n, active_service --luacheck: ignore 213 + in active_services:iter(nil, nil, nil, nil) do + if active_service.tls_cfg then + active_service.server.hosts[host] = nil; + end + end +end); + return { activate = activate; deactivate = deactivate; -- cgit v1.2.3 From 700f9dceb670bb7dacddbbb5800e6ba12927b409 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 10 Mar 2019 19:32:54 +0100 Subject: net.server_event: Add SNI support (#409) Snippet adapted from server_epoll --- net/server_event.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/net/server_event.lua b/net/server_event.lua index b78bf412..6c9b941d 100644 --- a/net/server_event.lua +++ b/net/server_event.lua @@ -164,6 +164,15 @@ function interface_mt:_start_ssl(call_onconnect) -- old socket will be destroyed debug( "fatal error while ssl wrapping:", err ) return false end + + if self.conn.sni then + if self.servername then + self.conn:sni(self.servername); + elseif self._server and self._server.hosts then + self.conn:sni(self._server.hosts, true); + end + end + self.conn:settimeout( 0 ) -- set non blocking local handshakecallback = coroutine_wrap(function( event ) local _, err @@ -665,6 +674,7 @@ local function handleserver( server, addr, port, pattern, listener, sslctx, star _ip = addr, _port = port, _pattern = pattern, _sslctx = sslctx; + hosts = {}; } interface.id = tostring(interface):match("%x+$"); interface.readcallback = function( event ) -- server handler, called on incoming connections -- cgit v1.2.3 From b47d67c80fe77a3414c8f0ef1fa92cec96696e54 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 10 Mar 2019 19:32:33 +0100 Subject: net.server_select: SNI support (#409) --- net/server_select.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/net/server_select.lua b/net/server_select.lua index d82936e6..b52cc6d7 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -184,6 +184,7 @@ wrapserver = function( listeners, socket, ip, serverport, pattern, sslctx, ssldi handler.sslctx = function( ) return sslctx end + handler.hosts = {} -- sni handler.remove = function( ) connections = connections - 1 if handler then @@ -627,11 +628,20 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport out_put( "server.lua: attempting to start tls on " .. tostring( socket ) ) local oldsocket, err = socket socket, err = ssl_wrap( socket, sslctx ) -- wrap socket + if not socket then out_put( "server.lua: error while starting tls on client: ", tostring(err or "unknown error") ) return nil, err -- fatal error end + if socket.sni then + if self.servername then + socket:sni(self.servername); + elseif self.server() and self.server().hosts then + socket:sni(self.server().hosts, true); + end + end + socket:settimeout( 0 ) -- add the new socket to our system -- cgit v1.2.3 From a08b6d8226ab4f2fe5358f664990e245db01b9a6 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 10 Mar 2019 19:58:28 +0100 Subject: core.certmanager: Do not ask for client certificates by default Since it's mostly only mod_s2s that needs to request client certificates it makes some sense to have mod_s2s ask for this, instead of having eg mod_http ask to disable it. --- core/certmanager.lua | 2 +- plugins/mod_http.lua | 3 --- plugins/mod_s2s/mod_s2s.lua | 3 +++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/certmanager.lua b/core/certmanager.lua index 5282a6f5..63f314f8 100644 --- a/core/certmanager.lua +++ b/core/certmanager.lua @@ -106,7 +106,7 @@ local core_defaults = { capath = "/etc/ssl/certs"; depth = 9; protocol = "tlsv1+"; - verify = (ssl_x509 and { "peer", "client_once", }) or "none"; + verify = "none"; options = { cipher_server_preference = luasec_has.options.cipher_server_preference; no_ticket = luasec_has.options.no_ticket; diff --git a/plugins/mod_http.lua b/plugins/mod_http.lua index 829c2d02..17ea27e1 100644 --- a/plugins/mod_http.lua +++ b/plugins/mod_http.lua @@ -228,9 +228,6 @@ module:provides("net", { listener = server.listener; default_port = 5281; encryption = "ssl"; - ssl_config = { - verify = "none"; - }; multiplex = { pattern = "^[A-Z]"; }; diff --git a/plugins/mod_s2s/mod_s2s.lua b/plugins/mod_s2s/mod_s2s.lua index 79308847..b0d551fe 100644 --- a/plugins/mod_s2s/mod_s2s.lua +++ b/plugins/mod_s2s/mod_s2s.lua @@ -738,6 +738,9 @@ module:provides("net", { listener = listener; default_port = 5269; encryption = "starttls"; + ssl_config = { + verify = { "peer", "client_once", }; + }; multiplex = { pattern = "^<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:server%1.*>"; }; -- cgit v1.2.3 From 2bc176f1d072da9404436a40581745033d1e41b6 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 11 Mar 2019 13:00:51 +0100 Subject: net.server: Only add alternate SNI contexts if at least one is provided Fixes use of when a client sends SNI, which would send no certificate otherwise. --- net/server_epoll.lua | 2 +- net/server_event.lua | 2 +- net/server_select.lua | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index 3c8b2613..4bdc2e21 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -512,7 +512,7 @@ function interface:tlshandskake() if conn.sni then if self.servername then conn:sni(self.servername); - elseif self._server and self._server.hosts then + elseif self._server and type(self._server.hosts) == "table" and next(self._server.hosts) ~= nil then conn:sni(self._server.hosts, true); end end diff --git a/net/server_event.lua b/net/server_event.lua index 6c9b941d..2bee614a 100644 --- a/net/server_event.lua +++ b/net/server_event.lua @@ -168,7 +168,7 @@ function interface_mt:_start_ssl(call_onconnect) -- old socket will be destroyed if self.conn.sni then if self.servername then self.conn:sni(self.servername); - elseif self._server and self._server.hosts then + elseif self._server and type(self._server.hosts) == "table" and next(self._server.hosts) ~= nil then self.conn:sni(self._server.hosts, true); end end diff --git a/net/server_select.lua b/net/server_select.lua index b52cc6d7..4b156409 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -637,7 +637,7 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport if socket.sni then if self.servername then socket:sni(self.servername); - elseif self.server() and self.server().hosts then + elseif self._server and type(self._server.hosts) == "table" and next(self._server.hosts) ~= nil then socket:sni(self.server().hosts, true); end end -- cgit v1.2.3 From 4b022060a27216380565302aa0fc3ca805a532eb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 11 Mar 2019 13:07:59 +0100 Subject: mod_tls: Restore querying for certificates on s2s The 'ssl_config' setting in the mod_s2s network service is not used. Only direct TLS ports use this currently. --- plugins/mod_s2s/mod_s2s.lua | 2 +- plugins/mod_tls.lua | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/mod_s2s/mod_s2s.lua b/plugins/mod_s2s/mod_s2s.lua index b0d551fe..f0fdc5fb 100644 --- a/plugins/mod_s2s/mod_s2s.lua +++ b/plugins/mod_s2s/mod_s2s.lua @@ -738,7 +738,7 @@ module:provides("net", { listener = listener; default_port = 5269; encryption = "starttls"; - ssl_config = { + ssl_config = { -- FIXME This is not used atm, see mod_tls verify = { "peer", "client_once", }; }; multiplex = { diff --git a/plugins/mod_tls.lua b/plugins/mod_tls.lua index 4ead60dc..d8bf02c4 100644 --- a/plugins/mod_tls.lua +++ b/plugins/mod_tls.lua @@ -53,13 +53,17 @@ function module.load() local parent_s2s = rawgetopt(parent, "s2s_ssl") or NULL; local host_s2s = rawgetopt(modhost, "s2s_ssl") or parent_s2s; + local request_client_certs = { verify = { "peer", "client_once", }; }; + ssl_ctx_c2s, err_c2s, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err_c2s); end - ssl_ctx_s2sout, err_s2sout, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s); -- for outgoing server connections + -- for outgoing server connections + ssl_ctx_s2sout, err_s2sout, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s, request_client_certs); if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err_s2sout); end - ssl_ctx_s2sin, err_s2sin, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s); -- for incoming server connections + -- for incoming server connections + ssl_ctx_s2sin, err_s2sin, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s, request_client_certs); if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err_s2sin); end end -- cgit v1.2.3 From 39291da9614b3a9f01afa877d319b11d5b942087 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 12 Mar 2019 23:13:51 +0100 Subject: net.server_epoll: Optimize timer handling --- net/server_epoll.lua | 83 +++++++++++++++++++--------------------------------- 1 file changed, 30 insertions(+), 53 deletions(-) diff --git a/net/server_epoll.lua b/net/server_epoll.lua index 4bdc2e21..4037f7ab 100644 --- a/net/server_epoll.lua +++ b/net/server_epoll.lua @@ -6,9 +6,7 @@ -- -local t_sort = table.sort; local t_insert = table.insert; -local t_remove = table.remove; local t_concat = table.concat; local setmetatable = setmetatable; local tostring = tostring; @@ -20,6 +18,7 @@ local log = require "util.logger".init("server_epoll"); local socket = require "socket"; local luasec = require "ssl"; local gettime = require "util.time".now; +local indexedbheap = require "util.indexedbheap"; local createtable = require "util.table".create; local inet = require "util.net"; local inet_pton = inet.pton; @@ -69,22 +68,24 @@ local fds = createtable(10, 0); -- FD -> conn -- Timer and scheduling -- -local timers = {}; +local timers = indexedbheap.create(); local function noop() end local function closetimer(t) t[1] = 0; t[2] = noop; + timers:remove(t.id); end --- Set to true when timers have changed -local resort_timers = false; +local function reschedule(t, time) + t[1] = time; + timers:reprioritize(t.id, time); +end -- Add absolute timer local function at(time, f) - local timer = { time, f, close = closetimer }; - t_insert(timers, timer); - resort_timers = true; + local timer = { time, f, close = closetimer, reschedule = reschedule, id = nil }; + timer.id = timers:insert(timer, time); return timer; end @@ -97,54 +98,32 @@ end -- Return time until next timeout local function runtimers(next_delay, min_wait) -- Any timers at all? - if not timers[1] then - return next_delay; - end + local now = gettime(); + local peek = timers:peek(); + while peek do - if resort_timers then - -- Sort earliest timers to the end - t_sort(timers, function (a, b) return a[1] > b[1]; end); - resort_timers = false; - end - - -- Iterate from the end and remove completed timers - for i = #timers, 1, -1 do - local timer = timers[i]; - local t, f = timer[1], timer[2]; - -- Get time for every iteration to increase accuracy - local now = gettime(); - if t > now then - -- This timer should not fire yet - local diff = t - now; - if diff < next_delay then - next_delay = diff; - end + if peek > now then + next_delay = peek - now; break; end - local new_timeout = f(now); - if new_timeout then - -- Schedule for 'delay' from the time actually scheduled, not from now, - -- in order to prevent timer drift, unless it already drifted way out of sync. - if (t + new_timeout) > ( now - new_timeout ) then - timer[1] = t + new_timeout; - else - timer[1] = now + new_timeout; - end - resort_timers = true; - else - t_remove(timers, i); + + local _, timer, id = timers:pop(); + local ok, ret = pcall(timer[2], now); + if ok and type(ret) == "number" then + local next_time = now+ret; + timer[1] = next_time; + timers:insert(timer, next_time); end - end - if resort_timers or next_delay < min_wait then - -- Timers may be added from within a timer callback. - -- Those would not be considered for next_delay, - -- and we might sleep for too long, so instead - -- we return a shorter timeout so we can - -- properly sort all new timers. - next_delay = min_wait; + peek = timers:peek(); + end + if peek == nil then + return next_delay; end + if next_delay < min_wait then + return min_wait; + end return next_delay; end @@ -251,8 +230,7 @@ function interface:setreadtimeout(t) end t = t or cfg.read_timeout; if self._readtimeout then - self._readtimeout[1] = gettime() + t; - resort_timers = true; + self._readtimeout:reschedule(gettime() + t); else self._readtimeout = addtimer(t, function () if self:on("readtimeout") then @@ -276,8 +254,7 @@ function interface:setwritetimeout(t) end t = t or cfg.send_timeout; if self._writetimeout then - self._writetimeout[1] = gettime() + t; - resort_timers = true; + self._writetimeout:reschedule(gettime() + t); else self._writetimeout = addtimer(t, function () self:on("disconnect", "write timeout"); -- cgit v1.2.3 From 82a10071d06089c724860a2cbd64bc0e8b6b38e8 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 14 Mar 2019 16:13:14 +0000 Subject: doc/coding_style.{txt,md}: Update coding style guide --- HACKERS | 2 +- doc/coding_style.md | 805 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 806 insertions(+), 1 deletion(-) create mode 100644 doc/coding_style.md diff --git a/HACKERS b/HACKERS index cae98091..240de63d 100644 --- a/HACKERS +++ b/HACKERS @@ -5,7 +5,7 @@ involved you can join us on our mailing list and discussion rooms. More information on these at https://prosody.im/discuss Patches are welcome, though before sending we would appreciate if you read -docs/coding_style.txt for guidelines on how to format your code, and other tips. +docs/coding_style.md for guidelines on how to format your code, and other tips. Documentation for developers can be found at https://prosody.im/doc/developers diff --git a/doc/coding_style.md b/doc/coding_style.md new file mode 100644 index 00000000..6f34d371 --- /dev/null +++ b/doc/coding_style.md @@ -0,0 +1,805 @@ + +# Prosody Coding Style Guide + +This style guides lists the coding conventions used in the +[Prosody](https://prosody.im/) project. It is based heavily on the [style guide used by the LuaRocks project](https://github.com/luarocks/lua-style-guide). + +## Indentation and formatting + +* Prosody code is indented with tabs at the start of the line, a single + tab per logical indent level: + +```lua +for i, pkg in ipairs(packages) do + for name, version in pairs(pkg) do + if name == searched then + print(version) + end + end +end +``` + +Tab width is configurable in editors, so never assume a particular width. +Specically this means you should not mix tabs and spaces, or use tabs for +alignment of items at different indentation levels. + +* Use LF (Unix) line endings. + +## Comments + +* Comments are encouraged where necessary to explain non-obvious code. + +* In general comments should be used to explain 'why', not 'how' + +### Comment tags + +A comment may be prefixed with one of the following tags: + +* **FIXME**: Indicates a serious problem with the code that should be addressed +* **TODO**: Indicates an open task, feature request or code restructuring that + is primarily of interest to developers (otherwise it should be in the + issue tracker). +* **COMPAT**: Must be used on all code that is present only for backwards-compatibility, + and may be removed one day. For example code that is added to support old + or buggy third-party software or dependencies. + +**Example:** + +```lua +-- TODO: implement method +local function something() + -- FIXME: check conditions +end + +``` + +## Variable names + +* Variable names with larger scope should be more descriptive than those with +smaller scope. One-letter variable names should be avoided except for very +small scopes (less than ten lines) or for iterators. + +* `i` should be used only as a counter variable in for loops (either numeric for +or `ipairs`). + +* Prefer more descriptive names than `k` and `v` when iterating with `pairs`, +unless you are writing a function that operates on generic tables. + +* Use `_` for ignored variables (e.g. in for loops:) + +```lua +for _, item in ipairs(items) do + do_something_with_item(item) +end +``` + +* Generally all identifiers (variables and function names) should use `snake_case`, + i.e. lowercase words joined by `_`. + +```lua +-- bad +local OBJEcttsssss = {} +local thisIsMyObject = {} +local c = function() + -- ...stuff... +end + +-- good +local this_is_my_object = {} + +local function do_that_thing() + -- ...stuff... +end +``` + +> **Rationale:** The standard library uses lowercase APIs, with `joinedlowercase` +names, but this does not scale too well for more complex APIs. `snake_case` +tends to look good enough and not too out-of-place along side the standard +APIs. + +```lua +for _, name in pairs(names) do + -- ...stuff... +end +``` + +* Prefer using `is_` when naming boolean functions: + +```lua +-- bad +local function evil(alignment) + return alignment < 100 +end + +-- good +local function is_evil(alignment) + return alignment < 100 +end +``` + +* `UPPER_CASE` is to be used sparingly, with "constants" only. + +> **Rationale:** "Sparingly", since Lua does not have real constants. This +notation is most useful in libraries that bind C libraries, when bringing over +constants from C. + +* Do not use uppercase names starting with `_`, they are reserved by Lua. + +## Tables + +* When creating a table, prefer populating its fields all at once, if possible: + +```lua +local player = { name = "Jack", class = "Rogue" } +} +``` + +* Items should be separated by commas. If there are many items, put each + key/value on a separate line and use a semi-colon after each item (including + the last one): + +```lua +local player = { + name = "Jack"; + class = "Rogue"; +} +``` + +> **Rationale:** This makes the structure of your tables more evident at a glance. +Trailing commas make it quicker to add new fields and produces shorter diffs. + +* Use plain `key` syntax whenever possible, use `["key"]` syntax when using names +that can't be represented as identifiers and avoid mixing representations in +a declaration: + +```lua +local mytable = { + ["1394-E"] = val1, + ["UTF-8"] = val2, + ["and"] = val2, +} +``` + +## Strings + +* Use `"double quotes"` for strings; use `'single quotes'` when writing strings +that contain double quotes. + +```lua +local name = "Prosody" +local sentence = 'The name of the program is "Prosody"' +``` + +> **Rationale:** Double quotes are used as string delimiters in a larger number of +programming languages. Single quotes are useful for avoiding escaping when +using double quotes in literals. + +## Line lengths + +* There are no hard or soft limits on line lengths. Line lengths are naturally +limited by using one statement per line. If that still produces lines that are +too long (e.g. an expression that produces a line over 256-characters long, +for example), this means the expression is too complex and would do better +split into subexpressions with reasonable names. + +> **Rationale:** No one works on VT100 terminals anymore. If line lengths are a proxy +for code complexity, we should address code complexity instead of using line +breaks to fit mind-bending statements over multiple lines. + +## Function declaration syntax + +* Prefer function syntax over variable syntax. This helps differentiate between +named and anonymous functions. + +```lua +-- bad +local nope = function(name, options) + -- ...stuff... +end + +-- good +local function yup(name, options) + -- ...stuff... +end +``` + +* Perform validation early and return as early as possible. + +```lua +-- bad +local function is_good_name(name, options, arg) + local is_good = #name > 3 + is_good = is_good and #name < 30 + + -- ...stuff... + + return is_good +end + +-- good +local function is_good_name(name, options, args) + if #name < 3 or #name > 30 then + return false + end + + -- ...stuff... + + return true +end +``` + +## Function calls + +* Even though Lua allows it, generally you should not omit parentheses + for functions that take a unique string literal argument. + +```lua +-- bad +local data = get_data"KRP"..tostring(area_number) +-- good +local data = get_data("KRP"..tostring(area_number)) +local data = get_data("KRP")..tostring(area_number) +``` + +> **Rationale:** It is not obvious at a glace what the precedence rules are +when omitting the parentheses in a function call. Can you quickly tell which +of the two "good" examples in equivalent to the "bad" one? (It's the second +one). + +* You should not omit parenthesis for functions that take a unique table +argument on a single line. You may do so for table arguments that span several +lines. + +```lua +local an_instance = a_module.new { + a_parameter = 42, + another_parameter = "yay", +} +``` + +> **Rationale:** The use as in `a_module.new` above occurs alone in a statement, +so there are no precedence issues. + +## Table attributes + +* Use dot notation when accessing known properties. + +```lua +local luke = { + jedi = true, + age = 28, +} + +-- bad +local is_jedi = luke["jedi"] + +-- good +local is_jedi = luke.jedi +``` + +* Use subscript notation `[]` when accessing properties with a variable or if using a table as a list. + +```lua +local vehicles = load_vehicles_from_disk("vehicles.dat") + +if vehicles["Porsche"] then + porsche_handler(vehicles["Porsche"]) + vehicles["Porsche"] = nil +end +for name, cars in pairs(vehicles) do + regular_handler(cars) +end +``` + +> **Rationale:** Using dot notation makes it clearer that the given key is meant +to be used as a record/object field. + +## Functions in tables + +* When declaring modules and classes, declare functions external to the table definition: + +```lua +local my_module = {} + +function my_module.a_function(x) + -- code +end +``` + +* When declaring metatables, declare function internal to the table definition. + +```lua +local version_mt = { + __eq = function(a, b) + -- code + end; + __lt = function(a, b) + -- code + end; +} +``` + +> **Rationale:** Metatables contain special behavior that affect the tables +they're assigned (and are used implicitly at the call site), so it's good to +be able to get a view of the complete behavior of the metatable at a glance. + +This is not as important for objects and modules, which usually have way more +code, and which don't fit in a single screen anyway, so nesting them inside +the table does not gain much: when scrolling a longer file, it is more evident +that `check_version` is a method of `Api` if it says `function Api:check_version()` +than if it says `check_version = function()` under some indentation level. + +## Variable declaration + +* Always use `local` to declare variables. + +```lua +-- bad +superpower = get_superpower() + +-- good +local superpower = get_superpower() +``` + +> **Rationale:** Not doing so will result in global variables to avoid polluting +the global namespace. + +## Variable scope + +* Assign variables with the smallest possible scope. + +```lua +-- bad +local function good() + local name = get_name() + + test() + print("doing stuff..") + + --...other stuff... + + if name == "test" then + return false + end + + return name +end + +-- good +local bad = function() + test() + print("doing stuff..") + + --...other stuff... + + local name = get_name() + + if name == "test" then + return false + end + + return name +end +``` + +> **Rationale:** Lua has proper lexical scoping. Declaring the function later means that its +scope is smaller, so this makes it easier to check for the effects of a variable. + +## Conditional expressions + +* False and nil are falsy in conditional expressions. Use shortcuts when you +can, unless you need to know the difference between false and nil. + +```lua +-- bad +if name ~= nil then + -- ...stuff... +end + +-- good +if name then + -- ...stuff... +end +``` + +* Avoid designing APIs which depend on the difference between `nil` and `false`. + +* Use the `and`/`or` idiom for the pseudo-ternary operator when it results in +more straightforward code. When nesting expressions, use parentheses to make it +easier to scan visually: + +```lua +local function default_name(name) + -- return the default "Waldo" if name is nil + return name or "Waldo" +end + +local function brew_coffee(machine) + return (machine and machine.is_loaded) and "coffee brewing" or "fill your water" +end +``` + +Note that the `x and y or z` as a substitute for `x ? y : z` does not work if +`y` may be `nil` or `false` so avoid it altogether for returning booleans or +values which may be nil. + +## Blocks + +* Use single-line blocks only for `then return`, `then break` and `function return` (a.k.a "lambda") constructs: + +```lua +-- good +if test then break end + +-- good +if not ok then return nil, "this failed for this reason: " .. reason end + +-- good +use_callback(x, function(k) return k.last end) + +-- good +if test then + return false +end + +-- bad +if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == 10 then do_other_complicated_function() end + +-- good +if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == 10 then + do_other_complicated_function() + return false +end +``` + +* Separate statements onto multiple lines. Do not use semicolons as statement terminators. + +```lua +-- bad +local whatever = "sure"; +a = 1; b = 2 + +-- good +local whatever = "sure" +a = 1 +b = 2 +``` + +## Spacing + +* Use a space after `--`. + +```lua +--bad +-- good +``` + +* Always put a space after commas and between operators and assignment signs: + +```lua +-- bad +local x = y*9 +local numbers={1,2,3} +numbers={1 , 2 , 3} +numbers={1 ,2 ,3} +local strings = { "hello" + , "Lua" + , "world" + } +dog.set( "attr",{ + age="1 year", + breed="Bernese Mountain Dog" +}) + +-- good +local x = y * 9 +local numbers = {1, 2, 3} +local strings = { + "hello"; + "Lua"; + "world"; +} +dog.set("attr", { + age = "1 year", + breed = "Bernese Mountain Dog", +}) +``` + +* Indent tables and functions according to the start of the line, not the construct: + +```lua +-- bad +local my_table = { + "hello", + "world", + } +using_a_callback(x, function(...) + print("hello") + end) + +-- good +local my_table = { + "hello"; + "world"; +} +using_a_callback(x, function(...) + print("hello") +end) +``` + +> **Rationale:** This keep indentation levels aligned at predictable places. You don't +need to realign the entire block if something in the first line changes (such as +replacing `x` with `xy` in the `using_a_callback` example above). + +* The concatenation operator gets a pass for avoiding spaces: + +```lua +-- okay +local message = "Hello, "..user.."! This is your day # "..day.." in our platform!" +``` + +> **Rationale:** Being at the baseline, the dots already provide some visual spacing. + +* No spaces after the name of a function in a declaration or in its arguments: + +```lua +-- bad +local function hello ( name, language ) + -- code +end + +-- good +local function hello(name, language) + -- code +end +``` + +* Add blank lines between functions: + +```lua +-- bad +local function foo() + -- code +end +local function bar() + -- code +end + +-- good +local function foo() + -- code +end + +local function bar() + -- code +end +``` + +* Avoid aligning variable declarations: + +```lua +-- bad +local a = 1 +local long_identifier = 2 + +-- good +local a = 1 +local long_identifier = 2 +``` + +> **Rationale:** This produces extra diffs which add noise to `git blame`. + +* Alignment is occasionally useful when logical correspondence is to be highlighted: + +```lua +-- okay +sys_command(form, UI_FORM_UPDATE_NODE, "a", FORM_NODE_HIDDEN, false) +sys_command(form, UI_FORM_UPDATE_NODE, "sample", FORM_NODE_VISIBLE, false) +``` + +## Typing + +* In non-performance critical code, it can be useful to add type-checking assertions +for function arguments: + +```lua +function manif.load_manifest(repo_url, lua_version) + assert(type(repo_url) == "string") + assert(type(lua_version) == "string" or not lua_version) + + -- ... +end +``` + +* Use the standard functions for type conversion, avoid relying on coercion: + +```lua +-- bad +local total_score = review_score .. "" + +-- good +local total_score = tostring(review_score) +``` + +## Errors + +* Functions that can fail for reasons that are expected (e.g. I/O) should +return `nil` and a (string) error message on error, possibly followed by other +return values such as an error code. + +* On errors such as API misuse, an error should be thrown, either with `error()` +or `assert()`. + +## Modules + +Follow [these guidelines](http://hisham.hm/2014/01/02/how-to-write-lua-modules-in-a-post-module-world/) for writing modules. In short: + +* Always require a module into a local variable named after the last component of the module’s full name. + +```lua +local bar = require("foo.bar") -- requiring the module + +bar.say("hello") -- using the module +``` + +* Don’t rename modules arbitrarily: + +```lua +-- bad +local skt = require("socket") +``` + +> **Rationale:** Code is much harder to read if we have to keep going back to the top +to check how you chose to call a module. + +* Start a module by declaring its table using the same all-lowercase local +name that will be used to require it. You may use an LDoc comment to identify +the whole module path. + +```lua +--- @module foo.bar +local bar = {} +``` + +* Try to use names that won't clash with your local variables. For instance, don't +name your module something like “size”. + +* Use `local function` to declare _local_ functions only: that is, functions +that won’t be accessible from outside the module. + +That is, `local function helper_foo()` means that `helper_foo` is really local. + +* Public functions are declared in the module table, with dot syntax: + +```lua +function bar.say(greeting) + print(greeting) +end +``` + +> **Rationale:** Visibility rules are made explicit through syntax. + +* Do not set any globals in your module and always return a table in the end. + +* If you would like your module to be used as a function, you may set the +`__call` metamethod on the module table instead. + +> **Rationale:** Modules should return tables in order to be amenable to have their +contents inspected via the Lua interactive interpreter or other tools. + +* Requiring a module should cause no side-effect other than loading other +modules and returning the module table. + +* A module should not have state. If a module needs configuration, turn + it into a factory. For example, do not make something like this: + +```lua +-- bad +local mp = require "MessagePack" +mp.set_integer("unsigned") +``` + +and do something like this instead: + +```lua +-- good +local messagepack = require("messagepack") +local mpack = messagepack.new({integer = "unsigned"}) +``` + +* The invocation of require may omit parentheses around the module name: + +```lua +local bla = require "bla" +``` + +## Metatables, classes and objects + +If creating a new type of object that has a metatable and methods, the +metatable and methods table should be separate, and the metatable name +should end with `_mt`. + +```lua +local mytype_methods = {}; +local mytype_mt = { __index = mytype_methods }; + +function mytype_methods:add_new_thing(thing) +end + +local function new() + return setmetatable({}, mytype_mt); +end + +return { new = new }; +``` + +* Use the method notation when invoking methods: + +``` +-- bad +my_object.my_method(my_object) + +-- good +my_object:my_method() +``` + +> **Rationale:** This makes it explicit that the intent is to use the function as a method. + +* Do not rely on the `__gc` metamethod to release resources other than memory. +If your object manage resources such as files, add a `close` method to their +APIs and do not auto-close via `__gc`. Auto-closing via `__gc` would entice +users of your module to not close resources as soon as possible. (Note that +the standard `io` library does not follow this recommendation, and users often +forget that not closing files immediately can lead to "too many open files" +errors when the program runs for a while.) + +> **Rationale:** The garbage collector performs automatic *memory* management, +dealing with memory only. There is no guarantees as to when the garbage +collector will be invoked, and memory pressure does not correlate to pressure +on other resources. + +## File structure + +* Lua files should be named in all lowercase. + +* Tests should be in a top-level `spec` directory. Prosody uses +[Busted](http://olivinelabs.com/busted/) for testing. + +## Static checking + +All code should pass [luacheck](https://github.com/mpeterv/luacheck) using +the `.luacheckrc` provided in the Prosody repository, and using miminal +inline exceptions. + +* luacheck warnings of class 211, 212, 213 (unused variable, argument or loop +variable) may be ignored, if the unused variable was added explicitly: for +example, sometimes it is useful, for code understandability, to spell out what +the keys and values in a table are, even if you're only using one of them. +Another example is a function that needs to follow a given signature for API +reasons (e.g. a callback that follows a given format) but doesn't use some of +its arguments; it's better to spell out in the argument what the API the +function implements is, instead of adding `_` variables. + +``` +local foo, bar = some_function() --luacheck: ignore 212/foo +print(bar) +``` + +* luacheck warning 542 (empty if branch) can also be ignored, when a sequence +of `if`/`elseif`/`else` blocks implements a "switch/case"-style list of cases, +and one of the cases is meant to mean "pass". For example: + +```lua +if warning >= 600 and warning <= 699 then + print("no whitespace warnings") +elseif warning == 542 then --luacheck: ignore 542 + -- pass +else + print("got a warning: "..warning) +end +``` + +> **Rationale:** This avoids writing negated conditions in the final fallback +case, and it's easy to add another case to the construct without having to +edit the fallback. + -- cgit v1.2.3 From ae2df5b05cac3568c34f61c173391c3097f90817 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Thu, 14 Mar 2019 16:18:00 +0000 Subject: Actually remove coding_style.txt --- doc/coding_style.txt | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 doc/coding_style.txt diff --git a/doc/coding_style.txt b/doc/coding_style.txt deleted file mode 100644 index af44da1a..00000000 --- a/doc/coding_style.txt +++ /dev/null @@ -1,33 +0,0 @@ -This file describes some coding styles to try and adhere to when contributing to this project. -Please try to follow, and feel free to fix code you see not following this standard. - -== Indentation == - - 1 tab indentation for all blocks - -== Spacing == - -No space between function names and parenthesis and parenthesis and parameters: - - function foo(bar, baz) - -Single space between braces and key/value pairs in table constructors: - - { foo = "bar", bar = "foo" } - -== Local variable naming == - -In this project there are many places where use of globals is restricted, and locals used for faster access. - -Local versions of standard functions should follow the below form: - - math.random -> m_random - string.char -> s_char - -== Miscellaneous == - -Single-statement blocks may be written on one line when short - - if foo then bar(); end - -'do' and 'then' keywords should be placed at the end of the line, and never on a line by themself. -- cgit v1.2.3 From ab78dfb2abc1cfa7c3119d0bad619da9e155c535 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 16 Mar 2019 18:43:11 +0100 Subject: configure: Separate flags related to compiler warnings This should make it more obvious that these are related --- configure | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure b/configure index dec7b60d..516ebaec 100755 --- a/configure +++ b/configure @@ -23,7 +23,8 @@ EXCERTS="yes" PRNG= PRNGLIBS= -CFLAGS="-fPIC -Wall -pedantic -std=c99" +CFLAGS="-fPIC -std=c99" +CFLAGS="$CFLAGS -Wall -pedantic" LDFLAGS="-shared" IDN_LIBRARY="idn" -- cgit v1.2.3 From 84be765efd340d27079e3524acf20df5bfb11f1e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 16 Mar 2019 18:51:02 +0100 Subject: configure: Enable more compiler warnings --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 516ebaec..1343d34b 100755 --- a/configure +++ b/configure @@ -24,7 +24,7 @@ PRNG= PRNGLIBS= CFLAGS="-fPIC -std=c99" -CFLAGS="$CFLAGS -Wall -pedantic" +CFLAGS="$CFLAGS -Wall -pedantic -Wextra -Wshadow -Wformat=2" LDFLAGS="-shared" IDN_LIBRARY="idn" -- cgit v1.2.3 From b4f63e714100773e131d69792239875583595c15 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 18 Mar 2019 09:50:23 +0000 Subject: MUC: Update error message for consistency --- plugins/muc/muc.lib.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua index a8d3d790..c828d17d 100644 --- a/plugins/muc/muc.lib.lua +++ b/plugins/muc/muc.lib.lua @@ -498,7 +498,7 @@ function room_mt:handle_normal_presence(origin, stanza) if orig_occupant == nil and not muc_x and stanza.attr.type == nil then module:log("debug", "Attempted join without , possibly desynced"); origin.send(st.error_reply(stanza, "cancel", "item-not-found", - "You must join the room before sending presence updates")); + "You are not currently connected to this chat")); return true; end -- cgit v1.2.3 From 8dbde146973dbdf875e1a5fc5a455218426f454c Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 17 Mar 2019 20:40:01 +0100 Subject: util.serialization: Optimize handling of last table separator Fewer next() calls and a step towards allowing use of a different iterator. --- util/serialization.lua | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/util/serialization.lua b/util/serialization.lua index 7ae77a3a..c64bfec1 100644 --- a/util/serialization.lua +++ b/util/serialization.lua @@ -163,7 +163,9 @@ local function new(opt) local indent = s_rep(indentwith, d); local numkey = 1; local ktyp, vtyp; + local had_items = false; for k,v in next,t do + had_items = true; o[l], l = itemstart, l + 1; o[l], l = indent, l + 1; ktyp, vtyp = type(k), type(v); @@ -194,14 +196,10 @@ local function new(opt) else o[l], l = ser(v), l + 1; end - -- last item? - if next(t, k) ~= nil then - o[l], l = itemsep, l + 1; - else - o[l], l = itemlast, l + 1; - end + o[l], l = itemsep, l + 1; end - if next(t) ~= nil then + if had_items then + o[l - 1] = itemlast; o[l], l = s_rep(indentwith, d-1), l + 1; end o[l], l = tend, l +1; -- cgit v1.2.3 From a335b6de27f7da7df46ec7a4091b6d1ea8ba2fba Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 17 Mar 2019 21:16:27 +0100 Subject: util.serialization: Allow overriding table iterator Could be useful to eg swap it out with sorted_pairs to get a stable serialization. Default to next() wrapper to avoid metatable tricks from pairs(). --- util/serialization.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/util/serialization.lua b/util/serialization.lua index c64bfec1..2ead8c12 100644 --- a/util/serialization.lua +++ b/util/serialization.lua @@ -33,6 +33,10 @@ local function to_hex(s) return (s_gsub(s, ".", char_to_hex)); end +local function rawpairs(t) + return next, t, nil; +end + local function fatal_error(obj, why) error("Can't serialize "..type(obj) .. (why and ": ".. why or "")); end @@ -122,6 +126,7 @@ local function new(opt) local freeze = opt.freeze; local maxdepth = opt.maxdepth or 127; local multirefs = opt.multiref; + local table_pairs = opt.table_iterator or rawpairs; -- serialize one table, recursively -- t - table being serialized @@ -164,7 +169,7 @@ local function new(opt) local numkey = 1; local ktyp, vtyp; local had_items = false; - for k,v in next,t do + for k,v in table_pairs(t) do had_items = true; o[l], l = itemstart, l + 1; o[l], l = indent, l + 1; -- cgit v1.2.3 From 8999b2a97c8d5e0d9dcf10d7692f928287d6bc5e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 17 Mar 2019 21:25:33 +0100 Subject: util.serialization: Use util.hex --- util/serialization.lua | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/util/serialization.lua b/util/serialization.lua index 2ead8c12..60e341cf 100644 --- a/util/serialization.lua +++ b/util/serialization.lua @@ -16,6 +16,8 @@ local s_char = string.char; local s_match = string.match; local t_concat = table.concat; +local to_hex = require "util.hex".to; + local pcall = pcall; local envload = require"util.envload".envload; @@ -24,15 +26,6 @@ local m_type = math.type or function (n) return n % 1 == 0 and n <= 9007199254740992 and n >= -9007199254740992 and "integer" or "float"; end; -local char_to_hex = {}; -for i = 0,255 do - char_to_hex[s_char(i)] = s_format("%02x", i); -end - -local function to_hex(s) - return (s_gsub(s, ".", char_to_hex)); -end - local function rawpairs(t) return next, t, nil; end -- cgit v1.2.3 From dda9c3ad7236b58675d66b7eb23bfb969f9c90d0 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 19 Mar 2019 09:04:40 +0000 Subject: moduleapi: New API for modules to set a status --- core/moduleapi.lua | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/core/moduleapi.lua b/core/moduleapi.lua index c6193cfd..2db7433a 100644 --- a/core/moduleapi.lua +++ b/core/moduleapi.lua @@ -17,6 +17,8 @@ local st = require "util.stanza"; local cache = require "util.cache"; local errutil = require "util.error"; local promise = require "util.promise"; +local time_now = require "util.time".now; +local format = require "util.format".format; local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat; local error, setmetatable, type = error, setmetatable, type; @@ -513,4 +515,33 @@ function api:measure_global_event(event_name, stat_name) return self:measure_object_event(prosody.events.wrappers, event_name, stat_name); end +local status_priorities = { error = 3, warn = 2, info = 1, core = 0 }; + +function api:set_status(status_type, status_message, override) + local priority = status_priorities[status_type]; + if not priority then + self:log("error", "set_status: Invalid status type '%s', assuming 'info'"); + status_type, priority = "info", status_priorities.info; + end + local current_priority = status_priorities[self.status_type] or 0; + -- By default an 'error' status can only be overwritten by another 'error' status + if (current_priority >= status_priorities.error and priority < current_priority and override ~= true) + or (override == false and current_priority > priority) then + self:log("debug", "Ignoring status"); + return; + end + self.status_type, self.status_message, self.status_time = status_type, status_message, time_now(); + self:log("debug", "New status: %s", status_type); + self:fire_event("module-status/updated", { name = self.name }); +end + +function api:log_status(level, msg, ...) + self:set_status(level, format(msg, ...)); + return self:log(level, msg, ...); +end + +function api:get_status() + return self.status_type, self.status_message, self.status_time; +end + return api; -- cgit v1.2.3 From 958b92cc3a56ad4cb4fe8d1549bf611df6d633e6 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 19 Mar 2019 09:05:15 +0000 Subject: modulemanager: Set module status on successful or failed module load --- core/modulemanager.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/modulemanager.lua b/core/modulemanager.lua index 17602459..0d24381a 100644 --- a/core/modulemanager.lua +++ b/core/modulemanager.lua @@ -169,6 +169,7 @@ local function do_load_module(host, module_name, state) local mod, err = pluginloader.load_code(module_name, nil, pluginenv); if not mod then log("error", "Unable to load module '%s': %s", module_name or "nil", err or "nil"); + api_instance:set_status("error", "Failed to load (see log)"); return nil, err; end @@ -182,6 +183,7 @@ local function do_load_module(host, module_name, state) ok, err = call_module_method(pluginenv, "load"); if not ok then log("warn", "Error loading module '%s' on '%s': %s", module_name, host, err or "nil"); + api_instance:set_status("warn", "Error during load (see log)"); end end api_instance.reloading, api_instance.saved_state = nil, nil; @@ -204,6 +206,9 @@ local function do_load_module(host, module_name, state) if not ok then modulemap[api_instance.host][module_name] = nil; log("error", "Error initializing module '%s' on '%s': %s", module_name, host, err or "nil"); + api_instance:set_status("warn", "Error during load (see log)"); + else + api_instance:set_status("core", "Loaded", false); end return ok and pluginenv, err; end -- cgit v1.2.3 From d684b288de7ac00a0e12795cc3fb8f02bfed1217 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 19 Mar 2019 09:05:37 +0000 Subject: mod_admin_telnet: Show module status in module:list() --- plugins/mod_admin_telnet.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 7fae8983..34cc4dc6 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -480,7 +480,12 @@ function def_env.module:list(hosts) end else for _, name in ipairs(modules) do - print(" "..name); + local status, status_text = modulemanager.get_module(host, name).module:get_status(); + local status_summary = ""; + if status == "warn" or status == "error" then + status_summary = (" (%s: %s)"):format(status, status_text); + end + print((" %s%s"):format(name, status_summary)); end end end -- cgit v1.2.3 From 75db23afc67254286b7d0bb94c8e5945f32c8068 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 19 Mar 2019 09:07:36 +0000 Subject: .luacheckrc: Update to reflect new module API methods --- .luacheckrc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.luacheckrc b/.luacheckrc index b2fa7cdb..f0cb93a8 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -34,7 +34,6 @@ files["plugins/"] = { "module.name", "module.host", "module._log", - "module.log", "module.event_handlers", "module.reloading", "module.saved_state", @@ -65,12 +64,15 @@ files["plugins/"] = { "module.get_option_scalar", "module.get_option_set", "module.get_option_string", + "module.get_status", "module.handle_items", "module.hook", "module.hook_global", "module.hook_object_event", "module.hook_tag", "module.load_resource", + "module.log", + "module.log_status", "module.measure", "module.measure_event", "module.measure_global_event", @@ -82,6 +84,7 @@ files["plugins/"] = { "module.send", "module.send_iq", "module.set_global", + "module.set_status", "module.shared", "module.unhook", "module.unhook_object_event", -- cgit v1.2.3 From 8189ab0bcdceea2c6ee2df4c1fb522242d8e1533 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 19 Mar 2019 09:08:06 +0000 Subject: mod_component: Set module status to indicate whether component is connected --- plugins/mod_component.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/mod_component.lua b/plugins/mod_component.lua index b8c87dee..b1ffc81d 100644 --- a/plugins/mod_component.lua +++ b/plugins/mod_component.lua @@ -49,6 +49,7 @@ function module.add_host(module) local send; local function on_destroy(session, err) --luacheck: ignore 212/err + module:set_status("warn", err and ("Disconnected: "..err) or "Disconnected"); env.connected = false; env.session = false; send = nil; @@ -102,6 +103,7 @@ function module.add_host(module) module:log("info", "External component successfully authenticated"); session.send(st.stanza("handshake")); module:fire_event("component-authenticated", { session = session }); + module:set_status("info", "Connected"); return true; end -- cgit v1.2.3 From 9fb58cf11f3de8f167a940c221d9a7b7a0c2da56 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 19 Mar 2019 09:08:33 +0000 Subject: mod_s2s: Set warning status if not listening on any ports --- plugins/mod_s2s/s2sout.lib.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_s2s/s2sout.lib.lua b/plugins/mod_s2s/s2sout.lib.lua index 5f765da8..34e322d2 100644 --- a/plugins/mod_s2s/s2sout.lib.lua +++ b/plugins/mod_s2s/s2sout.lib.lua @@ -318,7 +318,7 @@ module:hook_global("service-added", function (event) local s2s_sources = portmanager.get_active_services():get("s2s"); if not s2s_sources then - module:log("warn", "s2s not listening on any ports, outgoing connections may fail"); + module:log_status("warn", "s2s not listening on any ports, outgoing connections may fail"); return; end for source, _ in pairs(s2s_sources) do -- cgit v1.2.3 From 143c6bb57368837d9860a3f8a3678304689d5d46 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 19 Mar 2019 09:08:56 +0000 Subject: mod_muc_mam: Set error status if loaded on incorrect host type --- plugins/mod_muc_mam.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_muc_mam.lua b/plugins/mod_muc_mam.lua index d414a449..7d429482 100644 --- a/plugins/mod_muc_mam.lua +++ b/plugins/mod_muc_mam.lua @@ -4,7 +4,7 @@ -- This file is MIT/X11 licensed. if module:get_host_type() ~= "component" then - module:log("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name); + module:log_status("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name); return; end -- cgit v1.2.3 From 67b08fccf7d370c1df426890057d2775edd49797 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 20 Mar 2019 12:18:34 +0000 Subject: util.startup: Give function a more generic name so it can apply to all warnings --- util/startup.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/startup.lua b/util/startup.lua index c101c290..4d3c6e4e 100644 --- a/util/startup.lua +++ b/util/startup.lua @@ -96,7 +96,7 @@ function startup.init_logging() end); end -function startup.log_dependency_warnings() +function startup.log_startup_warnings() dependencies.log_warnings(); end @@ -518,7 +518,7 @@ function startup.prosodyctl() startup.read_version(); startup.switch_user(); startup.check_dependencies(); - startup.log_dependency_warnings(); + startup.log_startup_warnings(); startup.check_unwriteable(); startup.load_libraries(); startup.init_http_client(); @@ -543,7 +543,7 @@ function startup.prosody() startup.add_global_prosody_functions(); startup.read_version(); startup.log_greeting(); - startup.log_dependency_warnings(); + startup.log_startup_warnings(); startup.load_secondary_libraries(); startup.init_http_client(); startup.init_data_store(); -- cgit v1.2.3 From d4db66791dd4136374610889c68789ec3b7a186c Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 20 Mar 2019 12:19:43 +0000 Subject: configmanager: Add support for returning warnings --- core/configmanager.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/configmanager.lua b/core/configmanager.lua index 1e67da9b..579db3b0 100644 --- a/core/configmanager.lua +++ b/core/configmanager.lua @@ -9,7 +9,7 @@ local _G = _G; local setmetatable, rawget, rawset, io, os, error, dofile, type, pairs = setmetatable, rawget, rawset, io, os, error, dofile, type, pairs; -local format, math_max = string.format, math.max; +local format, math_max, t_insert = string.format, math.max, table.insert; local envload = require"util.envload".envload; local deps = require"util.dependencies"; @@ -102,6 +102,7 @@ do local pcall = _G.pcall; parser = {}; function parser.load(data, config_file, config_table) + local warnings = {}; local env; -- The ' = true' are needed so as not to set off __newindex when we assign the functions below env = setmetatable({ @@ -217,7 +218,7 @@ do return nil, err; end - return true; + return true, warnings; end end -- cgit v1.2.3 From 6a11648a077700aa5efedd3e300f370c8a95f665 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 20 Mar 2019 12:20:51 +0000 Subject: configmanager: Emit warning for duplicated config options --- core/configmanager.lua | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/core/configmanager.lua b/core/configmanager.lua index 579db3b0..41034df8 100644 --- a/core/configmanager.lua +++ b/core/configmanager.lua @@ -16,6 +16,7 @@ local deps = require"util.dependencies"; local resolve_relative_path = require"util.paths".resolve_relative_path; local glob_to_pattern = require"util.paths".glob_to_pattern; local path_sep = package.config:sub(1,1); +local get_traceback_table = require "util.debug".get_traceback_table; local encodings = deps.softreq"util.encodings"; local nameprep = encodings and encodings.stringprep.nameprep or function (host) return host:lower(); end @@ -100,8 +101,17 @@ end -- Built-in Lua parser do local pcall = _G.pcall; + local function get_line_number(config_file) + local tb = get_traceback_table(nil, 2); + for i = 1, #tb do + if tb[i].info.short_src == config_file then + return tb[i].info.currentline; + end + end + end parser = {}; function parser.load(data, config_file, config_table) + local set_options = {}; -- set_options[host.."/"..option_name] = true (when the option has been set already in this file) local warnings = {}; local env; -- The ' = true' are needed so as not to set off __newindex when we assign the functions below @@ -116,6 +126,12 @@ do return rawget(_G, k); end, __newindex = function (_, k, v) + local host = env.__currenthost or "*"; + local option_path = host.."/"..k; + if set_options[option_path] then + t_insert(warnings, ("%s:%d: Duplicate option '%s'"):format(config_file, get_line_number(config_file), k)); + end + set_options[option_path] = true; set(config_table, env.__currenthost or "*", k, v); end }); -- cgit v1.2.3 From eb63c91d91f503946fe9aa731d975f0b289e6065 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 20 Mar 2019 12:45:08 +0000 Subject: configmanager: Pass through warnings from included files --- core/configmanager.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/configmanager.lua b/core/configmanager.lua index 41034df8..090a6a0a 100644 --- a/core/configmanager.lua +++ b/core/configmanager.lua @@ -7,8 +7,8 @@ -- local _G = _G; -local setmetatable, rawget, rawset, io, os, error, dofile, type, pairs = - setmetatable, rawget, rawset, io, os, error, dofile, type, pairs; +local setmetatable, rawget, rawset, io, os, error, dofile, type, pairs, ipairs = + setmetatable, rawget, rawset, io, os, error, dofile, type, pairs, ipairs; local format, math_max, t_insert = string.format, math.max, table.insert; local envload = require"util.envload".envload; @@ -212,6 +212,11 @@ do if f then local ret, err = parser.load(f:read("*a"), file, config_table); if not ret then error(err:gsub("%[string.-%]", file), 0); end + if err then + for _, warning in ipairs(err) do + t_insert(warnings, warning); + end + end end if not f then error("Error loading included "..file..": "..err, 0); end return f, err; -- cgit v1.2.3 From 798be44f563b8764dad50e893f4b25268b42e5a8 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 20 Mar 2019 12:45:58 +0000 Subject: util.startup: Log configuration warnings at startup --- util/startup.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/util/startup.lua b/util/startup.lua index 4d3c6e4e..966f2934 100644 --- a/util/startup.lua +++ b/util/startup.lua @@ -7,6 +7,7 @@ local logger = require "util.logger"; local log = logger.init("startup"); local config = require "core.configmanager"; +local config_warnings; local dependencies = require "util.dependencies"; @@ -64,6 +65,8 @@ function startup.read_config() print("**************************"); print(""); os.exit(1); + elseif err and #err > 0 then + config_warnings = err; end prosody.config_loaded = true; end @@ -98,6 +101,9 @@ end function startup.log_startup_warnings() dependencies.log_warnings(); + for _, warning in ipairs(config_warnings) do + log("warn", "Configuration warning: %s", warning); + end end function startup.sanity_check() -- cgit v1.2.3 From c77d416cfffd6604cc4d9a633f9f7941f6fd5f10 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 20 Mar 2019 13:44:29 +0000 Subject: util.startup: Don't die if there are no config warnings to log (thanks buildbot) --- util/startup.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/util/startup.lua b/util/startup.lua index 966f2934..7a1a95aa 100644 --- a/util/startup.lua +++ b/util/startup.lua @@ -101,8 +101,10 @@ end function startup.log_startup_warnings() dependencies.log_warnings(); - for _, warning in ipairs(config_warnings) do - log("warn", "Configuration warning: %s", warning); + if config_warnings then + for _, warning in ipairs(config_warnings) do + log("warn", "Configuration warning: %s", warning); + end end end -- cgit v1.2.3 From 8aef0281ab608e1b1d421673f924ff91d439097d Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Fri, 20 Oct 2017 12:53:53 +0200 Subject: mod_storage_internal,_sql: Add limit to number of items in an archive store (fixes #733) --- plugins/mod_storage_internal.lua | 35 +++++++++++++++++++++++++++++++++++ plugins/mod_storage_sql.lua | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index 42b451bd..d812c0e9 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -1,3 +1,4 @@ +local cache = require "util.cache"; local datamanager = require "core.storagemanager".olddm; local array = require "util.array"; local datetime = require "util.datetime"; @@ -7,6 +8,9 @@ local id = require "util.id".medium; local host = module.host; +local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000); +local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000)); + local driver = {}; function driver:open(store, typ) @@ -54,28 +58,56 @@ function archive:append(username, key, value, when, with) value.attr.stamp = datetime.datetime(when); value.attr.stamp_legacy = datetime.legacy(when); + local item_count = archive_item_count_cache:get(username); + if key then local items, err = datamanager.list_load(username, host, self.store); if not items and err then return items, err; end + + -- Check the quota + item_count = items and #items or 0; + archive_item_count_cache:set(username, item_count); + if item_count >= archive_item_limit then + module:log("debug", "%s reached or over quota, not adding to store", username); + return nil, "quota-limit"; + end + if items then + -- Filter out any item with the same key as the one being added items = array(items); items:filter(function (item) return item.key ~= key; end); + value.key = key; items:push(value); local ok, err = datamanager.list_store(username, host, self.store, items); if not ok then return ok, err; end + archive_item_count_cache:set(username, #items); return key; end else + if not item_count then -- Item count not cached? + -- We need to load the list to get the number of items currently stored + local items, err = datamanager.list_load(username, host, self.store); + if not items and err then return items, err; end + item_count = items and #items or 0; + archive_item_count_cache:set(username, item_count); + end + if item_count >= archive_item_limit then + module:log("debug", "%s reached or over quota, not adding to store", username); + return nil, "quota-limit"; + end key = id(); end + module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit); + value.key = key; local ok, err = datamanager.list_append(username, host, self.store, value); if not ok then return ok, err; end + archive_item_count_cache:set(username, item_count+1); return key; end @@ -158,6 +190,7 @@ end function archive:delete(username, query) if not query or next(query) == nil then + archive_item_count_cache:set(username, nil); return datamanager.list_store(username, host, self.store, nil); end local items, err = datamanager.list_load(username, host, self.store); @@ -165,6 +198,7 @@ function archive:delete(username, query) if err then return items, err; end + archive_item_count_cache:set(username, 0); -- Store is empty return 0; end @@ -214,6 +248,7 @@ function archive:delete(username, query) end local ok, err = datamanager.list_store(username, host, self.store, items); if not ok then return ok, err; end + archive_item_count_cache:set(username, #items); return count; end diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 5c0c0208..4fe2a262 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -1,6 +1,7 @@ -- luacheck: ignore 212/self +local cache = require "util.cache"; local json = require "util.json"; local sql = require "util.sql"; local xml_parse = require "util.xml".parse; @@ -148,6 +149,9 @@ end --- Archive store API +local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000); +local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000)); + -- luacheck: ignore 512 431/user 431/store local map_store = {}; map_store.__index = map_store; @@ -231,6 +235,32 @@ archive_store.caps = { }; archive_store.__index = archive_store function archive_store:append(username, key, value, when, with) + local item_count = archive_item_count_cache:get(username); + if not item_count then + local ok, ret = engine:transaction(function() + local count_sql = [[ + SELECT COUNT(*) FROM "prosodyarchive" + WHERE "host"=? AND "user"=? AND "store"=?; + ]]; + local result = engine:select(count_sql, host, user, store); + if result then + for row in result do + item_count = row[1]; + end + end + end); + if not ok or not item_count then + module:log("error", "Failed while checking quota for %s: %s", username, ret); + return nil, "Failure while checking quota"; + end + archive_item_count_cache:set(username, item_count); + end + + module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit); + if item_count >= archive_item_limit then + return nil, "quota-limit"; + end + local user,store = username,self.store; when = when or os.time(); with = with or ""; @@ -245,12 +275,18 @@ function archive_store:append(username, key, value, when, with) VALUES (?,?,?,?,?,?,?,?); ]]; if key then - engine:delete(delete_sql, host, user or "", store, key); + local result, err = engine:delete(delete_sql, host, user or "", store, key); + if result then + item_count = item_count - result:affected(); + archive_item_count_cache:set(username, item_count); + end else + item_count = item_count + 1; key = uuid.generate(); end local t, encoded_value = assert(serialize(value)); engine:insert(insert_sql, host, user or "", store, when, with, key, t, encoded_value); + archive_item_count_cache:set(username, item_count+1); return key; end); if not ok then return ok, ret; end @@ -422,6 +458,7 @@ function archive_store:delete(username, query) end return engine:delete(sql_query, unpack(args)); end); + archive_item_count_cache:set(username, nil); return ok and stmt:affected(), stmt; end -- cgit v1.2.3 From 351783fd1d132fea8098aa37084258afe81afccf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 18:01:24 +0100 Subject: mod_storage_internal,_sql: Expose archive capabilities feature set This was planned to be added long ago but was forgotten. --- plugins/mod_storage_internal.lua | 6 ++++++ plugins/mod_storage_sql.lua | 2 ++ 2 files changed, 8 insertions(+) diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index d812c0e9..c21fe0dc 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -47,6 +47,12 @@ end local archive = {}; driver.archive = { __index = archive }; +archive.caps = { + total = true; + quota = archive_item_limit; + truncate = true; +}; + function archive:append(username, key, value, when, with) when = when or now(); if not st.is_stanza(value) then diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 4fe2a262..82c5c3fe 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -232,6 +232,8 @@ end local archive_store = {} archive_store.caps = { total = true; + quota = archive_item_limit; + truncate = true; }; archive_store.__index = archive_store function archive_store:append(username, key, value, when, with) -- cgit v1.2.3 From 8c6c1d0fd0fa2f907b0821954b7aa03a5fe2618d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 18:02:27 +0100 Subject: mod_storage_internal,_sql: Key item count cache on both username and store --- plugins/mod_storage_internal.lua | 19 +++++++++++-------- plugins/mod_storage_sql.lua | 13 ++++++++----- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index c21fe0dc..52ca4da8 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -5,6 +5,7 @@ local datetime = require "util.datetime"; local st = require "util.stanza"; local now = require "util.time".now; local id = require "util.id".medium; +local jid_join = require "util.jid".join; local host = module.host; @@ -64,7 +65,8 @@ function archive:append(username, key, value, when, with) value.attr.stamp = datetime.datetime(when); value.attr.stamp_legacy = datetime.legacy(when); - local item_count = archive_item_count_cache:get(username); + local cache_key = jid_join(username, host, self.store); + local item_count = archive_item_count_cache:get(cache_key); if key then local items, err = datamanager.list_load(username, host, self.store); @@ -72,7 +74,7 @@ function archive:append(username, key, value, when, with) -- Check the quota item_count = items and #items or 0; - archive_item_count_cache:set(username, item_count); + archive_item_count_cache:set(cache_key, item_count); if item_count >= archive_item_limit then module:log("debug", "%s reached or over quota, not adding to store", username); return nil, "quota-limit"; @@ -89,7 +91,7 @@ function archive:append(username, key, value, when, with) items:push(value); local ok, err = datamanager.list_store(username, host, self.store, items); if not ok then return ok, err; end - archive_item_count_cache:set(username, #items); + archive_item_count_cache:set(cache_key, #items); return key; end else @@ -98,7 +100,7 @@ function archive:append(username, key, value, when, with) local items, err = datamanager.list_load(username, host, self.store); if not items and err then return items, err; end item_count = items and #items or 0; - archive_item_count_cache:set(username, item_count); + archive_item_count_cache:set(cache_key, item_count); end if item_count >= archive_item_limit then module:log("debug", "%s reached or over quota, not adding to store", username); @@ -113,7 +115,7 @@ function archive:append(username, key, value, when, with) local ok, err = datamanager.list_append(username, host, self.store, value); if not ok then return ok, err; end - archive_item_count_cache:set(username, item_count+1); + archive_item_count_cache:set(cache_key, item_count+1); return key; end @@ -195,8 +197,9 @@ function archive:dates(username) end function archive:delete(username, query) + local cache_key = jid_join(username, host, self.store); if not query or next(query) == nil then - archive_item_count_cache:set(username, nil); + archive_item_count_cache:set(cache_key, nil); return datamanager.list_store(username, host, self.store, nil); end local items, err = datamanager.list_load(username, host, self.store); @@ -204,7 +207,7 @@ function archive:delete(username, query) if err then return items, err; end - archive_item_count_cache:set(username, 0); + archive_item_count_cache:set(cache_key, 0); -- Store is empty return 0; end @@ -254,7 +257,7 @@ function archive:delete(username, query) end local ok, err = datamanager.list_store(username, host, self.store, items); if not ok then return ok, err; end - archive_item_count_cache:set(username, #items); + archive_item_count_cache:set(cache_key, #items); return count; end diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 82c5c3fe..3028bb72 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -7,6 +7,7 @@ local sql = require "util.sql"; local xml_parse = require "util.xml".parse; local uuid = require "util.uuid"; local resolve_relative_path = require "util.paths".resolve_relative_path; +local jid_join = require "util.jid".join; local is_stanza = require"util.stanza".is_stanza; local t_concat = table.concat; @@ -237,7 +238,8 @@ archive_store.caps = { }; archive_store.__index = archive_store function archive_store:append(username, key, value, when, with) - local item_count = archive_item_count_cache:get(username); + local cache_key = jid_join(username, host, self.store); + local item_count = archive_item_count_cache:get(cache_key); if not item_count then local ok, ret = engine:transaction(function() local count_sql = [[ @@ -255,7 +257,7 @@ function archive_store:append(username, key, value, when, with) module:log("error", "Failed while checking quota for %s: %s", username, ret); return nil, "Failure while checking quota"; end - archive_item_count_cache:set(username, item_count); + archive_item_count_cache:set(cache_key, item_count); end module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit); @@ -280,7 +282,7 @@ function archive_store:append(username, key, value, when, with) local result, err = engine:delete(delete_sql, host, user or "", store, key); if result then item_count = item_count - result:affected(); - archive_item_count_cache:set(username, item_count); + archive_item_count_cache:set(cache_key, item_count); end else item_count = item_count + 1; @@ -288,7 +290,7 @@ function archive_store:append(username, key, value, when, with) end local t, encoded_value = assert(serialize(value)); engine:insert(insert_sql, host, user or "", store, when, with, key, t, encoded_value); - archive_item_count_cache:set(username, item_count+1); + archive_item_count_cache:set(cache_key, item_count+1); return key; end); if not ok then return ok, ret; end @@ -460,7 +462,8 @@ function archive_store:delete(username, query) end return engine:delete(sql_query, unpack(args)); end); - archive_item_count_cache:set(username, nil); + local cache_key = jid_join(username, host, self.store); + archive_item_count_cache:set(cache_key, nil); return ok and stmt:affected(), stmt; end -- cgit v1.2.3 From db917661a6946e25b53b38b79ba7c8f4f01e50db Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 7 Nov 2017 18:58:52 +0100 Subject: mod_mam: Trim archive when quota has been exceeded --- plugins/mod_mam/mod_mam.lua | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 88cbaa08..5be4bc24 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -40,6 +40,10 @@ local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://ja local archive_store = module:get_option_string("archive_store", "archive"); local archive = module:open_store(archive_store, "archive"); +local cleanup_after = module:get_option_string("archive_expires_after", "1w"); +local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60); +local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000); + if not archive.find then error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n" .."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information"); @@ -295,7 +299,20 @@ local function message_handler(event, c2s) log("debug", "Archiving stanza: %s", stanza:top_tag()); -- And stash it - local ok = archive:append(store_user, nil, clone_for_storage, time_now(), with); + local time = time_now(); + local ok, err = archive:append(store_user, nil, clone_for_storage, time, with); + if not ok and err == "quota-limit" then + if archive.caps and archive.caps.truncate then + module:log("debug", "User '%s' over quota, trimming archive", store_user); + local truncated = archive:delete(store_user, { + truncate = archive_item_limit - 1; + ["end"] = type(cleanup_after) == "number" and (os.time() - cleanup_after) or nil; + }); + if truncated then + ok, err = archive:append(store_user, nil, clone_for_storage, time, with); + end + end + end if ok then local clone_for_other_handlers = st.clone(stanza); local id = ok; @@ -321,8 +338,6 @@ end module:hook("pre-message/bare", strip_stanza_id_after_other_events, -1); module:hook("pre-message/full", strip_stanza_id_after_other_events, -1); -local cleanup_after = module:get_option_string("archive_expires_after", "1w"); -local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60); if cleanup_after ~= "never" then local cleanup_storage = module:open_store("archive_cleanup"); local cleanup_map = module:open_store("archive_cleanup", "map"); -- cgit v1.2.3 From d1ad7d498828b48d2a2477319083227b91b5441d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 20 Mar 2019 12:14:45 +0100 Subject: mod_storage_memory: Add support for archive item limits --- plugins/mod_storage_memory.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua index 745e394b..8e1cf879 100644 --- a/plugins/mod_storage_memory.lua +++ b/plugins/mod_storage_memory.lua @@ -8,6 +8,8 @@ local new_id = require "util.id".medium; local auto_purge_enabled = module:get_option_boolean("storage_memory_temporary", false); local auto_purge_stores = module:get_option_set("storage_memory_temporary_stores", {}); +local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000); + local memory = setmetatable({}, { __index = function(t, k) local store = module:shared(k) @@ -51,6 +53,12 @@ archive_store.__index = archive_store; archive_store.users = _users; +archive_store.caps = { + total = true; + quota = archive_item_limit; + truncate = true; +}; + function archive_store:append(username, key, value, when, with) if is_stanza(value) then value = st.preserialize(value); @@ -70,6 +78,8 @@ function archive_store:append(username, key, value, when, with) end if a[key] then table.remove(a, a[key]); + elseif #a >= archive_item_limit then + return nil, "quota-limit"; end local i = #a+1; a[i] = v; -- cgit v1.2.3 From 92595546031e1f8a6222e9e164f8cca45e949bdf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:04:34 +0100 Subject: mod_storage_internal: Increase default quota to 10 000 Performance doesn't seem great but 10k should be far enough from limits inherited by the Lua parser. 1000 messages seemed pretty close to what an active user might produce in one week. --- plugins/mod_storage_internal.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index 52ca4da8..cb88f10f 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -9,7 +9,7 @@ local jid_join = require "util.jid".join; local host = module.host; -local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000); +local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000); local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000)); local driver = {}; -- cgit v1.2.3 From 7e9722a7362c9e7ffd6cca6c756ee61acc52ff93 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:09:38 +0100 Subject: mod_storage_sql: Don't increment counter twice (fixes accounting error) --- plugins/mod_storage_sql.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 3028bb72..154daf06 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -285,7 +285,6 @@ function archive_store:append(username, key, value, when, with) archive_item_count_cache:set(cache_key, item_count); end else - item_count = item_count + 1; key = uuid.generate(); end local t, encoded_value = assert(serialize(value)); -- cgit v1.2.3 From 669da78e01c61d83912210877b2c5514360fe7e9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:10:46 +0100 Subject: mod_storage_sql: Fix to use currently queried store Was using the previously queried store due to this being cached in an upvalue. --- plugins/mod_storage_sql.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 154daf06..325fab94 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -238,7 +238,8 @@ archive_store.caps = { }; archive_store.__index = archive_store function archive_store:append(username, key, value, when, with) - local cache_key = jid_join(username, host, self.store); + local user,store = username,self.store; + local cache_key = jid_join(username, host, store); local item_count = archive_item_count_cache:get(cache_key); if not item_count then local ok, ret = engine:transaction(function() @@ -265,7 +266,6 @@ function archive_store:append(username, key, value, when, with) return nil, "quota-limit"; end - local user,store = username,self.store; when = when or os.time(); with = with or ""; local ok, ret = engine:transaction(function() -- cgit v1.2.3 From 79bebb21f83365e33208644f1c0060752f4fc532 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:12:02 +0100 Subject: mod_storage_sql: Skip cache write This would cause the cache to be wrong in case the the later INSERT fails and the transaction is aborted. --- plugins/mod_storage_sql.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 325fab94..35a16870 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -282,7 +282,6 @@ function archive_store:append(username, key, value, when, with) local result, err = engine:delete(delete_sql, host, user or "", store, key); if result then item_count = item_count - result:affected(); - archive_item_count_cache:set(cache_key, item_count); end else key = uuid.generate(); -- cgit v1.2.3 From e21f31c7c282aa5fba3c37e6d1e829edb3b78a6b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:13:27 +0100 Subject: mod_storage_sql: Cache total count if it's calculated as part of the current query --- plugins/mod_storage_sql.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 35a16870..b3bd5171 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -384,6 +384,9 @@ function archive_store:find(username, query) total = row[1]; end end + if query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then + archive_item_count_cache:set(cache_key, total); + end if query.limit == 0 then -- Skip the real query return noop, total; end -- cgit v1.2.3 From 2a073294ae4a6eccc10ffcef2a2146bfb9abe1ae Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:18:54 +0100 Subject: mod_storage_sql: Return cached count if only this is queried for --- plugins/mod_storage_sql.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index b3bd5171..8c03da01 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -362,7 +362,11 @@ end function archive_store:find(username, query) query = query or {}; local user,store = username,self.store; - local total; + local cache_key = jid_join(username, host, self.store); + local total = archive_item_count_cache:get(cache_key); + if total ~= nil and query.limit == 0 and query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then + return noop, total; + end local ok, result = engine:transaction(function() local sql_query = [[ SELECT "key", "type", "value", "when", "with" -- cgit v1.2.3 From 36900805270064f089c7e3ae9b2e3be6b7a63df0 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:22:21 +0100 Subject: mod_mam: On quota hit, separately delete by time and by item count This is to work around a possible SQL issue where offsets and time stamps don't interact correctly. --- plugins/mod_mam/mod_mam.lua | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 5be4bc24..632de9ea 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -302,11 +302,19 @@ local function message_handler(event, c2s) local time = time_now(); local ok, err = archive:append(store_user, nil, clone_for_storage, time, with); if not ok and err == "quota-limit" then - if archive.caps and archive.caps.truncate then - module:log("debug", "User '%s' over quota, trimming archive", store_user); + if type(cleanup_after) == "number" then + module:log("debug", "User '%s' over quota, cleaning archive", store_user); + local cleaned = archive:delete(store_user, { + ["end"] = (os.time() - cleanup_after); + }); + if cleaned then + ok, err = archive:append(store_user, nil, clone_for_storage, time, with); + end + end + if not ok and (archive.caps and archive.caps.truncate) then + module:log("debug", "User '%s' over quota, truncating archive", store_user); local truncated = archive:delete(store_user, { truncate = archive_item_limit - 1; - ["end"] = type(cleanup_after) == "number" and (os.time() - cleanup_after) or nil; }); if truncated then ok, err = archive:append(store_user, nil, clone_for_storage, time, with); -- cgit v1.2.3 From a763f1f8235b4cc9633847203d3a282871ebfca4 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 02:24:48 +0100 Subject: mod_storage_internal: Include store name when reporting quota status --- plugins/mod_storage_internal.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index cb88f10f..c87d01be 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -109,7 +109,7 @@ function archive:append(username, key, value, when, with) key = id(); end - module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit); + module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store); value.key = key; -- cgit v1.2.3 From 1a68e66b842a4af6fe19e99f2e09044a540b404e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 22 Mar 2019 16:30:53 +0100 Subject: mod_storage_sql: No archive item limit by default --- plugins/mod_storage_sql.lua | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 8c03da01..6b26759f 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -150,7 +150,7 @@ end --- Archive store API -local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000); +local archive_item_limit = module:get_option_number("storage_archive_item_limit"); local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000)); -- luacheck: ignore 512 431/user 431/store @@ -261,9 +261,11 @@ function archive_store:append(username, key, value, when, with) archive_item_count_cache:set(cache_key, item_count); end - module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit); - if item_count >= archive_item_limit then - return nil, "quota-limit"; + if archive_item_limit then + module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit); + if item_count >= archive_item_limit then + return nil, "quota-limit"; + end end when = when or os.time(); -- cgit v1.2.3 From 49b2878157c4ed51205580848642058e53acdb59 Mon Sep 17 00:00:00 2001 From: Maxime ?pep? Buquet Date: Sat, 23 Mar 2019 01:57:12 +0000 Subject: net/server_event: fix typo in comment --- net/server_event.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/server_event.lua b/net/server_event.lua index 2bee614a..42c9af2e 100644 --- a/net/server_event.lua +++ b/net/server_event.lua @@ -658,7 +658,7 @@ local function handleclient( client, ip, port, server, pattern, listener, sslctx return interface end -local function handleserver( server, addr, port, pattern, listener, sslctx, startssl ) -- creates an server interface +local function handleserver( server, addr, port, pattern, listener, sslctx, startssl ) -- creates a server interface debug "creating server interface..." local interface = { _connections = 0; -- cgit v1.2.3 From 6d02c463f7436544bfce141a7f36bfda417d497f Mon Sep 17 00:00:00 2001 From: Maxime ?pep? Buquet Date: Sat, 23 Mar 2019 02:27:45 +0000 Subject: doc/coding_style: remove superfulous bracket in example --- doc/coding_style.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/coding_style.md b/doc/coding_style.md index 6f34d371..af1a2502 100644 --- a/doc/coding_style.md +++ b/doc/coding_style.md @@ -131,7 +131,6 @@ constants from C. ```lua local player = { name = "Jack", class = "Rogue" } -} ``` * Items should be separated by commas. If there are many items, put each -- cgit v1.2.3 From fc06f69a4f3b22c97cdca1114cca191aa5e259ea Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 23 Mar 2019 03:56:55 +0100 Subject: doc/coding_style: Trim trailing whitespace --- doc/coding_style.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/coding_style.md b/doc/coding_style.md index af1a2502..17b7c037 100644 --- a/doc/coding_style.md +++ b/doc/coding_style.md @@ -330,7 +330,7 @@ than if it says `check_version = function()` under some indentation level. ## Variable declaration -* Always use `local` to declare variables. +* Always use `local` to declare variables. ```lua -- bad @@ -446,8 +446,8 @@ if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == -- good if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == 10 then - do_other_complicated_function() - return false + do_other_complicated_function() + return false end ``` @@ -466,7 +466,7 @@ b = 2 ## Spacing -* Use a space after `--`. +* Use a space after `--`. ```lua --bad -- cgit v1.2.3 From 268e487245b73baee4f8d8add6d12707d278f371 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 23 Mar 2019 04:00:55 +0100 Subject: doc/coding_style: The codebase uses semicolons --- doc/coding_style.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/coding_style.md b/doc/coding_style.md index 17b7c037..6ca527fa 100644 --- a/doc/coding_style.md +++ b/doc/coding_style.md @@ -451,17 +451,17 @@ if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == end ``` -* Separate statements onto multiple lines. Do not use semicolons as statement terminators. +* Separate statements onto multiple lines. Use semicolons as statement terminators. ```lua -- bad -local whatever = "sure"; -a = 1; b = 2 +local whatever = "sure" +a = 1 b = 2 -- good -local whatever = "sure" -a = 1 -b = 2 +local whatever = "sure"; +a = 1; +b = 2; ``` ## Spacing -- cgit v1.2.3 From 2080433daf5a956d0456c6ce24ce1ef9183a023e Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Sat, 23 Mar 2019 08:47:55 +0000 Subject: util.queue: Add 'consume()' convenience iterator --- spec/util_queue_spec.lua | 37 +++++++++++++++++++++++++++++++++++++ util/queue.lua | 3 +++ 2 files changed, 40 insertions(+) diff --git a/spec/util_queue_spec.lua b/spec/util_queue_spec.lua index 7cd3d695..d73f523d 100644 --- a/spec/util_queue_spec.lua +++ b/spec/util_queue_spec.lua @@ -100,4 +100,41 @@ describe("util.queue", function() end); end); + describe("consume()", function () + it("should work", function () + local q = queue.new(10); + for i = 1, 5 do + q:push(i); + end + local c = 0; + for i in q:consume() do + assert(i == c + 1); + assert(q:count() == (5-i)); + c = i; + end + end); + + it("should work even if items are pushed in the loop", function () + local q = queue.new(10); + for i = 1, 5 do + q:push(i); + end + local c = 0; + for i in q:consume() do + assert(i == c + 1); + if c < 3 then + assert(q:count() == (5-i)); + else + assert(q:count() == (6-i)); + end + + c = i; + + if c == 3 then + q:push(6); + end + end + assert.equal(c, 6); + end); + end); end); diff --git a/util/queue.lua b/util/queue.lua index 728e905f..e63b3f1c 100644 --- a/util/queue.lua +++ b/util/queue.lua @@ -64,6 +64,9 @@ local function new(size, allow_wrapping) return pos+1, t._items[read_pos]; end, self, 0; end; + consume = function (self) + return self.pop, self; + end; }; end -- cgit v1.2.3 From bd52e2269b0fff09fc8aea6c6807a9f08727bb31 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Sat, 23 Mar 2019 08:52:57 +0000 Subject: util.queue: Update :items() to consistently use private data directly It will perform better this way, and we were accessing private variables already within the iterator. --- core/loggingmanager.lua | 21 ++++++++++++++++++++- util/queue.lua | 9 ++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/core/loggingmanager.lua b/core/loggingmanager.lua index cfa8246a..b510617f 100644 --- a/core/loggingmanager.lua +++ b/core/loggingmanager.lua @@ -18,6 +18,9 @@ local getstyle, getstring = require "util.termcolours".getstyle, require "util.t local config = require "core.configmanager"; local logger = require "util.logger"; +local have_pposix, pposix = pcall(require, "util.pposix"); +have_pposix = have_pposix and pposix._VERSION == "0.4.4"; + local _ENV = nil; -- luacheck: std none @@ -45,7 +48,8 @@ local function add_rule(sink_config) local sink = sink_maker(sink_config); -- Set sink for all chosen levels - for level in pairs(get_levels(sink_config.levels or logging_levels)) do + local levels = get_levels(sink_config.levels or logging_levels); + for level in pairs(levels) do logger.add_level_sink(level, sink); end end @@ -232,6 +236,21 @@ local function log_to_console(sink_config) end log_sink_types.console = log_to_console; +if have_pposix then + local syslog_opened; + local function log_to_syslog(sink_config) -- luacheck: ignore 212/sink_config + if not syslog_opened then + pposix.syslog_open(sink_config.syslog_name or "prosody", sink_config.syslog_facility or config.get("*", "syslog_facility")); + syslog_opened = true; + end + local syslog = pposix.syslog_log; + return function (name, level, message, ...) + syslog(level, name, format(message, ...)); + end; + end + log_sink_types.syslog = log_to_syslog; +end + local function register_sink_type(name, sink_maker) local old_sink_maker = log_sink_types[name]; log_sink_types[name] = sink_maker; diff --git a/util/queue.lua b/util/queue.lua index e63b3f1c..66ed098b 100644 --- a/util/queue.lua +++ b/util/queue.lua @@ -52,16 +52,15 @@ local function new(size, allow_wrapping) return t[tail]; end; items = function (self) - --luacheck: ignore 431/t - return function (t, pos) - if pos >= t:count() then + return function (_, pos) + if pos >= items then return nil; end local read_pos = tail + pos; - if read_pos > t.size then + if read_pos > self.size then read_pos = (read_pos%size); end - return pos+1, t._items[read_pos]; + return pos+1, t[read_pos]; end, self, 0; end; consume = function (self) -- cgit v1.2.3 From c1a1511e2662fdd1749c30cfc1cabc3d4b89f082 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 25 Feb 2019 15:48:28 +0100 Subject: mod_storage_internal: Implement a summary API returning message counts per contact --- doc/storage.tld | 3 +++ plugins/mod_storage_internal.lua | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/doc/storage.tld b/doc/storage.tld index f1d33e58..057649a4 100644 --- a/doc/storage.tld +++ b/doc/storage.tld @@ -47,6 +47,9 @@ interface archive_store -- Array of dates which do have messages (Optional?) dates : ( self, string? ) -> ({ string }) | (nil, string) + + -- Map of counts per "with" field + summary : ( self, string?, archive_query? ) -> ( { string : integer } ) | (nil, string) end -- This represents moduleapi diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index c87d01be..aa5c3c8a 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -196,6 +196,16 @@ function archive:dates(username) return array(items):pluck("when"):map(datetime.date):unique(); end +function archive:summary(username, query) + local iter, err = self:find(username, query) + if not iter then return iter, err; end + local summary = {}; + for _, _, _, with in iter do + summary[with] = (summary[with] or 0) + 1; + end + return summary; +end + function archive:delete(username, query) local cache_key = jid_join(username, host, self.store); if not query or next(query) == nil then -- cgit v1.2.3 From aea959880cf8ae4d81c41d4e6ef224cc978ef278 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 25 Feb 2019 15:51:55 +0100 Subject: mod_storage_sql: Implement archive summary API --- plugins/mod_storage_sql.lua | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 6b26759f..ffe48ab8 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -419,6 +419,41 @@ function archive_store:find(username, query) end, total; end +function archive_store:summary(username, query) + query = query or {}; + local user,store = username,self.store; + local ok, result = engine:transaction(function() + local sql_query = [[ + SELECT DISTINCT "with", COUNT(*) + FROM "prosodyarchive" + WHERE %s + GROUP BY "with" + ORDER BY "sort_id" %s%s; + ]]; + local args = { host, user or "", store, }; + local where = { "\"host\" = ?", "\"user\" = ?", "\"store\" = ?", }; + + archive_where(query, args, where); + + archive_where_id_range(query, args, where); + + if query.limit then + args[#args+1] = query.limit; + end + + sql_query = sql_query:format(t_concat(where, " AND "), query.reverse + and "DESC" or "ASC", query.limit and " LIMIT ?" or ""); + return engine:select(sql_query, unpack(args)); + end); + if not ok then return ok, result end + local summary = {}; + for row in result do + local with, count = row[1], row[2]; + summary[with] = count; + end + return summary; +end + function archive_store:delete(username, query) query = query or {}; local user,store = username,self.store; -- cgit v1.2.3 From 4a2f05236136f1a2e1a74e120bb6131c628a278a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 23 Mar 2019 22:05:08 +0100 Subject: mod_storage_memory: Implement archive summary API --- plugins/mod_storage_memory.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua index 8e1cf879..41180aba 100644 --- a/plugins/mod_storage_memory.lua +++ b/plugins/mod_storage_memory.lua @@ -147,6 +147,16 @@ function archive_store:find(username, query) end, count; end +function archive:summary(username, query) + local iter, err = self:find(username, query) + if not iter then return iter, err; end + local summary = {}; + for _, _, _, with in iter do + summary[with] = (summary[with] or 0) + 1; + end + return summary; +end + function archive_store:delete(username, query) if not query or next(query) == nil then -- cgit v1.2.3 From eda65b96f853b9210c8a7e650eef5578a1ce34bf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 23 Mar 2019 22:05:42 +0100 Subject: mod_storage_memory: Fix copypaste mistake --- plugins/mod_storage_memory.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua index 41180aba..dde2d571 100644 --- a/plugins/mod_storage_memory.lua +++ b/plugins/mod_storage_memory.lua @@ -147,7 +147,7 @@ function archive_store:find(username, query) end, count; end -function archive:summary(username, query) +function archive_store:summary(username, query) local iter, err = self:find(username, query) if not iter then return iter, err; end local summary = {}; -- cgit v1.2.3 From ff3904d881514abc83443b02e10cd621873bed64 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 4 Jan 2019 10:20:51 +0100 Subject: util.x509: Add function that extracts usable names from a certificate --- util/x509.lua | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/util/x509.lua b/util/x509.lua index 15cc4d3c..1cdf07dc 100644 --- a/util/x509.lua +++ b/util/x509.lua @@ -20,6 +20,7 @@ local nameprep = require "util.encodings".stringprep.nameprep; local idna_to_ascii = require "util.encodings".idna.to_ascii; +local idna_to_unicode = require "util.encodings".idna.to_unicode; local base64 = require "util.encodings".base64; local log = require "util.logger".init("x509"); local s_format = string.format; @@ -216,6 +217,32 @@ local function verify_identity(host, service, cert) return false end +-- TODO Support other SANs +local function get_identities(cert) --> set of names + if cert.setencode then + cert:setencode("utf8"); + end + + local names = {}; + + local ext = cert:extensions(); + local sans = ext[oid_subjectaltname]; + if sans and sans["dNSName"] then + for i = 1, #sans["dNSName"] do + names[ idna_to_unicode(sans["dNSName"][i]) ] = true; + end + end + + local subject = cert:subject(); + for i = 1, #subject do + local dn = subject[i]; + if dn.oid == oid_commonname and nameprep(dn.value) then + names[dn.value] = true; + end + end + return names; +end + local pat = "%-%-%-%-%-BEGIN ([A-Z ]+)%-%-%-%-%-\r?\n".. "([0-9A-Za-z+/=\r\n]*)\r?\n%-%-%-%-%-END %1%-%-%-%-%-"; @@ -237,6 +264,7 @@ end return { verify_identity = verify_identity; + get_identities = get_identities; pem2der = pem2der; der2pem = der2pem; }; -- cgit v1.2.3 From 3277b108c4f1b772e265d270c52ca2e5c9956879 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 08:18:19 +0000 Subject: sessionmanager: Split byte-level sending into separate session.rawsend --- core/sessionmanager.lua | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/core/sessionmanager.lua b/core/sessionmanager.lua index 2843001a..9a2456f2 100644 --- a/core/sessionmanager.lua +++ b/core/sessionmanager.lua @@ -32,20 +32,26 @@ local function new_session(conn) local session = { conn = conn, type = "c2s_unauthed", conntime = gettime() }; local filter = initialize_filters(session); local w = conn.write; + + function session.rawsend(t) + t = filter("bytes/out", tostring(t)); + if t then + local ret, err = w(conn, t); + if not ret then + session.log("debug", "Error writing to connection: %s", tostring(err)); + return false, err; + end + end + return true; + end + session.send = function (t) session.log("debug", "Sending[%s]: %s", session.type, t.top_tag and t:top_tag() or t:match("^[^>]*>?")); if t.name then t = filter("stanzas/out", t); end if t then - t = filter("bytes/out", tostring(t)); - if t then - local ret, err = w(conn, t); - if not ret then - session.log("debug", "Error writing to connection: %s", tostring(err)); - return false, err; - end - end + return session.rawsend(t); end return true; end -- cgit v1.2.3 From 7ef5adb448ad36074123a924dc80607e087ed200 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 24 Nov 2018 02:25:44 +0100 Subject: mod_csi_simple: Use write locks in net.server if available --- plugins/mod_csi_simple.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index da2dd953..abe65fce 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -86,7 +86,9 @@ end, -1); module:hook("csi-client-inactive", function (event) local session = event.origin; - if session.pump then + if session.conn and session.conn and session.conn.pause_writes then + session.conn:pause_writes(); + elseif session.pump then session.pump:pause(); else local bare_jid = jid.join(session.username, session.host); @@ -115,6 +117,8 @@ module:hook("csi-client-active", function (event) local session = event.origin; if session.pump then session.pump:resume(); + elseif session.conn and session.conn and session.conn.resume_writes then + session.conn:resume_writes(); end end); -- cgit v1.2.3 From ee4a464a26db437f1b187c4a6cd83a53492010fc Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 18:30:51 +0100 Subject: mod_c2s: Fire an event when outgoing buffers have been emptied --- plugins/mod_c2s.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/mod_c2s.lua b/plugins/mod_c2s.lua index 8d7b92fe..7c6d95f7 100644 --- a/plugins/mod_c2s.lua +++ b/plugins/mod_c2s.lua @@ -332,6 +332,13 @@ function listener.onreadtimeout(conn) end end +function listener.ondrain(conn) + local session = sessions[conn]; + if session then + return (hosts[session.host] or prosody).events.fire_event("c2s-ondrain", { session = session }); + end +end + local function keepalive(event) local session = event.session; if not session.notopen then -- cgit v1.2.3 From e31a4cdd89be95fd6b97617491ea8e4a9822af66 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 18:32:50 +0100 Subject: mod_csi_simple: Break out stanza timestamping into a function for future reuse --- plugins/mod_csi_simple.lua | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index abe65fce..6bd6c0bf 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -84,6 +84,14 @@ module:hook("csi-is-stanza-important", function (event) return true; end, -1); +local function with_timestamp(stanza, from) + if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then + stanza = st.clone(stanza); + stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = from, stamp = dt.datetime()})); + end + return stanza; +end + module:hook("csi-client-inactive", function (event) local session = event.origin; if session.conn and session.conn and session.conn.pause_writes then @@ -102,11 +110,7 @@ module:hook("csi-client-inactive", function (event) pump:flush(); send(stanza); else - if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then - stanza = st.clone(stanza); - stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = bare_jid, stamp = dt.datetime()})); - end - pump:push(stanza); + pump:push(with_timestamp(stanza, bare_jid)); end return true; end -- cgit v1.2.3 From 00f3e4db9c96b7939c1cf87c514a40f58ae7fde0 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 18:33:38 +0100 Subject: mod_csi_simple: Count buffered items and flush when it reaches configured limit In this mode, stanzas have been serialized to strings in the internal net.server buffer, so it is difficult to count them after the fact. --- plugins/mod_csi_simple.lua | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index 6bd6c0bf..5c829179 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -10,6 +10,7 @@ local jid = require "util.jid"; local st = require "util.stanza"; local dt = require "util.datetime"; local new_queue = require "util.queue".new; +local filters = require "util.filters"; local function new_pump(output, ...) -- luacheck: ignore 212/self @@ -92,10 +93,22 @@ local function with_timestamp(stanza, from) return stanza; end +local function manage_buffer(stanza, session) + local ctr = session.csi_counter or 0; + if ctr >= queue_size or module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then + session.conn:resume_writes(); + else + stanza = with_timestamp(stanza, jid.join(session.username, session.host)) + end + session.csi_counter = ctr + 1; + return stanza; +end + module:hook("csi-client-inactive", function (event) local session = event.origin; if session.conn and session.conn and session.conn.pause_writes then session.conn:pause_writes(); + filters.add_filter(session, "stanzas/out", manage_buffer); elseif session.pump then session.pump:pause(); else @@ -122,7 +135,16 @@ module:hook("csi-client-active", function (event) if session.pump then session.pump:resume(); elseif session.conn and session.conn and session.conn.resume_writes then + filters.remove_filter(session, "stanzas/out", manage_buffer); session.conn:resume_writes(); end end); + +module:hook("c2s-ondrain", function (event) + local session = event.session; + if session.state == "inactive" and session.conn and session.conn and session.conn.pause_writes then + session.csi_counter = 0; + session.conn:pause_writes(); + end +end); -- cgit v1.2.3 From 439a1e0ebb1045d72b0a3c106dc1d122783387e2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 18:58:53 +0100 Subject: mod_csi_simple: Trigger buffer flush on seeing incoming data I.e. the client sent us something, which means its network / radio is active. --- plugins/mod_csi_simple.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index 5c829179..07a8cfb3 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -104,11 +104,17 @@ local function manage_buffer(stanza, session) return stanza; end +local function flush_buffer(data, session) + session.conn:resume_writes(); + return data; +end + module:hook("csi-client-inactive", function (event) local session = event.origin; if session.conn and session.conn and session.conn.pause_writes then session.conn:pause_writes(); filters.add_filter(session, "stanzas/out", manage_buffer); + filters.add_filter(session, "bytes/in", flush_buffer); elseif session.pump then session.pump:pause(); else @@ -136,6 +142,7 @@ module:hook("csi-client-active", function (event) session.pump:resume(); elseif session.conn and session.conn and session.conn.resume_writes then filters.remove_filter(session, "stanzas/out", manage_buffer); + filters.remove_filter(session, "bytes/in", flush_buffer); session.conn:resume_writes(); end end); -- cgit v1.2.3 From 461c69ff4eca9672bac066cb7f14fdc98bdae32b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 19:02:40 +0100 Subject: mod_csi_simple: Also flush buffer in "pump" mode --- plugins/mod_csi_simple.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index 07a8cfb3..ee7e01c9 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -109,9 +109,15 @@ local function flush_buffer(data, session) return data; end +local function flush_pump(data, session) + session.pump:flush(); + return data; +end + module:hook("csi-client-inactive", function (event) local session = event.origin; if session.conn and session.conn and session.conn.pause_writes then + session.log("info", "Native net.server buffer management mode"); session.conn:pause_writes(); filters.add_filter(session, "stanzas/out", manage_buffer); filters.add_filter(session, "bytes/in", flush_buffer); @@ -124,6 +130,7 @@ module:hook("csi-client-inactive", function (event) local pump = new_pump(session.send, queue_size); pump:pause(); session.pump = pump; + filters.add_filter(session, "bytes/in", flush_pump); function session.send(stanza) if session.state == "active" or module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then pump:flush(); -- cgit v1.2.3 From ea52a16673127e4428efb59ab50d660a48b6f7e1 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 19:07:39 +0100 Subject: net.server_event: Allow writing into buffer of write-locked connections Check for 'nointerface' flag instead, whatever that means. --- net/server_event.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/server_event.lua b/net/server_event.lua index 42c9af2e..fde79d86 100644 --- a/net/server_event.lua +++ b/net/server_event.lua @@ -304,7 +304,7 @@ end -- Public methods function interface_mt:write(data) - if self.nowriting then return nil, "locked" end + if self.nointerface then return nil, "locked"; end --vdebug( "try to send data to client, id/data:", self.id, data ) data = tostring( data ) local len = #data @@ -316,7 +316,7 @@ function interface_mt:write(data) end t_insert(self.writebuffer, data) -- new buffer self.writebufferlen = total - if not self.eventwrite then -- register new write event + if not self.eventwrite and not self.nowriting then -- register new write event --vdebug( "register new write event" ) self.eventwrite = addevent( base, self.conn, EV_WRITE, self.writecallback, cfg.WRITE_TIMEOUT ) end -- cgit v1.2.3 From 8cb20bea8cbbcb0f8e2557738135511b617e79cf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 20:12:22 +0100 Subject: net.server_select: Fix write pause/resume functions Nothing would happen if the write buffer was empty. Also simplified the code because it took too long to understand what `if _sendlistlen ~= tmp then` did. --- net/server_select.lua | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/net/server_select.lua b/net/server_select.lua index 4b156409..5d554655 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -497,14 +497,12 @@ wrapconnection = function( server, listeners, socket, ip, serverport, clientport local tmp = _sendlistlen _sendlistlen = removesocket( _sendlist, socket, _sendlistlen ) _writetimes[ handler ] = nil - if _sendlistlen ~= tmp then - nosend = true - end + nosend = true end handler.resume_writes = function (self) - if nosend then - nosend = false - write( "" ) + nosend = false + if bufferlen > 0 then + _sendlistlen = addsocket(_sendlist, socket, _sendlistlen) end end -- cgit v1.2.3 From eefe3deafa99c7d227a4e77c179dde796824cb23 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 20:22:01 +0100 Subject: mod_csi_simple: Remove old "pump" queue/buffer method, handled in net.server now --- plugins/mod_csi_simple.lua | 63 ++-------------------------------------------- 1 file changed, 2 insertions(+), 61 deletions(-) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index ee7e01c9..0fa0d083 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -9,42 +9,8 @@ module:depends"csi" local jid = require "util.jid"; local st = require "util.stanza"; local dt = require "util.datetime"; -local new_queue = require "util.queue".new; local filters = require "util.filters"; -local function new_pump(output, ...) - -- luacheck: ignore 212/self - local q = new_queue(...); - local flush = true; - function q:pause() - flush = false; - end - function q:resume() - flush = true; - return q:flush(); - end - local push = q.push; - function q:push(item) - local ok = push(self, item); - if not ok then - q:flush(); - output(item, self); - elseif flush then - return q:flush(); - end - return true; - end - function q:flush() - local item = self:pop(); - while item do - output(item, self); - item = self:pop(); - end - return true; - end - return q; -end - local queue_size = module:get_option_number("csi_queue_size", 256); module:hook("csi-is-stanza-important", function (event) @@ -109,45 +75,20 @@ local function flush_buffer(data, session) return data; end -local function flush_pump(data, session) - session.pump:flush(); - return data; -end - module:hook("csi-client-inactive", function (event) local session = event.origin; if session.conn and session.conn and session.conn.pause_writes then - session.log("info", "Native net.server buffer management mode"); session.conn:pause_writes(); filters.add_filter(session, "stanzas/out", manage_buffer); filters.add_filter(session, "bytes/in", flush_buffer); - elseif session.pump then - session.pump:pause(); else - local bare_jid = jid.join(session.username, session.host); - local send = session.send; - session._orig_send = send; - local pump = new_pump(session.send, queue_size); - pump:pause(); - session.pump = pump; - filters.add_filter(session, "bytes/in", flush_pump); - function session.send(stanza) - if session.state == "active" or module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then - pump:flush(); - send(stanza); - else - pump:push(with_timestamp(stanza, bare_jid)); - end - return true; - end + session.log("warn", "Session connection does not support write pausing"); end end); module:hook("csi-client-active", function (event) local session = event.origin; - if session.pump then - session.pump:resume(); - elseif session.conn and session.conn and session.conn.resume_writes then + if session.conn and session.conn and session.conn.resume_writes then filters.remove_filter(session, "stanzas/out", manage_buffer); filters.remove_filter(session, "bytes/in", flush_buffer); session.conn:resume_writes(); -- cgit v1.2.3 From 8ad3c66121ddc50c102061c9365411545d660625 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 20:41:25 +0100 Subject: mod_csi_simple: Separate out functions to enable/disable optimizations This allows reusing this logic outside the events. Letting the functions be module globals makes it easier to access from eg the telnet console. --- plugins/mod_csi_simple.lua | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index 0fa0d083..14abbc68 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -75,8 +75,7 @@ local function flush_buffer(data, session) return data; end -module:hook("csi-client-inactive", function (event) - local session = event.origin; +function enable_optimizations(session) if session.conn and session.conn and session.conn.pause_writes then session.conn:pause_writes(); filters.add_filter(session, "stanzas/out", manage_buffer); @@ -84,15 +83,24 @@ module:hook("csi-client-inactive", function (event) else session.log("warn", "Session connection does not support write pausing"); end -end); +end -module:hook("csi-client-active", function (event) - local session = event.origin; +function disble_optimizations(session) if session.conn and session.conn and session.conn.resume_writes then filters.remove_filter(session, "stanzas/out", manage_buffer); filters.remove_filter(session, "bytes/in", flush_buffer); session.conn:resume_writes(); end +end + +module:hook("csi-client-inactive", function (event) + local session = event.origin; + enable_optimizations(session); +end); + +module:hook("csi-client-active", function (event) + local session = event.origin; + disble_optimizations(session); end); -- cgit v1.2.3 From 0c520e8a39f380940ad216a228eb0624d85e02bd Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 20:43:15 +0100 Subject: mod_csi_simple: Disable optimizations on unload and re-enable on load --- plugins/mod_csi_simple.lua | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index 14abbc68..d0de222e 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -111,3 +111,24 @@ module:hook("c2s-ondrain", function (event) session.conn:pause_writes(); end end); + +function module.load() + for _, user_session in pairs(prosody.hosts[module.host].sessions) do + for _, session in pairs(user_session.sessions) do + if session.state == "inactive" then + enable_optimizations(session); + end + end + end +end + +function module.unload() + for _, user_session in pairs(prosody.hosts[module.host].sessions) do + for _, session in pairs(user_session.sessions) do + if session.state == "inactive" then + disble_optimizations(session); + end + end + end +end + -- cgit v1.2.3 From dd0258ff779f4ccc4da11a19c344d054ba7f8430 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 20:53:49 +0100 Subject: mod_csi_simple: Add some debug logging --- plugins/mod_csi_simple.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index d0de222e..ff1fa86c 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -71,6 +71,7 @@ local function manage_buffer(stanza, session) end local function flush_buffer(data, session) + session.log("debug", "Client sent something, flushing buffer once"); session.conn:resume_writes(); return data; end @@ -109,6 +110,7 @@ module:hook("c2s-ondrain", function (event) if session.state == "inactive" and session.conn and session.conn and session.conn.pause_writes then session.csi_counter = 0; session.conn:pause_writes(); + session.log("debug", "Buffer flushed, resuming inactive mode"); end end); -- cgit v1.2.3 From ee003734474622524bcf3eea479e64a6c91f2802 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 24 Mar 2019 22:01:36 +0100 Subject: mod_csi_simple: Improve debug logs by mentioing why the buffer gets flushed --- plugins/mod_csi_simple.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index ff1fa86c..2dda1c42 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -61,7 +61,11 @@ end local function manage_buffer(stanza, session) local ctr = session.csi_counter or 0; - if ctr >= queue_size or module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then + if ctr >= queue_size then + session.log("debug", "Queue size limit hit, flushing buffer"); + session.conn:resume_writes(); + elseif module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then + session.log("debug", "Important stanza, flushing buffer"); session.conn:resume_writes(); else stanza = with_timestamp(stanza, jid.join(session.username, session.host)) -- cgit v1.2.3 From 3179518b6ac887fef63d5ffa83b2020691809299 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 25 Mar 2019 10:32:39 +0000 Subject: mod_csi_simple: Fix type in function name --- plugins/mod_csi_simple.lua | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index 2dda1c42..c79c56fc 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -90,7 +90,7 @@ function enable_optimizations(session) end end -function disble_optimizations(session) +function disable_optimizations(session) if session.conn and session.conn and session.conn.resume_writes then filters.remove_filter(session, "stanzas/out", manage_buffer); filters.remove_filter(session, "bytes/in", flush_buffer); @@ -105,7 +105,7 @@ end); module:hook("csi-client-active", function (event) local session = event.origin; - disble_optimizations(session); + disable_optimizations(session); end); @@ -132,9 +132,8 @@ function module.unload() for _, user_session in pairs(prosody.hosts[module.host].sessions) do for _, session in pairs(user_session.sessions) do if session.state == "inactive" then - disble_optimizations(session); + disable_optimizations(session); end end end end - -- cgit v1.2.3 From 65f01dec0dc42ee9e1997f2dcddeb6375cdc92c8 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 25 Mar 2019 15:20:28 +0100 Subject: mod_csi_simple: Include queue size in debug messages --- plugins/mod_csi_simple.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index c79c56fc..a9148618 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -62,10 +62,10 @@ end local function manage_buffer(stanza, session) local ctr = session.csi_counter or 0; if ctr >= queue_size then - session.log("debug", "Queue size limit hit, flushing buffer"); + session.log("debug", "Queue size limit hit, flushing buffer (queue size is %d)", session.csi_counter); session.conn:resume_writes(); elseif module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then - session.log("debug", "Important stanza, flushing buffer"); + session.log("debug", "Important stanza, flushing buffer (queue size is %d)", session.csi_counter); session.conn:resume_writes(); else stanza = with_timestamp(stanza, jid.join(session.username, session.host)) @@ -75,7 +75,7 @@ local function manage_buffer(stanza, session) end local function flush_buffer(data, session) - session.log("debug", "Client sent something, flushing buffer once"); + session.log("debug", "Client sent something, flushing buffer once (queue size is %d)", session.csi_counter); session.conn:resume_writes(); return data; end @@ -112,9 +112,9 @@ end); module:hook("c2s-ondrain", function (event) local session = event.session; if session.state == "inactive" and session.conn and session.conn and session.conn.pause_writes then - session.csi_counter = 0; session.conn:pause_writes(); - session.log("debug", "Buffer flushed, resuming inactive mode"); + session.log("debug", "Buffer flushed, resuming inactive mode (queue size was %d)", session.csi_counter); + session.csi_counter = 0; end end); -- cgit v1.2.3 From e9eca8cd5521f2bfc8f9285b606eb84de5a467e4 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 25 Mar 2019 14:37:43 +0000 Subject: util.stanza: Fix :top_tag() handling of namespaced attributes --- spec/util_stanza_spec.lua | 31 ++++++++++++++++++++++++ util/stanza.lua | 62 +++++++++++++++++++++++------------------------ 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/spec/util_stanza_spec.lua b/spec/util_stanza_spec.lua index 18e39554..38503ab7 100644 --- a/spec/util_stanza_spec.lua +++ b/spec/util_stanza_spec.lua @@ -381,4 +381,35 @@ describe("util.stanza", function() end); end); end); + + describe("top_tag", function () + local xml_parse = require "util.xml".parse; + it("works", function () + local s = st.message({type="chat"}, "Hello"); + local top_tag = s:top_tag(); + assert.is_string(top_tag); + assert.not_equal("/>", top_tag:sub(-2, -1)); + assert.equal(">", top_tag:sub(-1, -1)); + local s2 = xml_parse(top_tag..""); + assert(st.is_stanza(s2)); + assert.equal("message", s2.name); + assert.equal(0, #s2); + assert.equal(0, #s2.tags); + assert.equal("chat", s2.attr.type); + end); + + it("works with namespaced attributes", function () + local s = xml_parse[[]]; + local top_tag = s:top_tag(); + assert.is_string(top_tag); + assert.not_equal("/>", top_tag:sub(-2, -1)); + assert.equal(">", top_tag:sub(-1, -1)); + local s2 = xml_parse(top_tag..""); + assert(st.is_stanza(s2)); + assert.equal("message", s2.name); + assert.equal(0, #s2); + assert.equal(0, #s2.tags); + assert.equal("true", s2.attr["my-awesome-ns\1bar"]); + end); + end); end); diff --git a/util/stanza.lua b/util/stanza.lua index e9847ca6..7fe5c7ae 100644 --- a/util/stanza.lua +++ b/util/stanza.lua @@ -270,6 +270,34 @@ function stanza_mt:find(path) until not self end +local function _clone(stanza, only_top) + local attr, tags = {}, {}; + for k,v in pairs(stanza.attr) do attr[k] = v; end + local old_namespaces, namespaces = stanza.namespaces; + if old_namespaces then + namespaces = {}; + for k,v in pairs(old_namespaces) do namespaces[k] = v; end + end + local new = { name = stanza.name, attr = attr, namespaces = namespaces, tags = tags }; + if not only_top then + for i=1,#stanza do + local child = stanza[i]; + if child.name then + child = _clone(child); + t_insert(tags, child); + end + t_insert(new, child); + end + end + return setmetatable(new, stanza_mt); +end + +local function clone(stanza, only_top) + if not is_stanza(stanza) then + error("bad argument to clone: expected stanza, got "..type(stanza)); + end + return _clone(stanza, only_top); +end local escape_table = { ["'"] = "'", ["\""] = """, ["<"] = "<", [">"] = ">", ["&"] = "&" }; local function xml_escape(str) return (s_gsub(str, "['&<>\"]", escape_table)); end @@ -310,11 +338,8 @@ function stanza_mt.__tostring(t) end function stanza_mt.top_tag(t) - local attr_string = ""; - if t.attr then - for k, v in pairs(t.attr) do if type(k) == "string" then attr_string = attr_string .. s_format(" %s='%s'", k, xml_escape(tostring(v))); end end - end - return s_format("<%s%s>", t.name, attr_string); + local top_tag_clone = clone(t, true); + return tostring(top_tag_clone):sub(1,-3)..">"; end function stanza_mt.get_text(t) @@ -388,33 +413,6 @@ local function deserialize(serialized) end end -local function _clone(stanza) - local attr, tags = {}, {}; - for k,v in pairs(stanza.attr) do attr[k] = v; end - local old_namespaces, namespaces = stanza.namespaces; - if old_namespaces then - namespaces = {}; - for k,v in pairs(old_namespaces) do namespaces[k] = v; end - end - local new = { name = stanza.name, attr = attr, namespaces = namespaces, tags = tags }; - for i=1,#stanza do - local child = stanza[i]; - if child.name then - child = _clone(child); - t_insert(tags, child); - end - t_insert(new, child); - end - return setmetatable(new, stanza_mt); -end - -local function clone(stanza) - if not is_stanza(stanza) then - error("bad argument to clone: expected stanza, got "..type(stanza)); - end - return _clone(stanza); -end - local function message(attr, body) if not body then return new_stanza("message", attr); -- cgit v1.2.3 From 08ec4600470e32bbc4bb0afdfce9012f6a8cb5e9 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 26 Mar 2019 13:51:06 +0000 Subject: Backed out changeset 3eea63a68e0f Commit included intended changes to loggingmanager --- core/loggingmanager.lua | 21 +-------------------- util/queue.lua | 9 +++++---- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/core/loggingmanager.lua b/core/loggingmanager.lua index b510617f..cfa8246a 100644 --- a/core/loggingmanager.lua +++ b/core/loggingmanager.lua @@ -18,9 +18,6 @@ local getstyle, getstring = require "util.termcolours".getstyle, require "util.t local config = require "core.configmanager"; local logger = require "util.logger"; -local have_pposix, pposix = pcall(require, "util.pposix"); -have_pposix = have_pposix and pposix._VERSION == "0.4.4"; - local _ENV = nil; -- luacheck: std none @@ -48,8 +45,7 @@ local function add_rule(sink_config) local sink = sink_maker(sink_config); -- Set sink for all chosen levels - local levels = get_levels(sink_config.levels or logging_levels); - for level in pairs(levels) do + for level in pairs(get_levels(sink_config.levels or logging_levels)) do logger.add_level_sink(level, sink); end end @@ -236,21 +232,6 @@ local function log_to_console(sink_config) end log_sink_types.console = log_to_console; -if have_pposix then - local syslog_opened; - local function log_to_syslog(sink_config) -- luacheck: ignore 212/sink_config - if not syslog_opened then - pposix.syslog_open(sink_config.syslog_name or "prosody", sink_config.syslog_facility or config.get("*", "syslog_facility")); - syslog_opened = true; - end - local syslog = pposix.syslog_log; - return function (name, level, message, ...) - syslog(level, name, format(message, ...)); - end; - end - log_sink_types.syslog = log_to_syslog; -end - local function register_sink_type(name, sink_maker) local old_sink_maker = log_sink_types[name]; log_sink_types[name] = sink_maker; diff --git a/util/queue.lua b/util/queue.lua index 66ed098b..e63b3f1c 100644 --- a/util/queue.lua +++ b/util/queue.lua @@ -52,15 +52,16 @@ local function new(size, allow_wrapping) return t[tail]; end; items = function (self) - return function (_, pos) - if pos >= items then + --luacheck: ignore 431/t + return function (t, pos) + if pos >= t:count() then return nil; end local read_pos = tail + pos; - if read_pos > self.size then + if read_pos > t.size then read_pos = (read_pos%size); end - return pos+1, t[read_pos]; + return pos+1, t._items[read_pos]; end, self, 0; end; consume = function (self) -- cgit v1.2.3 From 8e20f176e04a261129c9d49705b72f9460999145 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 26 Mar 2019 13:54:14 +0000 Subject: util.queue: Update :items() to consistently use private data directly It will perform better this way, and we were accessing private variables already within the iterator. Replaces 3eea63a68e0f --- util/queue.lua | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/util/queue.lua b/util/queue.lua index e63b3f1c..66ed098b 100644 --- a/util/queue.lua +++ b/util/queue.lua @@ -52,16 +52,15 @@ local function new(size, allow_wrapping) return t[tail]; end; items = function (self) - --luacheck: ignore 431/t - return function (t, pos) - if pos >= t:count() then + return function (_, pos) + if pos >= items then return nil; end local read_pos = tail + pos; - if read_pos > t.size then + if read_pos > self.size then read_pos = (read_pos%size); end - return pos+1, t._items[read_pos]; + return pos+1, t[read_pos]; end, self, 0; end; consume = function (self) -- cgit v1.2.3 From 0c7ea2e51a59a69ab23abc505bbcd9b6ba3722bc Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 26 Mar 2019 14:48:21 +0000 Subject: loggingmanager, mod_posix: Move syslog to core, fixes #541 (in a way) --- core/loggingmanager.lua | 19 +++++++++++++++++++ plugins/mod_posix.lua | 13 ------------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/core/loggingmanager.lua b/core/loggingmanager.lua index cfa8246a..85a6380b 100644 --- a/core/loggingmanager.lua +++ b/core/loggingmanager.lua @@ -18,6 +18,9 @@ local getstyle, getstring = require "util.termcolours".getstyle, require "util.t local config = require "core.configmanager"; local logger = require "util.logger"; +local have_pposix, pposix = pcall(require, "util.pposix"); +have_pposix = have_pposix and pposix._VERSION == "0.4.0"; + local _ENV = nil; -- luacheck: std none @@ -232,6 +235,22 @@ local function log_to_console(sink_config) end log_sink_types.console = log_to_console; +if have_pposix then + local syslog_opened; + local function log_to_syslog(sink_config) -- luacheck: ignore 212/sink_config + if not syslog_opened then + local facility = sink_config.syslog_facility or config.get("*", "syslog_facility"); + pposix.syslog_open(sink_config.syslog_name or "prosody", facility); + syslog_opened = true; + end + local syslog = pposix.syslog_log; + return function (name, level, message, ...) + syslog(level, name, format(message, ...)); + end; + end + log_sink_types.syslog = log_to_syslog; +end + local function register_sink_type(name, sink_maker) local old_sink_maker = log_sink_types[name]; log_sink_types[name] = sink_maker; diff --git a/plugins/mod_posix.lua b/plugins/mod_posix.lua index 23df4d23..8367ae9e 100644 --- a/plugins/mod_posix.lua +++ b/plugins/mod_posix.lua @@ -113,19 +113,6 @@ local function write_pidfile() end end -local syslog_opened; -function syslog_sink_maker(config) -- luacheck: ignore 212/config - if not syslog_opened then - pposix.syslog_open("prosody", module:get_option_string("syslog_facility")); - syslog_opened = true; - end - local syslog = pposix.syslog_log; - return function (name, level, message, ...) - syslog(level, name, format(message, ...)); - end; -end -require "core.loggingmanager".register_sink_type("syslog", syslog_sink_maker); - local daemonize = module:get_option("daemonize", prosody.installed); local function remove_log_sinks() -- cgit v1.2.3 From f7b4b8ed7a50a034299e0e24ac9003757c01d78d Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 26 Mar 2019 14:59:42 +0000 Subject: mod_posix: Remove unnecessary import of util.format (thanks luacheck and buildbot) --- plugins/mod_posix.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/mod_posix.lua b/plugins/mod_posix.lua index 8367ae9e..a2a60dd0 100644 --- a/plugins/mod_posix.lua +++ b/plugins/mod_posix.lua @@ -20,7 +20,6 @@ if not have_signal then module:log("warn", "Couldn't load signal library, won't respond to SIGTERM"); end -local format = require "util.format".format; local lfs = require "lfs"; local stat = lfs.attributes; -- cgit v1.2.3 From 60f37d33c9d31e8978c52a782aff00d7d572ebb1 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 26 Mar 2019 17:22:25 +0000 Subject: moduleapi: Remove overly-verbose debug logging on module status change --- core/moduleapi.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/core/moduleapi.lua b/core/moduleapi.lua index 2db7433a..e9e4c6d3 100644 --- a/core/moduleapi.lua +++ b/core/moduleapi.lua @@ -531,7 +531,6 @@ function api:set_status(status_type, status_message, override) return; end self.status_type, self.status_message, self.status_time = status_type, status_message, time_now(); - self:log("debug", "New status: %s", status_type); self:fire_event("module-status/updated", { name = self.name }); end -- cgit v1.2.3 From e986a0e13786adbf60c42d352a019fd62b903c34 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 26 Mar 2019 17:22:56 +0000 Subject: moduleapi: Log suppressed status priority and message when not overriding --- core/moduleapi.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/moduleapi.lua b/core/moduleapi.lua index e9e4c6d3..b81bbeb2 100644 --- a/core/moduleapi.lua +++ b/core/moduleapi.lua @@ -527,7 +527,7 @@ function api:set_status(status_type, status_message, override) -- By default an 'error' status can only be overwritten by another 'error' status if (current_priority >= status_priorities.error and priority < current_priority and override ~= true) or (override == false and current_priority > priority) then - self:log("debug", "Ignoring status"); + self:log("debug", "moduleapi: ignoring status [prio %d override %s]: %s", priority, override, status_message); return; end self.status_type, self.status_message, self.status_time = status_type, status_message, time_now(); -- cgit v1.2.3 From cc28a5eb1232228097d229ea21a187f4ebf190cc Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 29 Mar 2019 22:45:54 +0100 Subject: core.s2smanager: Rename variable to be same in two functions --- core/s2smanager.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/s2smanager.lua b/core/s2smanager.lua index 0ba5e7c6..fbe0458b 100644 --- a/core/s2smanager.lua +++ b/core/s2smanager.lua @@ -26,10 +26,10 @@ local _ENV = nil; -- luacheck: std none local function new_incoming(conn) - local session = { conn = conn, type = "s2sin_unauthed", direction = "incoming", hosts = {} }; - session.log = logger_init("s2sin"..tostring(session):match("[a-f0-9]+$")); - incoming_s2s[session] = true; - return session; + local host_session = { conn = conn, type = "s2sin_unauthed", direction = "incoming", hosts = {} }; + host_session.log = logger_init("s2sin"..tostring(host_session):match("[a-f0-9]+$")); + incoming_s2s[host_session] = true; + return host_session; end local function new_outgoing(from_host, to_host) -- cgit v1.2.3 From 2abb3c16201ea848989d5c8bfcef67507faacf5c Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 29 Mar 2019 22:37:12 +0100 Subject: core.s2smanager: Spread out session tables over multiple lines Improves readability --- core/s2smanager.lua | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/core/s2smanager.lua b/core/s2smanager.lua index fbe0458b..e5540e1d 100644 --- a/core/s2smanager.lua +++ b/core/s2smanager.lua @@ -26,15 +26,26 @@ local _ENV = nil; -- luacheck: std none local function new_incoming(conn) - local host_session = { conn = conn, type = "s2sin_unauthed", direction = "incoming", hosts = {} }; + local host_session = { + conn = conn, + type = "s2sin_unauthed", + direction = "incoming", + hosts = {}, + }; host_session.log = logger_init("s2sin"..tostring(host_session):match("[a-f0-9]+$")); incoming_s2s[host_session] = true; return host_session; end local function new_outgoing(from_host, to_host) - local host_session = { to_host = to_host, from_host = from_host, host = from_host, - notopen = true, type = "s2sout_unauthed", direction = "outgoing" }; + local host_session = { + to_host = to_host, + from_host = from_host, + host = from_host, + notopen = true, + type = "s2sout_unauthed", + direction = "outgoing", + }; hosts[from_host].s2sout[to_host] = host_session; local conn_name = "s2sout"..tostring(host_session):match("[a-f0-9]*$"); host_session.log = logger_init(conn_name); -- cgit v1.2.3 From 00f1e6982dcbc263f8b58e11283b31078e048a01 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 29 Mar 2019 22:40:53 +0100 Subject: core.s2smanager: Use util.session to create sessions --- core/s2smanager.lua | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/core/s2smanager.lua b/core/s2smanager.lua index e5540e1d..46dcd108 100644 --- a/core/s2smanager.lua +++ b/core/s2smanager.lua @@ -13,6 +13,7 @@ local tostring, pairs, setmetatable = tostring, pairs, setmetatable; local logger_init = require "util.logger".init; +local sessionlib = require "util.session"; local log = logger_init("s2smanager"); @@ -26,29 +27,26 @@ local _ENV = nil; -- luacheck: std none local function new_incoming(conn) - local host_session = { - conn = conn, - type = "s2sin_unauthed", - direction = "incoming", - hosts = {}, - }; - host_session.log = logger_init("s2sin"..tostring(host_session):match("[a-f0-9]+$")); + local host_session = sessionlib.new("s2sin"); + sessionlib.set_id(host_session); + sessionlib.set_logger(host_session); + sessionlib.set_conn(host_session, conn); + host_session.direction = "incoming"; + host_session.session.hosts = {}; incoming_s2s[host_session] = true; return host_session; end local function new_outgoing(from_host, to_host) - local host_session = { - to_host = to_host, - from_host = from_host, - host = from_host, - notopen = true, - type = "s2sout_unauthed", - direction = "outgoing", - }; + local host_session = sessionlib.new("s2sout"); + sessionlib.set_id(host_session); + sessionlib.set_logger(host_session); + host_session.to_host = to_host; + host_session.from_host = from_host; + host_session.host = from_host; + host_session.notopen = true; + host_session.direction = "outgoing"; hosts[from_host].s2sout[to_host] = host_session; - local conn_name = "s2sout"..tostring(host_session):match("[a-f0-9]*$"); - host_session.log = logger_init(conn_name); return host_session; end -- cgit v1.2.3 From 19de299874a63dd0f5143844f638b879d571b17f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 29 Mar 2019 23:05:08 +0100 Subject: core.sessionmanager: Use util.session to create sessions --- core/sessionmanager.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/sessionmanager.lua b/core/sessionmanager.lua index 9a2456f2..f5af1185 100644 --- a/core/sessionmanager.lua +++ b/core/sessionmanager.lua @@ -21,6 +21,7 @@ local config_get = require "core.configmanager".get; local resourceprep = require "util.encodings".stringprep.resourceprep; local nodeprep = require "util.encodings".stringprep.nodeprep; local generate_identifier = require "util.id".short; +local sessionlib = require "util.session"; local initialize_filters = require "util.filters".initialize; local gettime = require "socket".gettime; @@ -29,7 +30,12 @@ local _ENV = nil; -- luacheck: std none local function new_session(conn) - local session = { conn = conn, type = "c2s_unauthed", conntime = gettime() }; + local session = sessionlib.new("c2s"); + sessionlib.set_id(session); + sessionlib.set_logger(session); + sessionlib.set_conn(session, conn); + + session.conntime = gettime(); local filter = initialize_filters(session); local w = conn.write; -- cgit v1.2.3 From a23b9fd0ea3b3404861fd56db9d99d9b7e32f3fd Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 30 Mar 2019 09:04:33 +0100 Subject: core.s2smanager: Fix previous commit (Thanks Martin) --- core/s2smanager.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/s2smanager.lua b/core/s2smanager.lua index 46dcd108..684bb94e 100644 --- a/core/s2smanager.lua +++ b/core/s2smanager.lua @@ -32,7 +32,7 @@ local function new_incoming(conn) sessionlib.set_logger(host_session); sessionlib.set_conn(host_session, conn); host_session.direction = "incoming"; - host_session.session.hosts = {}; + host_session.hosts = {}; incoming_s2s[host_session] = true; return host_session; end -- cgit v1.2.3 From d22524dbc05c76ac357f5e6dd5383efe8cd2e036 Mon Sep 17 00:00:00 2001 From: marc0s Date: Sat, 30 Mar 2019 18:44:34 +0100 Subject: doc/coding_style: apply consistent semi-colon usage Make all "good" statements in the coding style document use consistent statement-separator semi-colon --- doc/coding_style.md | 122 ++++++++++++++++++++++++++-------------------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/doc/coding_style.md b/doc/coding_style.md index 6ca527fa..f9a10ece 100644 --- a/doc/coding_style.md +++ b/doc/coding_style.md @@ -13,7 +13,7 @@ This style guides lists the coding conventions used in the for i, pkg in ipairs(packages) do for name, version in pairs(pkg) do if name == searched then - print(version) + print(version); end end end @@ -69,7 +69,7 @@ unless you are writing a function that operates on generic tables. ```lua for _, item in ipairs(items) do - do_something_with_item(item) + do_something_with_item(item); end ``` @@ -85,7 +85,7 @@ local c = function() end -- good -local this_is_my_object = {} +local this_is_my_object = {}; local function do_that_thing() -- ...stuff... @@ -113,7 +113,7 @@ end -- good local function is_evil(alignment) - return alignment < 100 + return alignment < 100; end ``` @@ -130,7 +130,7 @@ constants from C. * When creating a table, prefer populating its fields all at once, if possible: ```lua -local player = { name = "Jack", class = "Rogue" } +local player = { name = "Jack", class = "Rogue" }; ``` * Items should be separated by commas. If there are many items, put each @@ -145,7 +145,7 @@ local player = { ``` > **Rationale:** This makes the structure of your tables more evident at a glance. -Trailing commas make it quicker to add new fields and produces shorter diffs. +Trailing semi-colons make it quicker to add new fields and produces shorter diffs. * Use plain `key` syntax whenever possible, use `["key"]` syntax when using names that can't be represented as identifiers and avoid mixing representations in @@ -153,9 +153,9 @@ a declaration: ```lua local mytable = { - ["1394-E"] = val1, - ["UTF-8"] = val2, - ["and"] = val2, + ["1394-E"] = val1; + ["UTF-8"] = val2; + ["and"] = val2; } ``` @@ -165,8 +165,8 @@ local mytable = { that contain double quotes. ```lua -local name = "Prosody" -local sentence = 'The name of the program is "Prosody"' +local name = "Prosody"; +local sentence = 'The name of the program is "Prosody"'; ``` > **Rationale:** Double quotes are used as string delimiters in a larger number of @@ -218,12 +218,12 @@ end -- good local function is_good_name(name, options, args) if #name < 3 or #name > 30 then - return false + return false; end -- ...stuff... - return true + return true; end ``` @@ -236,8 +236,8 @@ end -- bad local data = get_data"KRP"..tostring(area_number) -- good -local data = get_data("KRP"..tostring(area_number)) -local data = get_data("KRP")..tostring(area_number) +local data = get_data("KRP"..tostring(area_number)); +local data = get_data("KRP")..tostring(area_number); ``` > **Rationale:** It is not obvious at a glace what the precedence rules are @@ -251,8 +251,8 @@ lines. ```lua local an_instance = a_module.new { - a_parameter = 42, - another_parameter = "yay", + a_parameter = 42; + another_parameter = "yay"; } ``` @@ -265,15 +265,15 @@ so there are no precedence issues. ```lua local luke = { - jedi = true, - age = 28, + jedi = true; + age = 28; } -- bad local is_jedi = luke["jedi"] -- good -local is_jedi = luke.jedi +local is_jedi = luke.jedi; ``` * Use subscript notation `[]` when accessing properties with a variable or if using a table as a list. @@ -282,11 +282,11 @@ local is_jedi = luke.jedi local vehicles = load_vehicles_from_disk("vehicles.dat") if vehicles["Porsche"] then - porsche_handler(vehicles["Porsche"]) - vehicles["Porsche"] = nil + porsche_handler(vehicles["Porsche"]); + vehicles["Porsche"] = nil; end for name, cars in pairs(vehicles) do - regular_handler(cars) + regular_handler(cars); end ``` @@ -298,7 +298,7 @@ to be used as a record/object field. * When declaring modules and classes, declare functions external to the table definition: ```lua -local my_module = {} +local my_module = {}; function my_module.a_function(x) -- code @@ -337,7 +337,7 @@ than if it says `check_version = function()` under some indentation level. superpower = get_superpower() -- good -local superpower = get_superpower() +local superpower = get_superpower(); ``` > **Rationale:** Not doing so will result in global variables to avoid polluting @@ -366,18 +366,18 @@ end -- good local bad = function() - test() - print("doing stuff..") + test(); + print("doing stuff.."); --...other stuff... - local name = get_name() + local name = get_name(); if name == "test" then - return false + return false; end - return name + return name; end ``` @@ -410,11 +410,11 @@ easier to scan visually: ```lua local function default_name(name) -- return the default "Waldo" if name is nil - return name or "Waldo" + return name or "Waldo"; end local function brew_coffee(machine) - return (machine and machine.is_loaded) and "coffee brewing" or "fill your water" + return (machine and machine.is_loaded) and "coffee brewing" or "fill your water"; end ``` @@ -434,7 +434,7 @@ if test then break end if not ok then return nil, "this failed for this reason: " .. reason end -- good -use_callback(x, function(k) return k.last end) +use_callback(x, function(k) return k.last end); -- good if test then @@ -446,8 +446,8 @@ if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == -- good if test < 1 and do_complicated_function(test) == false or seven == 8 and nine == 10 then - do_other_complicated_function() - return false + do_other_complicated_function(); + return false; end ``` @@ -491,17 +491,17 @@ dog.set( "attr",{ }) -- good -local x = y * 9 -local numbers = {1, 2, 3} +local x = y * 9; +local numbers = {1, 2, 3}; local strings = { "hello"; "Lua"; "world"; } dog.set("attr", { - age = "1 year", - breed = "Bernese Mountain Dog", -}) + age = "1 year"; + breed = "Bernese Mountain Dog"; +}); ``` * Indent tables and functions according to the start of the line, not the construct: @@ -522,7 +522,7 @@ local my_table = { "world"; } using_a_callback(x, function(...) - print("hello") + print("hello"); end) ``` @@ -534,7 +534,7 @@ replacing `x` with `xy` in the `using_a_callback` example above). ```lua -- okay -local message = "Hello, "..user.."! This is your day # "..day.." in our platform!" +local message = "Hello, "..user.."! This is your day # "..day.." in our platform!"; ``` > **Rationale:** Being at the baseline, the dots already provide some visual spacing. @@ -582,8 +582,8 @@ local a = 1 local long_identifier = 2 -- good -local a = 1 -local long_identifier = 2 +local a = 1; +local long_identifier = 2; ``` > **Rationale:** This produces extra diffs which add noise to `git blame`. @@ -592,8 +592,8 @@ local long_identifier = 2 ```lua -- okay -sys_command(form, UI_FORM_UPDATE_NODE, "a", FORM_NODE_HIDDEN, false) -sys_command(form, UI_FORM_UPDATE_NODE, "sample", FORM_NODE_VISIBLE, false) +sys_command(form, UI_FORM_UPDATE_NODE, "a", FORM_NODE_HIDDEN, false); +sys_command(form, UI_FORM_UPDATE_NODE, "sample", FORM_NODE_VISIBLE, false); ``` ## Typing @@ -603,8 +603,8 @@ for function arguments: ```lua function manif.load_manifest(repo_url, lua_version) - assert(type(repo_url) == "string") - assert(type(lua_version) == "string" or not lua_version) + assert(type(repo_url) == "string"); + assert(type(lua_version) == "string" or not lua_version); -- ... end @@ -617,7 +617,7 @@ end local total_score = review_score .. "" -- good -local total_score = tostring(review_score) +local total_score = tostring(review_score); ``` ## Errors @@ -636,9 +636,9 @@ Follow [these guidelines](http://hisham.hm/2014/01/02/how-to-write-lua-modules-i * Always require a module into a local variable named after the last component of the module’s full name. ```lua -local bar = require("foo.bar") -- requiring the module +local bar = require("foo.bar"); -- requiring the module -bar.say("hello") -- using the module +bar.say("hello"); -- using the module ``` * Don’t rename modules arbitrarily: @@ -657,7 +657,7 @@ the whole module path. ```lua --- @module foo.bar -local bar = {} +local bar = {}; ``` * Try to use names that won't clash with your local variables. For instance, don't @@ -672,7 +672,7 @@ That is, `local function helper_foo()` means that `helper_foo` is really local. ```lua function bar.say(greeting) - print(greeting) + print(greeting); end ``` @@ -702,14 +702,14 @@ and do something like this instead: ```lua -- good -local messagepack = require("messagepack") -local mpack = messagepack.new({integer = "unsigned"}) +local messagepack = require("messagepack"); +local mpack = messagepack.new({integer = "unsigned"}); ``` * The invocation of require may omit parentheses around the module name: ```lua -local bla = require "bla" +local bla = require "bla"; ``` ## Metatables, classes and objects @@ -739,7 +739,7 @@ return { new = new }; my_object.my_method(my_object) -- good -my_object:my_method() +my_object:my_method(); ``` > **Rationale:** This makes it explicit that the intent is to use the function as a method. @@ -780,8 +780,8 @@ its arguments; it's better to spell out in the argument what the API the function implements is, instead of adding `_` variables. ``` -local foo, bar = some_function() --luacheck: ignore 212/foo -print(bar) +local foo, bar = some_function(); --luacheck: ignore 212/foo +print(bar); ``` * luacheck warning 542 (empty if branch) can also be ignored, when a sequence @@ -790,11 +790,11 @@ and one of the cases is meant to mean "pass". For example: ```lua if warning >= 600 and warning <= 699 then - print("no whitespace warnings") + print("no whitespace warnings"); elseif warning == 542 then --luacheck: ignore 542 -- pass else - print("got a warning: "..warning) + print("got a warning: "..warning); end ``` -- cgit v1.2.3 From 1c244327eccfa235260ea2a27173da8633ad956c Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 1 Apr 2019 21:15:47 +0200 Subject: .luacheckrc: Correct indentation of 'exclude_files' list --- .luacheckrc | 60 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.luacheckrc b/.luacheckrc index f0cb93a8..f6760ee3 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -131,42 +131,42 @@ if os.getenv("PROSODY_STRICT_LINT") ~= "1" then unused_secondaries = false local exclude_files = { - "doc/net.server.lua"; + "doc/net.server.lua"; - "fallbacks/bit.lua"; - "fallbacks/lxp.lua"; + "fallbacks/bit.lua"; + "fallbacks/lxp.lua"; - "net/cqueues.lua"; - "net/dns.lua"; - "net/server_select.lua"; + "net/cqueues.lua"; + "net/dns.lua"; + "net/server_select.lua"; - "plugins/mod_storage_sql1.lua"; + "plugins/mod_storage_sql1.lua"; - "spec/core_configmanager_spec.lua"; - "spec/core_moduleapi_spec.lua"; - "spec/net_http_parser_spec.lua"; - "spec/util_events_spec.lua"; - "spec/util_http_spec.lua"; - "spec/util_ip_spec.lua"; - "spec/util_multitable_spec.lua"; - "spec/util_rfc6724_spec.lua"; - "spec/util_throttle_spec.lua"; - "spec/util_xmppstream_spec.lua"; + "spec/core_configmanager_spec.lua"; + "spec/core_moduleapi_spec.lua"; + "spec/net_http_parser_spec.lua"; + "spec/util_events_spec.lua"; + "spec/util_http_spec.lua"; + "spec/util_ip_spec.lua"; + "spec/util_multitable_spec.lua"; + "spec/util_rfc6724_spec.lua"; + "spec/util_throttle_spec.lua"; + "spec/util_xmppstream_spec.lua"; - "tools/ejabberd2prosody.lua"; - "tools/ejabberdsql2prosody.lua"; - "tools/erlparse.lua"; - "tools/jabberd14sql2prosody.lua"; - "tools/migration/migrator.cfg.lua"; - "tools/migration/migrator/jabberd14.lua"; - "tools/migration/migrator/mtools.lua"; - "tools/migration/migrator/prosody_files.lua"; - "tools/migration/migrator/prosody_sql.lua"; - "tools/migration/prosody-migrator.lua"; - "tools/openfire2prosody.lua"; - "tools/xep227toprosody.lua"; + "tools/ejabberd2prosody.lua"; + "tools/ejabberdsql2prosody.lua"; + "tools/erlparse.lua"; + "tools/jabberd14sql2prosody.lua"; + "tools/migration/migrator.cfg.lua"; + "tools/migration/migrator/jabberd14.lua"; + "tools/migration/migrator/mtools.lua"; + "tools/migration/migrator/prosody_files.lua"; + "tools/migration/migrator/prosody_sql.lua"; + "tools/migration/prosody-migrator.lua"; + "tools/openfire2prosody.lua"; + "tools/xep227toprosody.lua"; - "util/sasl/digest-md5.lua"; + "util/sasl/digest-md5.lua"; } for _, file in ipairs(exclude_files) do files[file] = { only = {} } -- cgit v1.2.3 From ad925d5dad90ae6462302e74325b74c88ff1fceb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 2 Apr 2019 21:17:28 +0200 Subject: mod_limits: Fix indentation Appears to have been messed up in 60e113f3682f --- plugins/mod_limits.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/mod_limits.lua b/plugins/mod_limits.lua index 914d5c44..32bc75a9 100644 --- a/plugins/mod_limits.lua +++ b/plugins/mod_limits.lua @@ -51,18 +51,18 @@ end local default_filter_set = {}; function default_filter_set.bytes_in(bytes, session) - local sess_throttle = session.throttle; - if sess_throttle then - local ok, balance, outstanding = sess_throttle:poll(#bytes, true); + local sess_throttle = session.throttle; + if sess_throttle then + local ok, balance, outstanding = sess_throttle:poll(#bytes, true); if not ok then - session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding); + session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding); outstanding = ceil(outstanding); session.conn:pause(); -- Read no more data from the connection until there is no outstanding data local outstanding_data = bytes:sub(-outstanding); bytes = bytes:sub(1, #bytes-outstanding); timer.add_task(limits_resolution, function () if not session.conn then return; end - if sess_throttle:peek(#outstanding_data) then + if sess_throttle:peek(#outstanding_data) then session.log("debug", "Resuming paused session"); session.conn:resume(); end -- cgit v1.2.3 From 0f802ffe183a94cc9407cdf92bb9eaa9c213e602 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 2 Apr 2019 20:38:51 +0200 Subject: mod_limits: Allow configuring a list of unrestricted JIDs (fixes #1323) --- plugins/mod_limits.lua | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/mod_limits.lua b/plugins/mod_limits.lua index 32bc75a9..56e3faaf 100644 --- a/plugins/mod_limits.lua +++ b/plugins/mod_limits.lua @@ -96,3 +96,20 @@ end function module.unload() filters.remove_filter_hook(filter_hook); end + +function module.add_host(module) + local unlimited_jids = module:get_option_inherited_set("unlimited_jids", {}); + + if not unlimited_jids:empy() then + module:hook("authentication-success", function (event) + local session = event.session; + local session_type = session.type:match("^[^_]+"); + local jid = session.username .. "@" .. session.host; + if unlimited_jids:contains(jid) then + local filter_set = type_filters[session_type]; + filters.remove_filter(session, "bytes/in", filter_set.bytes_in); + session.throttle = nil; + end + end); + end +end -- cgit v1.2.3 From fc818b419ebf2ef4991020744e0a2cb15ccf201a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 2 Apr 2019 21:22:20 +0200 Subject: mod_limits: Fix typo --- plugins/mod_limits.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_limits.lua b/plugins/mod_limits.lua index 56e3faaf..7ae8bb34 100644 --- a/plugins/mod_limits.lua +++ b/plugins/mod_limits.lua @@ -100,7 +100,7 @@ end function module.add_host(module) local unlimited_jids = module:get_option_inherited_set("unlimited_jids", {}); - if not unlimited_jids:empy() then + if not unlimited_jids:empty() then module:hook("authentication-success", function (event) local session = event.session; local session_type = session.type:match("^[^_]+"); -- cgit v1.2.3 From ada8fc186d60e1d3250e6fe844cbcc853eee64bd Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 3 Apr 2019 01:02:36 +0200 Subject: README: Remove mailing list where issue tracker changes went in the Google Code days --- README | 3 --- 1 file changed, 3 deletions(-) diff --git a/README b/README index 797a2175..f679402a 100644 --- a/README +++ b/README @@ -26,9 +26,6 @@ Mailing lists: Development discussion: https://groups.google.com/group/prosody-dev - Issue tracker changes: - https://groups.google.com/group/prosody-issues - ## Installation See the accompanying INSTALL file for help on building Prosody from source. Alternatively -- cgit v1.2.3 From 79b43ce34ca3914d01b24b1afb50a0d184bb292f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 3 Apr 2019 01:03:23 +0200 Subject: README: Add link to current issue tracker --- README | 1 + 1 file changed, 1 insertion(+) diff --git a/README b/README index f679402a..4b52186b 100644 --- a/README +++ b/README @@ -12,6 +12,7 @@ rapidly prototype new protocols. Homepage: https://prosody.im/ Download: https://prosody.im/download Documentation: https://prosody.im/doc/ +Issue tracker: https://issues.prosody.im/ Jabber/XMPP Chat: Address: -- cgit v1.2.3 From 1b3c29d26553854f4cf577a3bda0c12d61db8ae2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 3 Apr 2019 01:09:21 +0200 Subject: CONTRIBUTING: Add a short file referencing the online doc/contributing page --- CONTRIBUTING | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 CONTRIBUTING diff --git a/CONTRIBUTING b/CONTRIBUTING new file mode 100644 index 00000000..2e732c73 --- /dev/null +++ b/CONTRIBUTING @@ -0,0 +1,9 @@ +Thanks for your interest in contributing to the project! + +There are many ways to contribute, such as helping improve the +documentation, reporting bugs, spreading the word or testing the latest +development version. + +You can find more information on how to contribute at + +See also the HACKERS and README files. -- cgit v1.2.3 From c0e30a99fc9b49847577598541328ebe1d7eaaba Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 3 Apr 2019 17:20:57 +0200 Subject: util.session: Fix session id not include unauthed forever --- util/session.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/session.lua b/util/session.lua index b2a726ce..b9c6bec7 100644 --- a/util/session.lua +++ b/util/session.lua @@ -4,12 +4,13 @@ local logger = require "util.logger"; local function new_session(typ) local session = { type = typ .. "_unauthed"; + base_type = typ; }; return session; end local function set_id(session) - local id = session.type .. tostring(session):match("%x+$"):lower(); + local id = session.base_type .. tostring(session):match("%x+$"):lower(); session.id = id; return session; end -- cgit v1.2.3 From 4db92c1c120b6ed9f91bb2e1167b7159f2337129 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 5 Apr 2019 16:10:51 +0200 Subject: net.http.files: Copy of mod_http_files The intent is to make it easier to reuse and simplify mod_http_files. Currently modules will use the serve() function exported by mod_http_files in order to serve their own files. This makes it unclear whether mod_http_files should be doing anything on its own. Moving the logic into a separate module should help here, as well as make re-use outside of prosody easier. --- net/http/files.lua | 198 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 net/http/files.lua diff --git a/net/http/files.lua b/net/http/files.lua new file mode 100644 index 00000000..1dae0d6d --- /dev/null +++ b/net/http/files.lua @@ -0,0 +1,198 @@ +-- Prosody IM +-- Copyright (C) 2008-2010 Matthew Wild +-- Copyright (C) 2008-2010 Waqas Hussain +-- +-- This project is MIT/X11 licensed. Please see the +-- COPYING file in the source package for more information. +-- + +module:depends("http"); +local server = require"net.http.server"; +local lfs = require "lfs"; + +local os_date = os.date; +local open = io.open; +local stat = lfs.attributes; +local build_path = require"socket.url".build_path; +local path_sep = package.config:sub(1,1); + +local base_path = module:get_option_path("http_files_dir", module:get_option_path("http_path")); +local cache_size = module:get_option_number("http_files_cache_size", 128); +local cache_max_file_size = module:get_option_number("http_files_cache_max_file_size", 4096); +local dir_indices = module:get_option_array("http_index_files", { "index.html", "index.htm" }); +local directory_index = module:get_option_boolean("http_dir_listing"); + +local mime_map = module:shared("/*/http_files/mime").types; +if not mime_map then + mime_map = { + html = "text/html", htm = "text/html", + xml = "application/xml", + txt = "text/plain", + css = "text/css", + js = "application/javascript", + png = "image/png", + gif = "image/gif", + jpeg = "image/jpeg", jpg = "image/jpeg", + svg = "image/svg+xml", + }; + module:shared("/*/http_files/mime").types = mime_map; + + local mime_types, err = open(module:get_option_path("mime_types_file", "/etc/mime.types", "config"), "r"); + if mime_types then + local mime_data = mime_types:read("*a"); + mime_types:close(); + setmetatable(mime_map, { + __index = function(t, ext) + local typ = mime_data:match("\n(%S+)[^\n]*%s"..(ext:lower()).."%s") or "application/octet-stream"; + t[ext] = typ; + return typ; + end + }); + end +end + +local forbidden_chars_pattern = "[/%z]"; +if prosody.platform == "windows" then + forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]" +end + +local urldecode = require "util.http".urldecode; +function sanitize_path(path) + if not path then return end + local out = {}; + + local c = 0; + for component in path:gmatch("([^/]+)") do + component = urldecode(component); + if component:find(forbidden_chars_pattern) then + return nil; + elseif component == ".." then + if c <= 0 then + return nil; + end + out[c] = nil; + c = c - 1; + elseif component ~= "." then + c = c + 1; + out[c] = component; + end + end + if path:sub(-1,-1) == "/" then + out[c+1] = ""; + end + return "/"..table.concat(out, "/"); +end + +local cache = require "util.cache".new(cache_size); + +function serve(opts) + if type(opts) ~= "table" then -- assume path string + opts = { path = opts }; + end + -- luacheck: ignore 431 + local base_path = opts.path; + local dir_indices = opts.index_files or dir_indices; + local directory_index = opts.directory_index; + local function serve_file(event, path) + local request, response = event.request, event.response; + local sanitized_path = sanitize_path(path); + if path and not sanitized_path then + return 400; + end + path = sanitized_path; + local orig_path = sanitize_path(request.path); + local full_path = base_path .. (path or ""):gsub("/", path_sep); + local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows + if not attr then + return 404; + end + + local request_headers, response_headers = request.headers, response.headers; + + local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification); + response_headers.last_modified = last_modified; + + local etag = ('"%02x-%x-%x-%x"'):format(attr.dev or 0, attr.ino or 0, attr.size or 0, attr.modification or 0); + response_headers.etag = etag; + + local if_none_match = request_headers.if_none_match + local if_modified_since = request_headers.if_modified_since; + if etag == if_none_match + or (not if_none_match and last_modified == if_modified_since) then + return 304; + end + + local data = cache:get(orig_path); + if data and data.etag == etag then + response_headers.content_type = data.content_type; + data = data.data; + elseif attr.mode == "directory" and path then + if full_path:sub(-1) ~= "/" then + local dir_path = { is_absolute = true, is_directory = true }; + for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end + response_headers.location = build_path(dir_path); + return 301; + end + for i=1,#dir_indices do + if stat(full_path..dir_indices[i], "mode") == "file" then + return serve_file(event, path..dir_indices[i]); + end + end + + if directory_index then + data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path }); + end + if not data then + return 403; + end + cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; }); + response_headers.content_type = mime_map.html; + + else + local f, err = open(full_path, "rb"); + if not f then + module:log("debug", "Could not open %s. Error was %s", full_path, err); + return 403; + end + local ext = full_path:match("%.([^./]+)$"); + local content_type = ext and mime_map[ext]; + response_headers.content_type = content_type; + if attr.size > cache_max_file_size then + response_headers.content_length = attr.size; + module:log("debug", "%d > cache_max_file_size", attr.size); + return response:send_file(f); + else + data = f:read("*a"); + f:close(); + end + cache:set(orig_path, { data = data; content_type = content_type; etag = etag }); + end + + return response:send(data); + end + + return serve_file; +end + +function wrap_route(routes) + for route,handler in pairs(routes) do + if type(handler) ~= "function" then + routes[route] = serve(handler); + end + end + return routes; +end + +if base_path then + module:provides("http", { + route = { + ["GET /*"] = serve { + path = base_path; + directory_index = directory_index; + } + }; + }); +else + module:log("debug", "http_files_dir not set, assuming use by some other module"); +end + -- cgit v1.2.3 From f25e2bc13b7485b762bc3ff71b98da96ade2df54 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 5 Apr 2019 17:09:03 +0200 Subject: net.http.files: Make into standalone library --- net/http/files.lua | 78 ++++++++++-------------------------------------------- 1 file changed, 14 insertions(+), 64 deletions(-) diff --git a/net/http/files.lua b/net/http/files.lua index 1dae0d6d..0b898dc6 100644 --- a/net/http/files.lua +++ b/net/http/files.lua @@ -6,9 +6,10 @@ -- COPYING file in the source package for more information. -- -module:depends("http"); local server = require"net.http.server"; local lfs = require "lfs"; +local new_cache = require "util.cache".new; +local log = require "util.logger".init("net.http.files"); local os_date = os.date; local open = io.open; @@ -16,48 +17,14 @@ local stat = lfs.attributes; local build_path = require"socket.url".build_path; local path_sep = package.config:sub(1,1); -local base_path = module:get_option_path("http_files_dir", module:get_option_path("http_path")); -local cache_size = module:get_option_number("http_files_cache_size", 128); -local cache_max_file_size = module:get_option_number("http_files_cache_max_file_size", 4096); -local dir_indices = module:get_option_array("http_index_files", { "index.html", "index.htm" }); -local directory_index = module:get_option_boolean("http_dir_listing"); - -local mime_map = module:shared("/*/http_files/mime").types; -if not mime_map then - mime_map = { - html = "text/html", htm = "text/html", - xml = "application/xml", - txt = "text/plain", - css = "text/css", - js = "application/javascript", - png = "image/png", - gif = "image/gif", - jpeg = "image/jpeg", jpg = "image/jpeg", - svg = "image/svg+xml", - }; - module:shared("/*/http_files/mime").types = mime_map; - - local mime_types, err = open(module:get_option_path("mime_types_file", "/etc/mime.types", "config"), "r"); - if mime_types then - local mime_data = mime_types:read("*a"); - mime_types:close(); - setmetatable(mime_map, { - __index = function(t, ext) - local typ = mime_data:match("\n(%S+)[^\n]*%s"..(ext:lower()).."%s") or "application/octet-stream"; - t[ext] = typ; - return typ; - end - }); - end -end local forbidden_chars_pattern = "[/%z]"; -if prosody.platform == "windows" then +if package.config:sub(1,1) == "\\" then forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]" end local urldecode = require "util.http".urldecode; -function sanitize_path(path) +local function sanitize_path(path) --> util.paths or util.http? if not path then return end local out = {}; @@ -83,15 +50,16 @@ function sanitize_path(path) return "/"..table.concat(out, "/"); end -local cache = require "util.cache".new(cache_size); - -function serve(opts) +local function serve(opts) if type(opts) ~= "table" then -- assume path string opts = { path = opts }; end + local mime_map = opts.mime_map or { html = "text/html" }; + local cache = new_cache(opts.cache_size or 256); + local cache_max_file_size = tonumber(opts.cache_max_file_size) or 1024 -- luacheck: ignore 431 local base_path = opts.path; - local dir_indices = opts.index_files or dir_indices; + local dir_indices = opts.index_files or { "index.html", "index.htm" }; local directory_index = opts.directory_index; local function serve_file(event, path) local request, response = event.request, event.response; @@ -151,7 +119,7 @@ function serve(opts) else local f, err = open(full_path, "rb"); if not f then - module:log("debug", "Could not open %s. Error was %s", full_path, err); + log("debug", "Could not open %s. Error was %s", full_path, err); return 403; end local ext = full_path:match("%.([^./]+)$"); @@ -159,7 +127,7 @@ function serve(opts) response_headers.content_type = content_type; if attr.size > cache_max_file_size then response_headers.content_length = attr.size; - module:log("debug", "%d > cache_max_file_size", attr.size); + log("debug", "%d > cache_max_file_size", attr.size); return response:send_file(f); else data = f:read("*a"); @@ -174,25 +142,7 @@ function serve(opts) return serve_file; end -function wrap_route(routes) - for route,handler in pairs(routes) do - if type(handler) ~= "function" then - routes[route] = serve(handler); - end - end - return routes; -end - -if base_path then - module:provides("http", { - route = { - ["GET /*"] = serve { - path = base_path; - directory_index = directory_index; - } - }; - }); -else - module:log("debug", "http_files_dir not set, assuming use by some other module"); -end +return { + serve = serve; +} -- cgit v1.2.3 From 568fc99fea90769ea7ddac77cbf1028ca3c332d2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 5 Apr 2019 17:12:19 +0200 Subject: mod_http_files: Use net.http.files --- plugins/mod_http_files.lua | 169 +++++++++------------------------------------ 1 file changed, 33 insertions(+), 136 deletions(-) diff --git a/plugins/mod_http_files.lua b/plugins/mod_http_files.lua index 1dae0d6d..acb99501 100644 --- a/plugins/mod_http_files.lua +++ b/plugins/mod_http_files.lua @@ -7,14 +7,9 @@ -- module:depends("http"); -local server = require"net.http.server"; -local lfs = require "lfs"; -local os_date = os.date; local open = io.open; -local stat = lfs.attributes; -local build_path = require"socket.url".build_path; -local path_sep = package.config:sub(1,1); +local fileserver = require"net.http.files"; local base_path = module:get_option_path("http_files_dir", module:get_option_path("http_path")); local cache_size = module:get_option_number("http_files_cache_size", 128); @@ -51,148 +46,50 @@ if not mime_map then end end -local forbidden_chars_pattern = "[/%z]"; -if prosody.platform == "windows" then - forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]" -end - -local urldecode = require "util.http".urldecode; -function sanitize_path(path) - if not path then return end - local out = {}; - - local c = 0; - for component in path:gmatch("([^/]+)") do - component = urldecode(component); - if component:find(forbidden_chars_pattern) then - return nil; - elseif component == ".." then - if c <= 0 then - return nil; - end - out[c] = nil; - c = c - 1; - elseif component ~= "." then - c = c + 1; - out[c] = component; - end - end - if path:sub(-1,-1) == "/" then - out[c+1] = ""; - end - return "/"..table.concat(out, "/"); -end - -local cache = require "util.cache".new(cache_size); - +-- COMPAT -- TODO deprecate function serve(opts) if type(opts) ~= "table" then -- assume path string opts = { path = opts }; end - -- luacheck: ignore 431 - local base_path = opts.path; - local dir_indices = opts.index_files or dir_indices; - local directory_index = opts.directory_index; - local function serve_file(event, path) - local request, response = event.request, event.response; - local sanitized_path = sanitize_path(path); - if path and not sanitized_path then - return 400; - end - path = sanitized_path; - local orig_path = sanitize_path(request.path); - local full_path = base_path .. (path or ""):gsub("/", path_sep); - local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows - if not attr then - return 404; - end - - local request_headers, response_headers = request.headers, response.headers; - - local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification); - response_headers.last_modified = last_modified; - - local etag = ('"%02x-%x-%x-%x"'):format(attr.dev or 0, attr.ino or 0, attr.size or 0, attr.modification or 0); - response_headers.etag = etag; - - local if_none_match = request_headers.if_none_match - local if_modified_since = request_headers.if_modified_since; - if etag == if_none_match - or (not if_none_match and last_modified == if_modified_since) then - return 304; - end - - local data = cache:get(orig_path); - if data and data.etag == etag then - response_headers.content_type = data.content_type; - data = data.data; - elseif attr.mode == "directory" and path then - if full_path:sub(-1) ~= "/" then - local dir_path = { is_absolute = true, is_directory = true }; - for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end - response_headers.location = build_path(dir_path); - return 301; - end - for i=1,#dir_indices do - if stat(full_path..dir_indices[i], "mode") == "file" then - return serve_file(event, path..dir_indices[i]); - end - end - - if directory_index then - data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path }); - end - if not data then - return 403; - end - cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; }); - response_headers.content_type = mime_map.html; - - else - local f, err = open(full_path, "rb"); - if not f then - module:log("debug", "Could not open %s. Error was %s", full_path, err); - return 403; - end - local ext = full_path:match("%.([^./]+)$"); - local content_type = ext and mime_map[ext]; - response_headers.content_type = content_type; - if attr.size > cache_max_file_size then - response_headers.content_length = attr.size; - module:log("debug", "%d > cache_max_file_size", attr.size); - return response:send_file(f); - else - data = f:read("*a"); - f:close(); - end - cache:set(orig_path, { data = data; content_type = content_type; etag = etag }); - end - - return response:send(data); + if opts.directory_index == nil then + opts.directory_index = directory_index; end - - return serve_file; + if opts.mime_map == nil then + opts.mime_map = mime_map; + end + if opts.cache_size == nil then + opts.cache_size = cache_size; + end + if opts.cache_max_file_size == nil then + opts.cache_max_file_size = cache_max_file_size; + end + if opts.index_files == nil then + opts.index_files = dir_indices; + end + -- TODO Crank up to warning + module:log("debug", "Use of mod_http_files.serve() should be updated to use net.http.files"); + return fileserver.serve(opts); end function wrap_route(routes) + module:log("debug", "Use of mod_http_files.wrap_route() should be updated to use net.http.files"); for route,handler in pairs(routes) do if type(handler) ~= "function" then - routes[route] = serve(handler); + routes[route] = fileserver.serve(handler); end end return routes; end -if base_path then - module:provides("http", { - route = { - ["GET /*"] = serve { - path = base_path; - directory_index = directory_index; - } - }; - }); -else - module:log("debug", "http_files_dir not set, assuming use by some other module"); -end - +module:provides("http", { + route = { + ["GET /*"] = fileserver.serve({ + path = base_path; + directory_index = directory_index; + mime_map = mime_map; + cache_size = cache_size; + cache_max_file_size = cache_max_file_size; + index_files = dir_indices; + }); + }; +}); -- cgit v1.2.3 From 9163493840c796d31feabdf3d3d379442e3a7020 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 5 Apr 2019 18:18:23 +0200 Subject: mod_http_files: Try to determine which module using serve() needs updating --- plugins/mod_http_files.lua | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/mod_http_files.lua b/plugins/mod_http_files.lua index acb99501..c357ddfc 100644 --- a/plugins/mod_http_files.lua +++ b/plugins/mod_http_files.lua @@ -46,6 +46,12 @@ if not mime_map then end end +local function get_calling_module() + local info = debug.getinfo(3, "S"); + if not info then return "An unknown module"; end + return info.source:match"mod_[^/\\.]+" or info.short_src; +end + -- COMPAT -- TODO deprecate function serve(opts) if type(opts) ~= "table" then -- assume path string @@ -67,12 +73,12 @@ function serve(opts) opts.index_files = dir_indices; end -- TODO Crank up to warning - module:log("debug", "Use of mod_http_files.serve() should be updated to use net.http.files"); + module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module()); return fileserver.serve(opts); end function wrap_route(routes) - module:log("debug", "Use of mod_http_files.wrap_route() should be updated to use net.http.files"); + module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module()); for route,handler in pairs(routes) do if type(handler) ~= "function" then routes[route] = fileserver.serve(handler); -- cgit v1.2.3 From f5dd9bf4c61dcbc768490cabb25bf58ed0111eaf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 13 Apr 2019 23:55:34 +0200 Subject: util.poll: Minimize scope of methods File scope is enough --- util-src/poll.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/util-src/poll.c b/util-src/poll.c index 0ca0cf28..1e7b6da3 100644 --- a/util-src/poll.c +++ b/util-src/poll.c @@ -59,7 +59,7 @@ typedef struct Lpoll_state { /* * Add an FD to be watched */ -int Ladd(lua_State *L) { +static int Ladd(lua_State *L) { struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT); int fd = luaL_checkinteger(L, 2); @@ -137,7 +137,7 @@ int Ladd(lua_State *L) { /* * Set events to watch for, readable and/or writable */ -int Lset(lua_State *L) { +static int Lset(lua_State *L) { struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT); int fd = luaL_checkinteger(L, 2); @@ -200,7 +200,7 @@ int Lset(lua_State *L) { /* * Remove FDs */ -int Ldel(lua_State *L) { +static int Ldel(lua_State *L) { struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT); int fd = luaL_checkinteger(L, 2); @@ -247,7 +247,7 @@ int Ldel(lua_State *L) { /* * Check previously manipulated event state for FDs ready for reading or writing */ -int Lpushevent(lua_State *L, struct Lpoll_state *state) { +static int Lpushevent(lua_State *L, struct Lpoll_state *state) { #ifdef USE_EPOLL if(state->processed > 0) { @@ -281,7 +281,7 @@ int Lpushevent(lua_State *L, struct Lpoll_state *state) { /* * Wait for event */ -int Lwait(lua_State *L) { +static int Lwait(lua_State *L) { struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT); int ret = Lpushevent(L, state); @@ -344,7 +344,7 @@ int Lwait(lua_State *L) { /* * Return Epoll FD */ -int Lgetfd(lua_State *L) { +static int Lgetfd(lua_State *L) { struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT); lua_pushinteger(L, state->epoll_fd); return 1; @@ -353,7 +353,7 @@ int Lgetfd(lua_State *L) { /* * Close epoll FD */ -int Lgc(lua_State *L) { +static int Lgc(lua_State *L) { struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT); if(state->epoll_fd == -1) { @@ -375,7 +375,7 @@ int Lgc(lua_State *L) { /* * String representation */ -int Ltos(lua_State *L) { +static int Ltos(lua_State *L) { struct Lpoll_state *state = luaL_checkudata(L, 1, STATE_MT); lua_pushfstring(L, "%s: %p", STATE_MT, state); return 1; @@ -384,7 +384,7 @@ int Ltos(lua_State *L) { /* * Create a new context */ -int Lnew(lua_State *L) { +static int Lnew(lua_State *L) { /* Allocate state */ Lpoll_state *state = lua_newuserdata(L, sizeof(Lpoll_state)); luaL_setmetatable(L, STATE_MT); -- cgit v1.2.3 From 8041f3f5490a8f4893316437ab01415c516dcaa7 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 19 Apr 2019 12:41:38 +0200 Subject: util.hashes: Remove redundant semicolon --- util-src/hashes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-src/hashes.c b/util-src/hashes.c index 82f5876e..5cf8b5e7 100644 --- a/util-src/hashes.c +++ b/util-src/hashes.c @@ -215,7 +215,7 @@ LUALIB_API int luaopen_util_hashes(lua_State *L) { luaL_checkversion(L); #endif lua_newtable(L); - luaL_setfuncs(L, Reg, 0);; + luaL_setfuncs(L, Reg, 0); lua_pushliteral(L, "-3.14"); lua_setfield(L, -2, "version"); return 1; -- cgit v1.2.3 From 8e2e187ff819b98ebae24f2a5ebd9994a2d41c2f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 19 Apr 2019 12:46:24 +0200 Subject: util.hmac: Reflow code --- util/hmac.lua | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/util/hmac.lua b/util/hmac.lua index 2c4cc6ef..f3caae33 100644 --- a/util/hmac.lua +++ b/util/hmac.lua @@ -10,6 +10,8 @@ local hashes = require "util.hashes" -return { md5 = hashes.hmac_md5, - sha1 = hashes.hmac_sha1, - sha256 = hashes.hmac_sha256 }; +return { + md5 = hashes.hmac_md5, + sha1 = hashes.hmac_sha1, + sha256 = hashes.hmac_sha256, +}; -- cgit v1.2.3 From 1d11df9a67d5f650d1ff4404ba7d6668df99875e Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 19 Apr 2019 12:47:49 +0200 Subject: util.hmac: Expose hmac-sha-512 too All these are provided by util.hashes so why not? --- util/hmac.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/util/hmac.lua b/util/hmac.lua index f3caae33..4cad17cc 100644 --- a/util/hmac.lua +++ b/util/hmac.lua @@ -14,4 +14,5 @@ return { md5 = hashes.hmac_md5, sha1 = hashes.hmac_sha1, sha256 = hashes.hmac_sha256, + sha512 = hashes.hmac_sha512, }; -- cgit v1.2.3 From 3752c51a3a2e27a1d5231e9acb27deb864024bb0 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 11 Apr 2019 00:41:48 +0200 Subject: util.hmac: Generate test cases from RFC 4231 --- spec/util_hmac_spec.lua | 103 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 spec/util_hmac_spec.lua diff --git a/spec/util_hmac_spec.lua b/spec/util_hmac_spec.lua new file mode 100644 index 00000000..abcc92f1 --- /dev/null +++ b/spec/util_hmac_spec.lua @@ -0,0 +1,103 @@ +-- Test cases from RFC 4231 + +local hmac = require "util.hmac"; +local hex = require "util.hex"; + +describe("Test case 1", function () + local Key = hex.from("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"); + local Data = hex.from("4869205468657265"); + describe("HMAC-SHA-256", function () + it("works", function() + assert.equal("b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", hmac.sha256(Key, Data, true)) + end); + end); + describe("HMAC-SHA-512", function () + it("works", function() + assert.equal("87aa7cdea5ef619d4ff0b4241a1d6cb02379f4e2ce4ec2787ad0b30545e17cdedaa833b7d6b8a702038b274eaea3f4e4be9d914eeb61f1702e696c203a126854", hmac.sha512(Key, Data, true)) + end); + end); +end); +describe("Test case 2", function () + local Key = hex.from("4a656665"); + local Data = hex.from("7768617420646f2079612077616e7420666f72206e6f7468696e673f"); + describe("HMAC-SHA-256", function () + it("works", function() + assert.equal("5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", hmac.sha256(Key, Data, true)) + end); + end); + describe("HMAC-SHA-512", function () + it("works", function() + assert.equal("164b7a7bfcf819e2e395fbe73b56e0a387bd64222e831fd610270cd7ea2505549758bf75c05a994a6d034f65f8f0e6fdcaeab1a34d4a6b4b636e070a38bce737", hmac.sha512(Key, Data, true)) + end); + end); +end); +describe("Test case 3", function () + local Key = hex.from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + local Data = hex.from("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"); + describe("HMAC-SHA-256", function () + it("works", function() + assert.equal("773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", hmac.sha256(Key, Data, true)) + end); + end); + describe("HMAC-SHA-512", function () + it("works", function() + assert.equal("fa73b0089d56a284efb0f0756c890be9b1b5dbdd8ee81a3655f83e33b2279d39bf3e848279a722c806b485a47e67c807b946a337bee8942674278859e13292fb", hmac.sha512(Key, Data, true)) + end); + end); +end); +describe("Test case 4", function () + local Key = hex.from("0102030405060708090a0b0c0d0e0f10111213141516171819"); + local Data = hex.from("cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd"); + describe("HMAC-SHA-256", function () + it("works", function() + assert.equal("82558a389a443c0ea4cc819899f2083a85f0faa3e578f8077a2e3ff46729665b", hmac.sha256(Key, Data, true)) + end); + end); + describe("HMAC-SHA-512", function () + it("works", function() + assert.equal("b0ba465637458c6990e5a8c5f61d4af7e576d97ff94b872de76f8050361ee3dba91ca5c11aa25eb4d679275cc5788063a5f19741120c4f2de2adebeb10a298dd", hmac.sha512(Key, Data, true)) + end); + end); +end); +describe("Test case 5", function () + local Key = hex.from("0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c"); + local Data = hex.from("546573742057697468205472756e636174696f6e"); + describe("HMAC-SHA-256", function () + it("works", function() + assert.equal("a3b6167473100ee06e0c796c2955552b", hmac.sha256(Key, Data, true):sub(1,128/4)) + end); + end); + describe("HMAC-SHA-512", function () + it("works", function() + assert.equal("415fad6271580a531d4179bc891d87a6", hmac.sha512(Key, Data, true):sub(1,128/4)) + end); + end); +end); +describe("Test case 6", function () + local Key = hex.from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + local Data = hex.from("54657374205573696e67204c6172676572205468616e20426c6f636b2d53697a65204b6579202d2048617368204b6579204669727374"); + describe("HMAC-SHA-256", function () + it("works", function() + assert.equal("60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54", hmac.sha256(Key, Data, true)) + end); + end); + describe("HMAC-SHA-512", function () + it("works", function() + assert.equal("80b24263c7c1a3ebb71493c1dd7be8b49b46d1f41b4aeec1121b013783f8f3526b56d037e05f2598bd0fd2215d6a1e5295e64f73f63f0aec8b915a985d786598", hmac.sha512(Key, Data, true)) + end); + end); +end); +describe("Test case 7", function () + local Key = hex.from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + local Data = hex.from("5468697320697320612074657374207573696e672061206c6172676572207468616e20626c6f636b2d73697a65206b657920616e642061206c6172676572207468616e20626c6f636b2d73697a6520646174612e20546865206b6579206e6565647320746f20626520686173686564206265666f7265206265696e6720757365642062792074686520484d414320616c676f726974686d2e"); + describe("HMAC-SHA-256", function () + it("works", function() + assert.equal("9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2", hmac.sha256(Key, Data, true)) + end); + end); + describe("HMAC-SHA-512", function () + it("works", function() + assert.equal("e37b6a775dc87dbaa4dfa9f96e5e3ffddebd71f8867289865df5a32d20cdc944b6022cac3c4982b10d5eeb55c3e4de15134676fb6de0446065c97440fa8c6a58", hmac.sha512(Key, Data, true)) + end); + end); +end); -- cgit v1.2.3 From 593ab1bb69e15586b9da9d9fbd1f9e5fd8878b13 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 19 Apr 2019 13:17:49 +0200 Subject: util.hmac: Ignore long hex lines in tests --- spec/util_hmac_spec.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/util_hmac_spec.lua b/spec/util_hmac_spec.lua index abcc92f1..a2125c3a 100644 --- a/spec/util_hmac_spec.lua +++ b/spec/util_hmac_spec.lua @@ -1,5 +1,8 @@ -- Test cases from RFC 4231 +-- Yes, the lines are long, it's annoying to split the long hex things. +-- luacheck: ignore 631 + local hmac = require "util.hmac"; local hex = require "util.hex"; -- cgit v1.2.3 From eecd66b9941cc7e07b9a019fad141996b6b1a4a6 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 19 Apr 2019 14:12:28 +0200 Subject: util.hashes: Add test vectors from RFC 6070 for PBKDF2 (aka SCRAM Hi()) Number 4 is disabled by default beacuse of how long time it takes --- .busted | 2 +- spec/util_hashes_spec.lua | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 spec/util_hashes_spec.lua diff --git a/.busted b/.busted index f5f9472e..0206c093 100644 --- a/.busted +++ b/.busted @@ -2,7 +2,7 @@ return { _all = { }, default = { - ["exclude-tags"] = "mod_bosh,storage"; + ["exclude-tags"] = "mod_bosh,storage,SLOW"; }; bosh = { tags = "mod_bosh"; diff --git a/spec/util_hashes_spec.lua b/spec/util_hashes_spec.lua new file mode 100644 index 00000000..1e6187bb --- /dev/null +++ b/spec/util_hashes_spec.lua @@ -0,0 +1,37 @@ +-- Test vectors from RFC 6070 +local hashes = require "util.hashes"; +local hex = require "util.hex"; + +-- Also see spec for util.hmac where HMAC test cases reside + +describe("PBKDF2-SHA1", function () + it("test vector 1", function () + local P = "password" + local S = "salt" + local c = 1 + local DK = "0c60c80f961f0e71f3a9b524af6012062fe037a6"; + assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c))); + end); + it("test vector 2", function () + local P = "password" + local S = "salt" + local c = 2 + local DK = "ea6c014dc72d6f8ccd1ed92ace1d41f0d8de8957"; + assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c))); + end); + it("test vector 3", function () + local P = "password" + local S = "salt" + local c = 4096 + local DK = "4b007901b765489abead49d926f721d065a429c1"; + assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c))); + end); + it("test vector 4 #SLOW", function () + local P = "password" + local S = "salt" + local c = 16777216 + local DK = "eefe3d61cd4da4e4e9945b3d6ba2158c2634e984"; + assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c))); + end); +end); + -- cgit v1.2.3 From 3f3280d0c0bf789ecace5e3024ab24a9dc7fd339 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 13 Jan 2019 13:57:14 +0100 Subject: util.hashes: Use PBKDF2 from libcrypto --- util-src/hashes.c | 53 +++++++++-------------------------------------------- 1 file changed, 9 insertions(+), 44 deletions(-) diff --git a/util-src/hashes.c b/util-src/hashes.c index 8de4ef5b..ac6cac7e 100644 --- a/util-src/hashes.c +++ b/util-src/hashes.c @@ -26,6 +26,7 @@ typedef unsigned __int32 uint32_t; #include #include #include +#include #if (LUA_VERSION_NUM == 501) #define luaL_setfuncs(L, R, N) luaL_register(L, NULL, R) @@ -137,54 +138,18 @@ MAKE_HMAC_FUNCTION(Lhmac_sha512, EVP_sha512, SHA512_DIGEST_LENGTH, SHA512_CTX) MAKE_HMAC_FUNCTION(Lhmac_md5, EVP_md5, MD5_DIGEST_LENGTH, MD5_CTX) static int LscramHi(lua_State *L) { - union xory { - unsigned char bytes[SHA_DIGEST_LENGTH]; - uint32_t quadbytes[SHA_DIGEST_LENGTH / 4]; - }; - int i; - SHA_CTX ctx, ctxo; - unsigned char Ust[SHA_DIGEST_LENGTH]; - union xory Und; - union xory res; - size_t str_len, salt_len; - struct hash_desc desc; - const char *str = luaL_checklstring(L, 1, &str_len); - const char *salt = luaL_checklstring(L, 2, &salt_len); - char *salt2; - const int iter = luaL_checkinteger(L, 3); - - desc.Init = (int (*)(void *))SHA1_Init; - desc.Update = (int (*)(void *, const void *, size_t))SHA1_Update; - desc.Final = (int (*)(unsigned char *, void *))SHA1_Final; - desc.digestLength = SHA_DIGEST_LENGTH; - desc.ctx = &ctx; - desc.ctxo = &ctxo; - - salt2 = malloc(salt_len + 4); + unsigned char out[SHA_DIGEST_LENGTH]; - if(salt2 == NULL) { - return luaL_error(L, "Out of memory in scramHi"); - } - - memcpy(salt2, salt, salt_len); - memcpy(salt2 + salt_len, "\0\0\0\1", 4); - hmac(&desc, str, str_len, salt2, salt_len + 4, Ust); - free(salt2); - - memcpy(res.bytes, Ust, sizeof(res)); - - for(i = 1; i < iter; i++) { - int j; - hmac(&desc, str, str_len, (char *)Ust, sizeof(Ust), Und.bytes); - - for(j = 0; j < SHA_DIGEST_LENGTH / 4; j++) { - res.quadbytes[j] ^= Und.quadbytes[j]; - } + size_t pass_len, salt_len; + const char *pass = luaL_checklstring(L, 1, &pass_len); + const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len); + const int iter = luaL_checkinteger(L, 3); - memcpy(Ust, Und.bytes, sizeof(Ust)); + if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha1(), SHA_DIGEST_LENGTH, out) == 0) { + return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed"); } - lua_pushlstring(L, (char *)res.bytes, SHA_DIGEST_LENGTH); + lua_pushlstring(L, (char *)out, SHA_DIGEST_LENGTH); return 1; } -- cgit v1.2.3 From 2a2cdcecdf15dc13f939cf9b83959e9929a94eb4 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 19 Apr 2019 13:24:32 +0200 Subject: util.hashes: Remove now unused hmac() function --- util-src/hashes.c | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/util-src/hashes.c b/util-src/hashes.c index ac6cac7e..75eb116d 100644 --- a/util-src/hashes.c +++ b/util-src/hashes.c @@ -76,44 +76,6 @@ struct hash_desc { void *ctx, *ctxo; }; -static void hmac(struct hash_desc *desc, const char *key, size_t key_len, - const char *msg, size_t msg_len, unsigned char *result) { - union xory { - unsigned char bytes[64]; - uint32_t quadbytes[16]; - }; - - int i; - unsigned char hashedKey[64]; /* Maximum used digest length */ - union xory k_ipad, k_opad; - - if(key_len > 64) { - desc->Init(desc->ctx); - desc->Update(desc->ctx, key, key_len); - desc->Final(hashedKey, desc->ctx); - key = (const char *)hashedKey; - key_len = desc->digestLength; - } - - memcpy(k_ipad.bytes, key, key_len); - memset(k_ipad.bytes + key_len, 0, 64 - key_len); - memcpy(k_opad.bytes, k_ipad.bytes, 64); - - for(i = 0; i < 16; i++) { - k_ipad.quadbytes[i] ^= HMAC_IPAD; - k_opad.quadbytes[i] ^= HMAC_OPAD; - } - - desc->Init(desc->ctx); - desc->Update(desc->ctx, k_ipad.bytes, 64); - desc->Init(desc->ctxo); - desc->Update(desc->ctxo, k_opad.bytes, 64); - desc->Update(desc->ctx, msg, msg_len); - desc->Final(result, desc->ctx); - desc->Update(desc->ctxo, result, desc->digestLength); - desc->Final(result, desc->ctxo); -} - #define MAKE_HMAC_FUNCTION(myFunc, evp, size, type) \ static int myFunc(lua_State *L) { \ unsigned char hash[size], result[2*size]; \ -- cgit v1.2.3 From 701d64ed8507cf2becd68a4285b66d340f0d9e5a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 13 Jan 2019 13:57:18 +0100 Subject: util.hashes: Rename PBKDF2 function It's not SCRAM-specific --- util-src/hashes.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/util-src/hashes.c b/util-src/hashes.c index 75eb116d..7e790e9f 100644 --- a/util-src/hashes.c +++ b/util-src/hashes.c @@ -99,7 +99,7 @@ MAKE_HMAC_FUNCTION(Lhmac_sha256, EVP_sha256, SHA256_DIGEST_LENGTH, SHA256_CTX) MAKE_HMAC_FUNCTION(Lhmac_sha512, EVP_sha512, SHA512_DIGEST_LENGTH, SHA512_CTX) MAKE_HMAC_FUNCTION(Lhmac_md5, EVP_md5, MD5_DIGEST_LENGTH, MD5_CTX) -static int LscramHi(lua_State *L) { +static int Lpbkdf2_sha1(lua_State *L) { unsigned char out[SHA_DIGEST_LENGTH]; size_t pass_len, salt_len; @@ -127,7 +127,8 @@ static const luaL_Reg Reg[] = { { "hmac_sha256", Lhmac_sha256 }, { "hmac_sha512", Lhmac_sha512 }, { "hmac_md5", Lhmac_md5 }, - { "scram_Hi_sha1", LscramHi }, + { "scram_Hi_sha1", Lpbkdf2_sha1 }, /* COMPAT */ + { "pbkdf2_hmac_sha1", Lpbkdf2_sha1 }, { NULL, NULL } }; -- cgit v1.2.3 From d73c6a8327c79775989303c823bfa9be22e26116 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 13 Jan 2019 13:59:26 +0100 Subject: util.hashes: Add PBKDF2-HMAC-SHA256 --- util-src/hashes.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/util-src/hashes.c b/util-src/hashes.c index 7e790e9f..4c48b26f 100644 --- a/util-src/hashes.c +++ b/util-src/hashes.c @@ -116,6 +116,23 @@ static int Lpbkdf2_sha1(lua_State *L) { return 1; } + +static int Lpbkdf2_sha256(lua_State *L) { + unsigned char out[SHA256_DIGEST_LENGTH]; + + size_t pass_len, salt_len; + const char *pass = luaL_checklstring(L, 1, &pass_len); + const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len); + const int iter = luaL_checkinteger(L, 3); + + if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha256(), SHA256_DIGEST_LENGTH, out) == 0) { + return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed"); + } + + lua_pushlstring(L, (char *)out, SHA_DIGEST_LENGTH); + return 1; +} + static const luaL_Reg Reg[] = { { "sha1", Lsha1 }, { "sha224", Lsha224 }, @@ -129,6 +146,7 @@ static const luaL_Reg Reg[] = { { "hmac_md5", Lhmac_md5 }, { "scram_Hi_sha1", Lpbkdf2_sha1 }, /* COMPAT */ { "pbkdf2_hmac_sha1", Lpbkdf2_sha1 }, + { "pbkdf2_hmac_sha256", Lpbkdf2_sha256 }, { NULL, NULL } }; -- cgit v1.2.3 From 265d5cbec07c6297c811b72cf5416acb446869da Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 20 Apr 2019 15:11:04 +0200 Subject: util.hashes: Allow specifying output key length This is not needed for SCRAM but PBKDF2 takes this argument. --- spec/util_hashes_spec.lua | 16 ++++++++++++++++ util-src/hashes.c | 21 ++++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/spec/util_hashes_spec.lua b/spec/util_hashes_spec.lua index 1e6187bb..9099145a 100644 --- a/spec/util_hashes_spec.lua +++ b/spec/util_hashes_spec.lua @@ -33,5 +33,21 @@ describe("PBKDF2-SHA1", function () local DK = "eefe3d61cd4da4e4e9945b3d6ba2158c2634e984"; assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c))); end); + it("test vector 5", function () + local P = "passwordPASSWORDpassword" + local S = "saltSALTsaltSALTsaltSALTsaltSALTsalt" + local c = 4096 + local dkLen = 25 + local DK = "3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038" + assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c, dkLen))); + end); + it("works", function () + local P = "pass\0word" + local S = "sa\0lt" + local c = 4096 + local dkLen = 16 + local DK = "56fa6aa75548099dcc37d7f03425e0c3" + assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c, dkLen))); + end); end); diff --git a/util-src/hashes.c b/util-src/hashes.c index 4c48b26f..3fb849b9 100644 --- a/util-src/hashes.c +++ b/util-src/hashes.c @@ -100,36 +100,39 @@ MAKE_HMAC_FUNCTION(Lhmac_sha512, EVP_sha512, SHA512_DIGEST_LENGTH, SHA512_CTX) MAKE_HMAC_FUNCTION(Lhmac_md5, EVP_md5, MD5_DIGEST_LENGTH, MD5_CTX) static int Lpbkdf2_sha1(lua_State *L) { - unsigned char out[SHA_DIGEST_LENGTH]; - size_t pass_len, salt_len; const char *pass = luaL_checklstring(L, 1, &pass_len); const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len); const int iter = luaL_checkinteger(L, 3); + const size_t len = luaL_optinteger(L, 4, SHA_DIGEST_LENGTH); + + luaL_Buffer b; + unsigned char *out = (unsigned char *)luaL_buffinitsize(L, &b, len); - if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha1(), SHA_DIGEST_LENGTH, out) == 0) { + if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha1(), len, out) == 0) { return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed"); } - lua_pushlstring(L, (char *)out, SHA_DIGEST_LENGTH); - + luaL_pushresultsize(&b, len); return 1; } static int Lpbkdf2_sha256(lua_State *L) { - unsigned char out[SHA256_DIGEST_LENGTH]; - size_t pass_len, salt_len; const char *pass = luaL_checklstring(L, 1, &pass_len); const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len); const int iter = luaL_checkinteger(L, 3); + const int len = luaL_optinteger(L, 4, SHA256_DIGEST_LENGTH); + + luaL_Buffer b; + unsigned char *out = (unsigned char *)luaL_buffinitsize(L, &b, len); - if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha256(), SHA256_DIGEST_LENGTH, out) == 0) { + if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha256(), len, out) == 0) { return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed"); } - lua_pushlstring(L, (char *)out, SHA_DIGEST_LENGTH); + luaL_pushresultsize(&b, len); return 1; } -- cgit v1.2.3 From 65ca6b82048f1b260291ea9d920d3b117a2603b4 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 21 Apr 2019 00:59:36 +0200 Subject: Backed out changeset 61bc5c52c941 luaL_buffinitsize is only available in Lua 5.2+ --- spec/util_hashes_spec.lua | 16 ---------------- util-src/hashes.c | 21 +++++++++------------ 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/spec/util_hashes_spec.lua b/spec/util_hashes_spec.lua index 9099145a..1e6187bb 100644 --- a/spec/util_hashes_spec.lua +++ b/spec/util_hashes_spec.lua @@ -33,21 +33,5 @@ describe("PBKDF2-SHA1", function () local DK = "eefe3d61cd4da4e4e9945b3d6ba2158c2634e984"; assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c))); end); - it("test vector 5", function () - local P = "passwordPASSWORDpassword" - local S = "saltSALTsaltSALTsaltSALTsaltSALTsalt" - local c = 4096 - local dkLen = 25 - local DK = "3d2eec4fe41c849b80c8d83662c0e44a8b291a964cf2f07038" - assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c, dkLen))); - end); - it("works", function () - local P = "pass\0word" - local S = "sa\0lt" - local c = 4096 - local dkLen = 16 - local DK = "56fa6aa75548099dcc37d7f03425e0c3" - assert.equal(DK, hex.to(hashes.scram_Hi_sha1(P, S, c, dkLen))); - end); end); diff --git a/util-src/hashes.c b/util-src/hashes.c index 3fb849b9..4c48b26f 100644 --- a/util-src/hashes.c +++ b/util-src/hashes.c @@ -100,39 +100,36 @@ MAKE_HMAC_FUNCTION(Lhmac_sha512, EVP_sha512, SHA512_DIGEST_LENGTH, SHA512_CTX) MAKE_HMAC_FUNCTION(Lhmac_md5, EVP_md5, MD5_DIGEST_LENGTH, MD5_CTX) static int Lpbkdf2_sha1(lua_State *L) { + unsigned char out[SHA_DIGEST_LENGTH]; + size_t pass_len, salt_len; const char *pass = luaL_checklstring(L, 1, &pass_len); const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len); const int iter = luaL_checkinteger(L, 3); - const size_t len = luaL_optinteger(L, 4, SHA_DIGEST_LENGTH); - - luaL_Buffer b; - unsigned char *out = (unsigned char *)luaL_buffinitsize(L, &b, len); - if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha1(), len, out) == 0) { + if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha1(), SHA_DIGEST_LENGTH, out) == 0) { return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed"); } - luaL_pushresultsize(&b, len); + lua_pushlstring(L, (char *)out, SHA_DIGEST_LENGTH); + return 1; } static int Lpbkdf2_sha256(lua_State *L) { + unsigned char out[SHA256_DIGEST_LENGTH]; + size_t pass_len, salt_len; const char *pass = luaL_checklstring(L, 1, &pass_len); const unsigned char *salt = (unsigned char *)luaL_checklstring(L, 2, &salt_len); const int iter = luaL_checkinteger(L, 3); - const int len = luaL_optinteger(L, 4, SHA256_DIGEST_LENGTH); - - luaL_Buffer b; - unsigned char *out = (unsigned char *)luaL_buffinitsize(L, &b, len); - if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha256(), len, out) == 0) { + if(PKCS5_PBKDF2_HMAC(pass, pass_len, salt, salt_len, iter, EVP_sha256(), SHA256_DIGEST_LENGTH, out) == 0) { return luaL_error(L, "PKCS5_PBKDF2_HMAC() failed"); } - luaL_pushresultsize(&b, len); + lua_pushlstring(L, (char *)out, SHA_DIGEST_LENGTH); return 1; } -- cgit v1.2.3 From cb8a085c3bcbec83ad6b65631f8189e406bca042 Mon Sep 17 00:00:00 2001 From: Jo?o Duarte Date: Wed, 17 Apr 2019 10:11:22 -0700 Subject: mod_admin_telnet: Adds c2s:closeall() (Fixes #1315) --- plugins/mod_admin_telnet.lua | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 34cc4dc6..c66630ca 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -250,6 +250,7 @@ function commands.help(session, data) print [[c2s:show_secure() - Show all encrypted client connections]] print [[c2s:show_tls() - Show TLS cipher info for encrypted sessions]] print [[c2s:close(jid) - Close all sessions for the specified JID]] + print [[c2s:closeall() - Close all active c2s connections ]] elseif section == "s2s" then print [[s2s:show(domain) - Show all s2s connections for the given domain (or all if no domain given)]] print [[s2s:show_tls(domain) - Show TLS cipher info for encrypted sessions]] @@ -661,6 +662,16 @@ function def_env.c2s:close(match_jid) return true, "Total: "..count.." sessions closed"; end +function def_env.c2s:closeall() + local count = 0; + --luacheck: ignore 212/jid + show_c2s(function (jid, session) + count = count + 1; + session:close(); + end); + return true, "Total: "..count.." sessions closed"; +end + def_env.s2s = {}; function def_env.s2s:show(match_jid, annotate) -- cgit v1.2.3 From 83524f2d6a28d82db56233a5cf3660bb726fdb7a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 24 Apr 2019 22:40:38 +0200 Subject: util.encodings: Add binding to confusables skeleton function in ICU --- util-src/encodings.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/util-src/encodings.c b/util-src/encodings.c index e55a3f44..0d723913 100644 --- a/util-src/encodings.c +++ b/util-src/encodings.c @@ -268,6 +268,7 @@ static const luaL_Reg Reg_utf8[] = { #include #include #include +#include static int icu_stringprep_prep(lua_State *L, const UStringPrepProfile *profile) { size_t input_len; @@ -321,6 +322,7 @@ UStringPrepProfile *icu_nameprep; UStringPrepProfile *icu_nodeprep; UStringPrepProfile *icu_resourceprep; UStringPrepProfile *icu_saslprep; +USpoofChecker *icu_spoofcheck; /* initialize global ICU stringprep profiles */ void init_icu() { @@ -330,6 +332,8 @@ void init_icu() { icu_nodeprep = usprep_openByType(USPREP_RFC3920_NODEPREP, &err); icu_resourceprep = usprep_openByType(USPREP_RFC3920_RESOURCEPREP, &err); icu_saslprep = usprep_openByType(USPREP_RFC4013_SASLPREP, &err); + icu_spoofcheck = uspoof_open(&err); + uspoof_setChecks(icu_spoofcheck, USPOOF_CONFUSABLE, &err); if(U_FAILURE(err)) { fprintf(stderr, "[c] util.encodings: error: %s\n", u_errorName((UErrorCode)err)); @@ -477,6 +481,40 @@ static int Lidna_to_unicode(lua_State *L) { /** idna.to_unicode(s) */ } } +static int Lskeleton(lua_State *L) { + size_t len; + int32_t ulen, dest_len, output_len; + const char *s = luaL_checklstring(L, 1, &len); + UErrorCode err = U_ZERO_ERROR; + UChar ustr[1024]; + UChar dest[1024]; + char output[1024]; + + u_strFromUTF8(ustr, 1024, &ulen, s, len, &err); + + if(U_FAILURE(err)) { + lua_pushnil(L); + return 1; + } + + dest_len = uspoof_getSkeleton(icu_spoofcheck, 0, ustr, ulen, dest, 1024, &err); + + if(U_FAILURE(err)) { + lua_pushnil(L); + return 1; + } + + u_strToUTF8(output, 1024, &output_len, dest, dest_len, &err); + + if(U_SUCCESS(err)) { + lua_pushlstring(L, output, output_len); + return 1; + } + + lua_pushnil(L); + return 1; +} + #else /* USE_STRINGPREP_ICU */ /****************** libidn ********************/ @@ -558,6 +596,13 @@ LUALIB_API int luaopen_util_encodings(lua_State *L) { luaL_setfuncs(L, Reg_utf8, 0); lua_setfield(L, -2, "utf8"); +#ifdef USE_STRINGPREP_ICU + lua_newtable(L); + lua_pushcfunction(L, Lskeleton); + lua_setfield(L, -2, "skeleton"); + lua_setfield(L, -2, "confusable"); +#endif + lua_pushliteral(L, "-3.14"); lua_setfield(L, -2, "version"); return 1; -- cgit v1.2.3 From 150569cb8975a73f044c57684a004b4247005093 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Apr 2019 03:05:25 +0200 Subject: CHANGES: New in trunk so far --- CHANGES | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGES b/CHANGES index 136b7d2b..f5771512 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,14 @@ +TRUNK +===== + +- Module statuses +- SNI support (not completely finished) +- Improved message expiry in MAM +- CORS handling now provided by mod\_http +- CSI improvements +- mod\_limits: Exempted JIDs +- Archive quotas + 0.11.0 ====== -- cgit v1.2.3 From 5252a355d02d298f4ab6449abdbdbd761f51bc88 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Apr 2019 03:07:35 +0200 Subject: CHANGES: Remove MAM change that got rebased to 0.11 --- CHANGES | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGES b/CHANGES index f5771512..e2e90f4a 100644 --- a/CHANGES +++ b/CHANGES @@ -3,7 +3,6 @@ TRUNK - Module statuses - SNI support (not completely finished) -- Improved message expiry in MAM - CORS handling now provided by mod\_http - CSI improvements - mod\_limits: Exempted JIDs -- cgit v1.2.3 From e60915744d6dcb90bbe500aee78d3820b7f89b82 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Apr 2019 15:53:52 +0200 Subject: util.encodings: Add compat with ICU before version 58 --- util-src/encodings.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/util-src/encodings.c b/util-src/encodings.c index 0d723913..3b7f322d 100644 --- a/util-src/encodings.c +++ b/util-src/encodings.c @@ -324,6 +324,11 @@ UStringPrepProfile *icu_resourceprep; UStringPrepProfile *icu_saslprep; USpoofChecker *icu_spoofcheck; +#if (U_ICU_VERSION_MAJOR_NUM < 58) +/* COMPAT */ +#define USPOOF_CONFUSABLE (USPOOF_SINGLE_SCRIPT_CONFUSABLE | USPOOF_MIXED_SCRIPT_CONFUSABLE | USPOOF_WHOLE_SCRIPT_CONFUSABLE) +#endif + /* initialize global ICU stringprep profiles */ void init_icu() { UErrorCode err = U_ZERO_ERROR; -- cgit v1.2.3 From d8a52f2c8a9b1ca96d9f9c08024e5ac55476e89b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 6 Aug 2012 15:35:27 +0200 Subject: mod_mimicking: Prevents registration of confusable usernames (by Florob) (fixes #1347) --- CHANGES | 1 + plugins/mod_mimicking.lua | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 plugins/mod_mimicking.lua diff --git a/CHANGES b/CHANGES index e2e90f4a..8c65f4ec 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,7 @@ TRUNK - CSI improvements - mod\_limits: Exempted JIDs - Archive quotas +- mod\_mimicking 0.11.0 ====== diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua new file mode 100644 index 00000000..9004957a --- /dev/null +++ b/plugins/mod_mimicking.lua @@ -0,0 +1,49 @@ +-- Prosody IM +-- Copyright (C) 2012 Florian Zeitz +-- +-- This project is MIT/X11 licensed. Please see the +-- COPYING file in the source package for more information. +-- + +local skeleton = require "util.confusable".skeleton; +local datamanager = require "util.datamanager"; +local usage = require "util.prosodyctl".show_usage; +local warn = require "util.prosodyctl".show_warning; +local users = require "usermanager".users; + +module:hook("user-registered", function(user) + datamanager.store(skeleton(user.username), user.host, "skeletons", {username = user.username}); +end); + +module:hook("user-deregistered", function(user) + datamanager.store(skeleton(user.username), user.host, "skeletons", nil); +end); + +module:hook("registration-attempt", function(user) + if datamanager.load(skeleton(user.username), user.host, "skeletons") then + user.allowed = false; + end +end); + +function module.command(arg) + if (arg[1] ~= "bootstrap" or not arg[2]) then + usage("mod_mimicking bootstrap ", "Initialize skeleton database"); + return; + end + + local host = arg[2]; + + local host_session = prosody.hosts[host]; + if not host_session then + return "No such host"; + end + local provider = host_session.users; + if not(provider) or provider.name == "null" then + usermanager.initialize_host(host); + end + storagemanager.initialize_host(host); + + for user in users(host) do + datamanager.store(skeleton(user), host, "skeletons", {username = user}); + end +end -- cgit v1.2.3 From 701fd99947b8d897a60c74b1c2f4ab39fc620876 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Apr 2019 02:40:39 +0200 Subject: mod_mimicking: Import skeleton() from current location --- plugins/mod_mimicking.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua index 9004957a..da03967f 100644 --- a/plugins/mod_mimicking.lua +++ b/plugins/mod_mimicking.lua @@ -1,11 +1,15 @@ -- Prosody IM -- Copyright (C) 2012 Florian Zeitz +-- Copyright (C) 2019 Kim Alvefur -- -- This project is MIT/X11 licensed. Please see the -- COPYING file in the source package for more information. -- -local skeleton = require "util.confusable".skeleton; +local encodings = require "util.encodings"; +assert(encodings.confusable, "This module requires that Prosody be built with ICU"); +local skeleton = encodings.confusable.skeleton; + local datamanager = require "util.datamanager"; local usage = require "util.prosodyctl".show_usage; local warn = require "util.prosodyctl".show_warning; -- cgit v1.2.3 From 1054f2438b2f56790a7d5dcdcb3f841d460d27e7 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Apr 2019 02:37:35 +0200 Subject: mod_mimicking: Hook the correct event names --- plugins/mod_mimicking.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua index da03967f..8b8d6ee3 100644 --- a/plugins/mod_mimicking.lua +++ b/plugins/mod_mimicking.lua @@ -19,11 +19,11 @@ module:hook("user-registered", function(user) datamanager.store(skeleton(user.username), user.host, "skeletons", {username = user.username}); end); -module:hook("user-deregistered", function(user) +module:hook("user-deleted", function(user) datamanager.store(skeleton(user.username), user.host, "skeletons", nil); end); -module:hook("registration-attempt", function(user) +module:hook("user-registering", function(user) if datamanager.load(skeleton(user.username), user.host, "skeletons") then user.allowed = false; end -- cgit v1.2.3 From 00887cf07d565353152332f0adb854659e237fbf Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Apr 2019 02:38:55 +0200 Subject: mod_mimicking: Use new storage API --- plugins/mod_mimicking.lua | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua index 8b8d6ee3..0da68059 100644 --- a/plugins/mod_mimicking.lua +++ b/plugins/mod_mimicking.lua @@ -10,21 +10,22 @@ local encodings = require "util.encodings"; assert(encodings.confusable, "This module requires that Prosody be built with ICU"); local skeleton = encodings.confusable.skeleton; -local datamanager = require "util.datamanager"; local usage = require "util.prosodyctl".show_usage; local warn = require "util.prosodyctl".show_warning; local users = require "usermanager".users; +local skeletons = module:open_store("skeletons"); + module:hook("user-registered", function(user) - datamanager.store(skeleton(user.username), user.host, "skeletons", {username = user.username}); + skeletons:set(skeleton(user.username), { username = user.username }); end); module:hook("user-deleted", function(user) - datamanager.store(skeleton(user.username), user.host, "skeletons", nil); + skeletons:set(skeleton(user.username), nil); end); module:hook("user-registering", function(user) - if datamanager.load(skeleton(user.username), user.host, "skeletons") then + if skeletons:get(skeleton(user.username)) then user.allowed = false; end end); -- cgit v1.2.3 From 54d0329e4c0809920c954a45e448263d3a669a2b Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Apr 2019 02:40:14 +0200 Subject: mod_mimicking: Update command to work with current code --- plugins/mod_mimicking.lua | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua index 0da68059..a8d6803e 100644 --- a/plugins/mod_mimicking.lua +++ b/plugins/mod_mimicking.lua @@ -11,10 +11,15 @@ assert(encodings.confusable, "This module requires that Prosody be built with IC local skeleton = encodings.confusable.skeleton; local usage = require "util.prosodyctl".show_usage; -local warn = require "util.prosodyctl".show_warning; -local users = require "usermanager".users; +local usermanager = require "core.usermanager"; +local storagemanager = require "core.storagemanager"; -local skeletons = module:open_store("skeletons"); +local skeletons +function module.load() + if module.host ~= "*" then + skeletons = module:open_store("skeletons"); + end +end module:hook("user-registered", function(user) skeletons:set(skeleton(user.username), { username = user.username }); @@ -42,13 +47,13 @@ function module.command(arg) if not host_session then return "No such host"; end - local provider = host_session.users; - if not(provider) or provider.name == "null" then - usermanager.initialize_host(host); - end + storagemanager.initialize_host(host); + usermanager.initialize_host(host); + + skeletons = storagemanager.open(host, "skeletons"); - for user in users(host) do - datamanager.store(skeleton(user), host, "skeletons", {username = user}); + for user in usermanager.users(host) do + skeletons:set(skeleton(user), { username = user }); end end -- cgit v1.2.3 From 21d4a58469853ebffca787cfc76e9b7d8b40d9b4 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Apr 2019 17:26:56 +0200 Subject: mod_mimicking: Use more intuitive term "mimicry index" for skeletons Fits better with the module name too. --- plugins/mod_mimicking.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua index a8d6803e..43d165d2 100644 --- a/plugins/mod_mimicking.lua +++ b/plugins/mod_mimicking.lua @@ -37,7 +37,7 @@ end); function module.command(arg) if (arg[1] ~= "bootstrap" or not arg[2]) then - usage("mod_mimicking bootstrap ", "Initialize skeleton database"); + usage("mod_mimicking bootstrap ", "Initialize username mimicry index"); return; end -- cgit v1.2.3 From 41c9fe2b76ee5bd90eeb948c7bb75bf94906b102 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 29 Apr 2019 17:27:08 +0200 Subject: mod_mimicking: Improve error handling --- plugins/mod_mimicking.lua | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua index 43d165d2..b586a70c 100644 --- a/plugins/mod_mimicking.lua +++ b/plugins/mod_mimicking.lua @@ -22,22 +22,34 @@ function module.load() end module:hook("user-registered", function(user) - skeletons:set(skeleton(user.username), { username = user.username }); + local skel = skeleton(user.username); + local ok, err = skeletons:set(skel, { username = user.username }); + if not ok then + module:log("error", "Unable to store mimicry data (%q => %q): %s", user.username, skel, err); + end end); module:hook("user-deleted", function(user) - skeletons:set(skeleton(user.username), nil); + local skel = skeleton(user.username); + local ok, err = skeletons:set(skel, nil); + if not ok and err then + module:log("error", "Unable to clear mimicry data (%q): %s", skel, err); + end end); module:hook("user-registering", function(user) - if skeletons:get(skeleton(user.username)) then + local existing, err = skeletons:get(skeleton(user.username)); + if existing then + module:log("debug", "Attempt to register username '%s' which could be confused with '%s'", user.username, existing.username); user.allowed = false; + elseif err then + module:log("error", "Unable to check if new username '%s' can be confused with any existing user: %s", err); end end); function module.command(arg) if (arg[1] ~= "bootstrap" or not arg[2]) then - usage("mod_mimicking bootstrap ", "Initialize username mimicry index"); + usage("mod_mimicking bootstrap ", "Initialize username mimicry database"); return; end @@ -53,7 +65,21 @@ function module.command(arg) skeletons = storagemanager.open(host, "skeletons"); + local count = 0; for user in usermanager.users(host) do - skeletons:set(skeleton(user), { username = user }); + local skel = skeleton(user); + local existing, err = skeletons:get(skel); + if existing and existing.username ~= user then + module:log("warn", "Existing usernames '%s' and '%s' are confusable", existing.username, user); + elseif err then + module:log("error", "Error checking for existing mimicry data (%q = %q): %s", user, skel, err); + end + local ok, err = skeletons:set(skel, { username = user }); + if ok then + count = count + 1; + elseif err then + module:log("error", "Unable to store mimicry data (%q => %q): %s", user, skel, err); + end end + module:log("info", "%d usernames indexed", count); end -- cgit v1.2.3 From 30ef8e7ac46587732ab469e9775229e8d78e1845 Mon Sep 17 00:00:00 2001 From: Arc Riley Date: Thu, 2 May 2019 16:33:14 -0700 Subject: mod_admin_telnet: include BOSH connections in c2s session commands (#998) --- plugins/mod_admin_telnet.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index c66630ca..5ee25771 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -591,6 +591,7 @@ end local function show_c2s(callback) local c2s = array.collect(values(module:shared"/*/c2s/sessions")); + c2s:append(values(module:shared"/*/bosh/sessions")); c2s:sort(function(a, b) if a.host == b.host then if a.username == b.username then -- cgit v1.2.3 From cdbb6348f6536cb270544c933f3e45ccf66ef585 Mon Sep 17 00:00:00 2001 From: Arc Riley Date: Thu, 2 May 2019 17:28:49 -0700 Subject: mod_admin_telnet: added "(bosh)" and "(websocket)" connection flags (#998) --- plugins/mod_admin_telnet.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 5ee25771..248d6b07 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -554,6 +554,12 @@ local function session_flags(session, line) if session.is_bidi then line[#line+1] = "(bidi)"; end + if session.bosh_version then + line[#line+1] = "(bosh)"; + end + if session.websocket_request then + line[#line+1] = "(websocket)"; + end return table.concat(line, " "); end -- cgit v1.2.3 From da0d3e296ca8fef7882cbf571039b153525a1185 Mon Sep 17 00:00:00 2001 From: Arc Riley Date: Thu, 2 May 2019 17:44:21 -0700 Subject: mod_admin_telnet: include BOSH connections in c2s:count (#998) --- plugins/mod_admin_telnet.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 248d6b07..0fbd3ff9 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -612,7 +612,9 @@ local function show_c2s(callback) end function def_env.c2s:count() - return true, "Total: ".. iterators.count(values(module:shared"/*/c2s/sessions")) .." clients"; + local c2s_count = iterators.count(values(module:shared"/*/c2s/sessions")) + local bosh_count = iterators.count(values(module:shared"/*/bosh/sessions")) + return true, "Total: ".. c2s_count + bosh_count .." clients"; end function def_env.c2s:show(match_jid, annotate) -- cgit v1.2.3 From 5bf3c84d07ff55408262ab5644752f6f0e28415e Mon Sep 17 00:00:00 2001 From: Arc Riley Date: Fri, 3 May 2019 04:10:31 -0700 Subject: mod_bosh: Added metrics for active/inactive sessions, new BOSH sessions, BOSH errors, and timeouts (finishes #998) --- plugins/mod_bosh.lua | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/plugins/mod_bosh.lua b/plugins/mod_bosh.lua index 082ed961..d4e980f2 100644 --- a/plugins/mod_bosh.lua +++ b/plugins/mod_bosh.lua @@ -55,6 +55,27 @@ local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat; -- All sessions, and sessions that have no requests open local sessions = module:shared("sessions"); +local measure_active = module:measure("active_sessions", "amount"); +local measure_inactive = module:measure("inactive_sessions", "amount"); +local report_bad_host = module:measure("bad_host", "rate"); +local report_bad_sid = module:measure("bad_sid", "rate"); +local report_new_sid = module:measure("new_sid", "rate"); +local report_timeout = module:measure("timeout", "rate"); + +module:hook("stats-update", function () + local active = 0; + local inactive = 0; + for _, session in pairs(sessions) do + if #session.requests > 0 then + active = active + 1; + else + inactive = inactive + 1; + end + end + measure_active(active); + measure_inactive(inactive); +end); + -- Used to respond to idle sessions (those with waiting requests) function on_destroy_request(request) log("debug", "Request destroyed: %s", tostring(request)); @@ -74,7 +95,7 @@ function on_destroy_request(request) if session.inactive_timer then session.inactive_timer:stop(); end - session.inactive_timer = module:add_timer(max_inactive, check_inactive, session, request.context, + session.inactive_timer = module:add_timer(max_inactive, session_timeout, session, request.context, "BOSH client silent for over "..max_inactive.." seconds"); (session.log or log)("debug", "BOSH session marked as inactive (for %ds)", max_inactive); end @@ -85,8 +106,9 @@ function on_destroy_request(request) end end -function check_inactive(now, session, context, reason) -- luacheck: ignore 212/now +function session_timeout(now, session, context, reason) -- luacheck: ignore 212/now if not session.destroyed then + report_timeout(); sessions[context.sid] = nil; sm_destroy_session(session, reason); end @@ -186,6 +208,7 @@ function handle_POST(event) return; end module:log("warn", "Unable to associate request with a session (incomplete request?)"); + report_bad_sid(); local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate", ["xmlns:stream"] = xmlns_streams, condition = "item-not-found" }); return tostring(close_reply) .. "\n"; @@ -253,6 +276,7 @@ function stream_callbacks.streamopened(context, attr) local wait = tonumber(attr.wait); if not to_host then log("debug", "BOSH client tried to connect to invalid host: %s", tostring(attr.to)); + report_bad_host(); local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate", ["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" }); response:send(tostring(close_reply)); @@ -290,6 +314,7 @@ function stream_callbacks.streamopened(context, attr) session.log("debug", "BOSH session created for request from %s", session.ip); log("info", "New BOSH session, assigned it sid '%s'", sid); + report_new_sid(); module:fire_event("bosh-session", { session = session, request = request }); @@ -344,6 +369,7 @@ function stream_callbacks.streamopened(context, attr) if not session then -- Unknown sid log("info", "Client tried to use sid '%s' which we don't know about", sid); + report_bad_sid(); response:send(tostring(st.stanza("body", { xmlns = xmlns_bosh, type = "terminate", condition = "item-not-found" }))); context.notopen = nil; return; -- cgit v1.2.3 From 690c6a4eb240ede63cf7a31007c25df29fa123f2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 3 May 2019 20:54:24 +0200 Subject: Fix various spelling mistakes [codespell] --- net/server_select.lua | 4 ++-- plugins/mod_blocklist.lua | 2 +- plugins/mod_saslauth.lua | 2 +- spec/util_throttle_spec.lua | 2 +- util/datamanager.lua | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/net/server_select.lua b/net/server_select.lua index 5d554655..e14c126e 100644 --- a/net/server_select.lua +++ b/net/server_select.lua @@ -124,7 +124,7 @@ local _maxsslhandshake _server = { } -- key = port, value = table; list of listening servers _readlist = { } -- array with sockets to read from -_sendlist = { } -- arrary with sockets to write to +_sendlist = { } -- array with sockets to write to _timerlist = { } -- array of timer functions _socketlist = { } -- key = socket, value = wrapped socket (handlers) _readtimes = { } -- key = handler, value = timestamp of last data reading @@ -150,7 +150,7 @@ _checkinterval = 30 -- interval in secs to check idle clients _sendtimeout = 60000 -- allowed send idle time in secs _readtimeout = 14 * 60 -- allowed read idle time in secs -local is_windows = package.config:sub(1,1) == "\\" -- check the directory separator, to detemine whether this is Windows +local is_windows = package.config:sub(1,1) == "\\" -- check the directory separator, to determine whether this is Windows _maxfd = (is_windows and math.huge) or luasocket._SETSIZE or 1024 -- max fd number, limit to 1024 by default to prevent glibc buffer overflow, but not on Windows _maxselectlen = luasocket._SETSIZE or 1024 -- But this still applies on Windows diff --git a/plugins/mod_blocklist.lua b/plugins/mod_blocklist.lua index 8aca7332..2193a093 100644 --- a/plugins/mod_blocklist.lua +++ b/plugins/mod_blocklist.lua @@ -159,7 +159,7 @@ local function edit_blocklist(event) local blocklist = cache[username] or get_blocklist(username); local new_blocklist = { - -- We set the [false] key to someting as a signal not to migrate privacy lists + -- We set the [false] key to something as a signal not to migrate privacy lists [false] = blocklist[false] or { created = now; }; }; if type(blocklist[false]) == "table" then diff --git a/plugins/mod_saslauth.lua b/plugins/mod_saslauth.lua index ba30b9e6..3145cf9b 100644 --- a/plugins/mod_saslauth.lua +++ b/plugins/mod_saslauth.lua @@ -248,7 +248,7 @@ module:hook("stream-features", function(event) local sasl_handler = usermanager_get_sasl_handler(module.host, origin) origin.sasl_handler = sasl_handler; if origin.encrypted then - -- check wether LuaSec has the nifty binding to the function needed for tls-unique + -- check whether LuaSec has the nifty binding to the function needed for tls-unique -- FIXME: would be nice to have this check only once and not for every socket if sasl_handler.add_cb_handler then local socket = origin.conn:socket(); diff --git a/spec/util_throttle_spec.lua b/spec/util_throttle_spec.lua index 75daf1b9..985afae8 100644 --- a/spec/util_throttle_spec.lua +++ b/spec/util_throttle_spec.lua @@ -88,7 +88,7 @@ describe("util.throttle", function() later(0.1); a:update(); end - assert(math.abs(a.balance - 1) < 0.0001); -- incremental updates cause rouding errors + assert(math.abs(a.balance - 1) < 0.0001); -- incremental updates cause rounding errors end); end); diff --git a/util/datamanager.lua b/util/datamanager.lua index cf96887b..b52c77fa 100644 --- a/util/datamanager.lua +++ b/util/datamanager.lua @@ -24,7 +24,7 @@ local t_concat = table.concat; local envloadfile = require"util.envload".envloadfile; local serialize = require "util.serialization".serialize; local lfs = require "lfs"; --- Extract directory seperator from package.config (an undocumented string that comes with lua) +-- Extract directory separator from package.config (an undocumented string that comes with lua) local path_separator = assert ( package.config:match ( "^([^\n]+)" ) , "package.config not in standard form" ) local prosody = prosody; -- cgit v1.2.3 From b3741ec9ef19f4cc5dfb0bbc5832d632339dadcc Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 4 May 2019 04:48:40 +0200 Subject: net.http.files: Bump cache hits so they stay cached It's not an LRU cache unless this is done. --- net/http/files.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/net/http/files.lua b/net/http/files.lua index 0b898dc6..090b15c8 100644 --- a/net/http/files.lua +++ b/net/http/files.lua @@ -94,6 +94,7 @@ local function serve(opts) if data and data.etag == etag then response_headers.content_type = data.content_type; data = data.data; + cache:get(orig_path, data); elseif attr.mode == "directory" and path then if full_path:sub(-1) ~= "/" then local dir_path = { is_absolute = true, is_directory = true }; -- cgit v1.2.3 From 200664ab8a16a7f5a4d4a2de878c9f94cbeb4a8f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 08:12:16 +0200 Subject: mod_storage_internal,memory: Only return total count if requested --- plugins/mod_storage_internal.lua | 17 +++++++++++------ plugins/mod_storage_memory.lua | 17 ++++++++++++----- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index aa5c3c8a..f921747e 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -124,11 +124,14 @@ function archive:find(username, query) if not items then if err then return items, err; - else - return function () end, 0; + elseif query then + if query.total then + return function () end, 0; + end end + return function () end; end - local count = #items; + local count = nil; local i = 0; if query then items = array(items); @@ -152,11 +155,13 @@ function archive:find(username, query) return item.when <= query["end"]; end); end - count = #items; + if query.total then + count = #items; + end if query.reverse then items:reverse(); if query.before then - for j = 1, count do + for j = 1, #items do if (items[j].key or tostring(j)) == query.before then i = j; break; @@ -164,7 +169,7 @@ function archive:find(username, query) end end elseif query.after then - for j = 1, count do + for j = 1, #items do if (items[j].key or tostring(j)) == query.after then i = j; break; diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua index dde2d571..4655cb3a 100644 --- a/plugins/mod_storage_memory.lua +++ b/plugins/mod_storage_memory.lua @@ -90,9 +90,14 @@ end function archive_store:find(username, query) local items = self.store[username or NULL]; if not items then - return function () end, 0; + if query then + if query.total then + return function () end, 0; + end + end + return function () end; end - local count = #items; + local count = nil; local i = 0; if query then items = array():append(items); @@ -116,11 +121,13 @@ function archive_store:find(username, query) return item.when <= query["end"]; end); end - count = #items; + if query.total then + count = #items; + end if query.reverse then items:reverse(); if query.before then - for j = 1, count do + for j = 1, #items do if (items[j].key or tostring(j)) == query.before then i = j; break; @@ -128,7 +135,7 @@ function archive_store:find(username, query) end end elseif query.after then - for j = 1, count do + for j = 1, #items do if (items[j].key or tostring(j)) == query.after then i = j; break; -- cgit v1.2.3 From c39b08a25d18e6ca268a3c75f52992255d7b3e50 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 14:52:34 +0200 Subject: mod_muc_mam: Handle archive quotas Same as in mod_mam --- plugins/mod_muc_mam.lua | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/plugins/mod_muc_mam.lua b/plugins/mod_muc_mam.lua index 35eabd3b..fffe23e7 100644 --- a/plugins/mod_muc_mam.lua +++ b/plugins/mod_muc_mam.lua @@ -33,6 +33,9 @@ local m_min = math.min; local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date"); local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50); +local cleanup_after = module:get_option_string("muc_log_expires_after", "1w"); +local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60); + local default_history_length = 20; local max_history_length = module:get_option_number("max_history_messages", math.huge); @@ -50,6 +53,8 @@ local log_by_default = module:get_option_boolean("muc_log_by_default", true); local archive_store = "muc_log"; local archive = module:open_store(archive_store, "archive"); +local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000); + if archive.name == "null" or not archive.find then if not archive.find then module:log("error", "Attempt to open archive storage returned a driver without archive API support"); @@ -64,12 +69,15 @@ end local function archiving_enabled(room) if log_all_rooms then + module:log("debug", "Archiving all rooms"); return true; end local enabled = room._data.archiving; if enabled == nil then + module:log("debug", "Default is %s (for %s)", log_by_default, room.jid); return log_by_default; end + module:log("debug", "Logging in room %s is %s", room.jid, enabled); return enabled; end @@ -357,7 +365,29 @@ local function save_to_history(self, stanza) end -- And stash it - local id = archive:append(room_node, nil, stored_stanza, time_now(), with); + local time = time_now(); + local id, err = archive:append(room_node, nil, stored_stanza, time, with); + + if not id and err == "quota-limit" then + if type(cleanup_after) == "number" then + module:log("debug", "Room '%s' over quota, cleaning archive", room_node); + local cleaned = archive:delete(room_node, { + ["end"] = (os.time() - cleanup_after); + }); + if cleaned then + id, err = archive:append(room_node, nil, stored_stanza, time, with); + end + end + if not id and (archive.caps and archive.caps.truncate) then + module:log("debug", "User '%s' over quota, truncating archive", room_node); + local truncated = archive:delete(room_node, { + truncate = archive_item_limit - 1; + }); + if truncated then + id, err = archive:append(room_node, nil, stored_stanza, time, with); + end + end + end if id then schedule_cleanup(room_node); @@ -399,9 +429,6 @@ end); -- Cleanup -local cleanup_after = module:get_option_string("muc_log_expires_after", "1w"); -local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60); - if cleanup_after ~= "never" then local cleanup_storage = module:open_store("muc_log_cleanup"); local cleanup_map = module:open_store("muc_log_cleanup", "map"); -- cgit v1.2.3 From 017e7f74389c0e20a1baaae682fd85b825975592 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 16:07:16 +0200 Subject: mod_storage_internal: Add support for iterating over users in archive stores May help with writing a better migrator --- plugins/mod_storage_internal.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index f921747e..96780b33 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -211,6 +211,10 @@ function archive:summary(username, query) return summary; end +function archive:users() + return datamanager.users(host, self.store, "list"); +end + function archive:delete(username, query) local cache_key = jid_join(username, host, self.store); if not query or next(query) == nil then -- cgit v1.2.3 From 696da1f571e723216a1e6b74d793eed4fe98dbd8 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 16:26:01 +0200 Subject: mod_storage_sql: Add support for iterating over users in archive stores --- plugins/mod_storage_sql.lua | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index ffe48ab8..596687ae 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -509,6 +509,19 @@ function archive_store:delete(username, query) return ok and stmt:affected(), stmt; end +function archive_store:users() + local ok, result = engine:transaction(function() + local select_sql = [[ + SELECT DISTINCT "user" + FROM "prosodyarchive" + WHERE "host"=? AND "store"=?; + ]]; + return engine:select(select_sql, host, self.store); + end); + if not ok then error(result); end + return iterator(result); +end + local stores = { keyval = keyval_store; map = map_store; -- cgit v1.2.3 From 7112f319c35e19bbc75579c102f2bc37b4d911bb Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 21:32:34 +0200 Subject: migrator: Rewrite to use storage modules This allows migrating to and from any storage module that supports the right methods. Based on experimental mod_migrate work. --- CHANGES | 1 + tools/migration/Makefile | 7 +- tools/migration/migrator.cfg.lua | 34 +++++- tools/migration/migrator/mtools.lua | 58 --------- tools/migration/migrator/prosody_files.lua | 144 ---------------------- tools/migration/migrator/prosody_sql.lua | 190 ----------------------------- tools/migration/prosody-migrator.lua | 148 ++++++++++++++++------ 7 files changed, 143 insertions(+), 439 deletions(-) delete mode 100644 tools/migration/migrator/mtools.lua delete mode 100644 tools/migration/migrator/prosody_files.lua delete mode 100644 tools/migration/migrator/prosody_sql.lua diff --git a/CHANGES b/CHANGES index 8c65f4ec..8b6675a5 100644 --- a/CHANGES +++ b/CHANGES @@ -8,6 +8,7 @@ TRUNK - mod\_limits: Exempted JIDs - Archive quotas - mod\_mimicking +- Rewritten migrator 0.11.0 ====== diff --git a/tools/migration/Makefile b/tools/migration/Makefile index 713831d2..dc14e0da 100644 --- a/tools/migration/Makefile +++ b/tools/migration/Makefile @@ -12,16 +12,13 @@ INSTALLEDCONFIG = $(SYSCONFDIR) INSTALLEDMODULES = $(LIBDIR)/prosody/modules INSTALLEDDATA = $(DATADIR) -SOURCE_FILES = migrator/*.lua - -all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua $(SOURCE_FILES) +all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua install: prosody-migrator.install migrator.cfg.lua.install - install -d $(BIN) $(CONFIG) $(SOURCE) $(SOURCE)/migrator + install -d $(BIN) $(CONFIG) $(SOURCE) install -d $(MAN)/man1 install -d $(SOURCE)/migrator install -m755 ./prosody-migrator.install $(BIN)/prosody-migrator - install -m644 $(SOURCE_FILES) $(SOURCE)/migrator test -e $(CONFIG)/migrator.cfg.lua || install -m644 migrator.cfg.lua.install $(CONFIG)/migrator.cfg.lua clean: diff --git a/tools/migration/migrator.cfg.lua b/tools/migration/migrator.cfg.lua index fa37f2a3..c26fa869 100644 --- a/tools/migration/migrator.cfg.lua +++ b/tools/migration/migrator.cfg.lua @@ -1,12 +1,38 @@ local data_path = "../../data"; +local vhost = { + "accounts", + "account_details", + "roster", + "vcard", + "private", + "blocklist", + "privacy", + "archive-archive", + "offline-archive", + "pubsub_nodes", + -- "pubsub_*-archive", + "pep", + -- "pep_*-archive", +} +local muc = { + "persistent", + "config", + "state", + "muc_log-archive", +}; + input { - type = "prosody_files"; + hosts = { + ["example.com"] = vhost; + ["conference.example.com"] = muc; + }; + type = "internal"; path = data_path; } output { - type = "prosody_sql"; + type = "sql"; driver = "SQLite3"; database = data_path.."/prosody.sqlite"; } @@ -14,11 +40,11 @@ output { --[[ input { - type = "prosody_files"; + type = "internal"; path = data_path; } output { - type = "prosody_sql"; + type = "sql"; driver = "SQLite3"; database = data_path.."/prosody.sqlite"; } diff --git a/tools/migration/migrator/mtools.lua b/tools/migration/migrator/mtools.lua deleted file mode 100644 index cfbfcce8..00000000 --- a/tools/migration/migrator/mtools.lua +++ /dev/null @@ -1,58 +0,0 @@ - - -local print = print; -local t_insert = table.insert; -local t_sort = table.sort; - - -local function sorted(params) - - local reader = params.reader; -- iterator to get items from - local sorter = params.sorter; -- sorting function - local filter = params.filter; -- filter function - - local cache = {}; - for item in reader do - if filter then item = filter(item); end - if item then t_insert(cache, item); end - end - if sorter then - t_sort(cache, sorter); - end - local i = 0; - return function() - i = i + 1; - return cache[i]; - end; - -end - -local function merged(reader, merger) - - local item1 = reader(); - local merged = { item1 }; - return function() - while true do - if not item1 then return nil; end - local item2 = reader(); - if not item2 then item1 = nil; return merged; end - if merger(item1, item2) then - --print("merged") - item1 = item2; - t_insert(merged, item1); - else - --print("unmerged", merged) - item1 = item2; - local tmp = merged; - merged = { item1 }; - return tmp; - end - end - end; - -end - -return { - sorted = sorted; - merged = merged; -} diff --git a/tools/migration/migrator/prosody_files.lua b/tools/migration/migrator/prosody_files.lua deleted file mode 100644 index 4de09273..00000000 --- a/tools/migration/migrator/prosody_files.lua +++ /dev/null @@ -1,144 +0,0 @@ - -local print = print; -local assert = assert; -local setmetatable = setmetatable; -local tonumber = tonumber; -local char = string.char; -local coroutine = coroutine; -local lfs = require "lfs"; -local loadfile = loadfile; -local pcall = pcall; -local mtools = require "migrator.mtools"; -local next = next; -local pairs = pairs; -local json = require "util.json"; -local os_getenv = os.getenv; -local error = error; - -prosody = {}; -local dm = require "util.datamanager" - - -local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end -local function is_file(path) return lfs.attributes(path, "mode") == "file"; end -local function clean_path(path) - return path:gsub("\\", "/"):gsub("//+", "/"):gsub("^~", os_getenv("HOME") or "~"); -end -local encode, decode; do - local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end }); - decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end - encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end -end -local function decode_dir(x) - if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then - return decode(x); - end -end -local function decode_file(x) - if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then - return decode(x:gsub("%.dat$", "")); - end -end -local function prosody_dir(path, ondir, onfile, ...) - for x in lfs.dir(path) do - local xpath = path.."/"..x; - if decode_dir(x) and is_dir(xpath) then - ondir(xpath, x, ...); - elseif decode_file(x) and is_file(xpath) then - onfile(xpath, x, ...); - end - end -end - -local function handle_root_file(path, name) - --print("root file: ", decode_file(name)) - coroutine.yield { user = nil, host = nil, store = decode_file(name) }; -end -local function handle_host_file(path, name, host) - --print("host file: ", decode_dir(host).."/"..decode_file(name)) - coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) }; -end -local function handle_store_file(path, name, store, host) - --print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store)) - coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) }; -end -local function handle_host_store(path, name, host) - prosody_dir(path, function() end, handle_store_file, name, host); -end -local function handle_host_dir(path, name) - prosody_dir(path, handle_host_store, handle_host_file, name); -end -local function handle_root_dir(path) - prosody_dir(path, handle_host_dir, handle_root_file); -end - -local function decode_user(item) - local userdata = { - user = item[1].user; - host = item[1].host; - stores = {}; - }; - for i=1,#item do -- loop over stores - local result = {}; - local store = item[i]; - userdata.stores[store.store] = store.data; - store.user = nil; store.host = nil; store.store = nil; - end - return userdata; -end - -local function reader(input) - local path = clean_path(assert(input.path, "no input.path specified")); - assert(is_dir(path), "input.path is not a directory"); - local iter = coroutine.wrap(function()handle_root_dir(path);end); - -- get per-user stores, sorted - local iter = mtools.sorted { - reader = function() - local x = iter(); - while x do - dm.set_data_path(path); - local err; - x.data, err = dm.load(x.user, x.host, x.store); - if x.data == nil and err then - local p = dm.getpath(x.user, x.host, x.store); - print(("Error loading data at path %s for %s@%s (%s store): %s") - :format(p, x.user or "", x.host or "", x.store or "", err or "")); - else - return x; - end - x = iter(); - end - end; - sorter = function(a, b) - local a_host, a_user, a_store = a.host or "", a.user or "", a.store or ""; - local b_host, b_user, b_store = b.host or "", b.user or "", b.store or ""; - return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store); - end; - }; - -- merge stores to get users - iter = mtools.merged(iter, function(a, b) - return (a.host == b.host and a.user == b.user); - end); - - return function() - local x = iter(); - return x and decode_user(x); - end -end - -local function writer(output) - local path = clean_path(assert(output.path, "no output.path specified")); - assert(is_dir(path), "output.path is not a directory"); - return function(item) - if not item then return; end -- end of input - dm.set_data_path(path); - for store, data in pairs(item.stores) do - assert(dm.store(item.user, item.host, store, data)); - end - end -end - -return { - reader = reader; - writer = writer; -} diff --git a/tools/migration/migrator/prosody_sql.lua b/tools/migration/migrator/prosody_sql.lua deleted file mode 100644 index 6df2b025..00000000 --- a/tools/migration/migrator/prosody_sql.lua +++ /dev/null @@ -1,190 +0,0 @@ - -local assert = assert; -local have_DBI = pcall(require,"DBI"); -local print = print; -local type = type; -local next = next; -local pairs = pairs; -local t_sort = table.sort; -local json = require "util.json"; -local mtools = require "migrator.mtools"; -local tostring = tostring; -local tonumber = tonumber; - -if not have_DBI then - error("LuaDBI (required for SQL support) was not found, please see https://prosody.im/doc/depends#luadbi", 0); -end - -local sql = require "util.sql"; - -local function create_table(engine, name) -- luacheck: ignore 431/engine - local Table, Column, Index = sql.Table, sql.Column, sql.Index; - - local ProsodyTable = Table { - name= name or "prosody"; - Column { name="host", type="TEXT", nullable=false }; - Column { name="user", type="TEXT", nullable=false }; - Column { name="store", type="TEXT", nullable=false }; - Column { name="key", type="TEXT", nullable=false }; - Column { name="type", type="TEXT", nullable=false }; - Column { name="value", type="MEDIUMTEXT", nullable=false }; - Index { name="prosody_index", "host", "user", "store", "key" }; - }; - engine:transaction(function() - ProsodyTable:create(engine); - end); - -end - -local function serialize(value) - local t = type(value); - if t == "string" or t == "boolean" or t == "number" then - return t, tostring(value); - elseif t == "table" then - local value,err = json.encode(value); - if value then return "json", value; end - return nil, err; - end - return nil, "Unhandled value type: "..t; -end -local function deserialize(t, value) - if t == "string" then return value; - elseif t == "boolean" then - if value == "true" then return true; - elseif value == "false" then return false; end - elseif t == "number" then return tonumber(value); - elseif t == "json" then - return json.decode(value); - end -end - -local function decode_user(item) - local userdata = { - user = item[1][1].user; - host = item[1][1].host; - stores = {}; - }; - for i=1,#item do -- loop over stores - local result = {}; - local store = item[i]; - for i=1,#store do -- loop over store data - local row = store[i]; - local k = row.key; - local v = deserialize(row.type, row.value); - if k and v then - if k ~= "" then result[k] = v; elseif type(v) == "table" then - for a,b in pairs(v) do - result[a] = b; - end - end - end - userdata.stores[store[1].store] = result; - end - end - return userdata; -end - -local function needs_upgrade(engine, params) - if params.driver == "MySQL" then - local success = engine:transaction(function() - local result = engine:execute("SHOW COLUMNS FROM prosody WHERE Field='value' and Type='text'"); - assert(result:rowcount() == 0); - - -- COMPAT w/pre-0.10: Upgrade table to UTF-8 if not already - local check_encoding_query = [[ - SELECT "COLUMN_NAME","COLUMN_TYPE","TABLE_NAME" - FROM "information_schema"."columns" - WHERE "TABLE_NAME" LIKE 'prosody%%' AND ( "CHARACTER_SET_NAME"!='%s' OR "COLLATION_NAME"!='%s_bin' ); - ]]; - check_encoding_query = check_encoding_query:format(engine.charset, engine.charset); - local result = engine:execute(check_encoding_query); - assert(result:rowcount() == 0) - end); - if not success then - -- Upgrade required - return true; - end - end - return false; -end - -local function reader(input) - local engine = assert(sql:create_engine(input, function (engine) -- luacheck: ignore 431/engine - if needs_upgrade(engine, input) then - error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade"); - end - end)); - local keys = {"host", "user", "store", "key", "type", "value"}; - assert(engine:connect()); - local f,s,val = assert(engine:select("SELECT \"host\", \"user\", \"store\", \"key\", \"type\", \"value\" FROM \"prosody\";")); - -- get SQL rows, sorted - local iter = mtools.sorted { - reader = function() val = f(s, val); return val; end; - filter = function(x) - for i=1,#keys do - x[ keys[i] ] = x[i]; - end - if x.host == "" then x.host = nil; end - if x.user == "" then x.user = nil; end - if x.store == "" then x.store = nil; end - return x; - end; - sorter = function(a, b) - local a_host, a_user, a_store = a.host or "", a.user or "", a.store or ""; - local b_host, b_user, b_store = b.host or "", b.user or "", b.store or ""; - return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store); - end; - }; - -- merge rows to get stores - iter = mtools.merged(iter, function(a, b) - return (a.host == b.host and a.user == b.user and a.store == b.store); - end); - -- merge stores to get users - iter = mtools.merged(iter, function(a, b) - return (a[1].host == b[1].host and a[1].user == b[1].user); - end); - return function() - local x = iter(); - return x and decode_user(x); - end; -end - -local function writer(output, iter) - local engine = assert(sql:create_engine(output, function (engine) -- luacheck: ignore 431/engine - if needs_upgrade(engine, output) then - error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade"); - end - create_table(engine); - end)); - assert(engine:connect()); - assert(engine:delete("DELETE FROM \"prosody\"")); - local insert_sql = "INSERT INTO \"prosody\" (\"host\",\"user\",\"store\",\"key\",\"type\",\"value\") VALUES (?,?,?,?,?,?)"; - - return function(item) - if not item then assert(engine.conn:commit()) return end -- end of input - local host = item.host or ""; - local user = item.user or ""; - for store, data in pairs(item.stores) do - -- TODO transactions - local extradata = {}; - for key, value in pairs(data) do - if type(key) == "string" and key ~= "" then - local t, value = assert(serialize(value)); - local ok, err = assert(engine:insert(insert_sql, host, user, store, key, t, value)); - else - extradata[key] = value; - end - end - if next(extradata) ~= nil then - local t, extradata = assert(serialize(extradata)); - local ok, err = assert(engine:insert(insert_sql, host, user, store, "", t, extradata)); - end - end - end; -end - - -return { - reader = reader; - writer = writer; -} diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua index 1219d891..3cc10ecf 100644 --- a/tools/migration/prosody-migrator.lua +++ b/tools/migration/prosody-migrator.lua @@ -1,19 +1,43 @@ #!/usr/bin/env lua -CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR"); -CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR"); +CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR"); +CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR"); +CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR"); +CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR"); --- Substitute ~ with path to home directory in paths -if CFG_CONFIGDIR then - CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME")); +-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- + +local function is_relative(path) + local path_sep = package.config:sub(1,1); + return ((path_sep == "/" and path:sub(1,1) ~= "/") + or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\"))) end +-- Tell Lua where to find our libraries if CFG_SOURCEDIR then - CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME")); + local function filter_relative_paths(path) + if is_relative(path) then return ""; end + end + local function sanitise_paths(paths) + return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";")); + end + package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path); + package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath); +end + +-- Substitute ~ with path to home directory in data path +if CFG_DATADIR then + if os.getenv("HOME") then + CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME")); + end end local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua"; +local startup = require "util.startup"; +startup.prosodyctl(); +-- TODO startup.migrator ? + -- Command-line parsing local options = {}; local i = 1; @@ -29,13 +53,6 @@ while arg[i] do end end -if CFG_SOURCEDIR then - package.path = CFG_SOURCEDIR.."/?.lua;"..package.path; - package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath; -else - package.path = "../../?.lua;"..package.path - package.cpath = "../../?.so;"..package.cpath -end local envloadfile = require "util.envload".envloadfile; @@ -69,24 +86,14 @@ if not config[to_store] then print("Error: Output store '"..to_store.."' not found in the config file."); end -function load_store_handler(name) - local store_type = config[name].type; - if not store_type then - print("Error: "..name.." store type not specified in the config file"); - return false; - else - local ok, err = pcall(require, "migrator."..store_type); - if not ok then - print(("Error: Failed to initialize '%s' store:\n\t%s") - :format(name, err)); - return false; - end +for store, conf in pairs(config) do -- COMPAT + if conf.type == "prosody_files" then + conf.type = "internal"; + elseif conf.type == "prosody_sql" then + conf.type = "sql"; end - return true; end -have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output")); - if have_err then print(""); print("Usage: "..arg[0].." FROM_STORE TO_STORE"); @@ -101,17 +108,82 @@ if have_err then os.exit(1); end -local itype = config[from_store].type; -local otype = config[to_store].type; -local reader = require("migrator."..itype).reader(config[from_store]); -local writer = require("migrator."..otype).writer(config[to_store]); +local async = require "util.async"; +local server = require "net.server"; +local watchers = { + error = function (_, err) + error(err); + end; + waiting = function () + server.loop(); + end; +}; + +local cm = require "core.configmanager"; +local hm = require "core.hostmanager"; +local sm = require "core.storagemanager"; +local um = require "core.usermanager"; + +local function users(store, host) + if store.users then + return store:users(); + else + return um.users(host); + end +end -local json = require "util.json"; +local function prepare_config(host, conf) + if conf.type == "internal" then + sm.olddm.set_data_path(conf.path or prosody.paths.data); + elseif conf.type == "sql" then + cm.set(host, "sql", conf); + end +end -io.stderr:write("Migrating...\n"); -for x in reader do - --print(json.encode(x)) - writer(x); +local function get_driver(host, conf) + prepare_config(host, conf); + return assert(sm.load_driver(host, conf.type)); end -writer(nil); -- close + +local migration_runner = async.runner(function (job) + for host, stores in pairs(job.input.hosts) do + prosody.hosts[host] = startup.make_host(host); + sm.initialize_host(host); + um.initialize_host(host); + + local input_driver = get_driver(host, job.input); + + local output_driver = get_driver(host, job.output); + + for _, store in ipairs(stores) do + local p, typ = store:match("()%-(%w+)$"); + if typ then store = store:sub(1, p-1); else typ = "keyval"; end + log("info", "Migrating host %s store %s (%s)", host, store, typ); + + local origin = assert(input_driver:open(store, typ)); + local destination = assert(output_driver:open(store, typ)); + + if typ == "keyval" then -- host data + local data, err = origin:get(nil); + assert(not err, err); + assert(destination:set(nil, data)); + end + + for user in users(origin, host) do + if typ == "keyval" then + local data, err = origin:get(user); + assert(not err, err); + assert(destination:set(user, data)); + else + error("Don't know how to migrate data of type '"..typ.."'."); + end + end + end + end +end, watchers); + +io.stderr:write("Migrating...\n"); + +migration_runner:run({ input = config[from_store], output = config[to_store] }); + io.stderr:write("Done!\n"); -- cgit v1.2.3 From 015d0616141b8ff16abf47b6e45d2c486f102e07 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 21:31:15 +0200 Subject: migrator: Add support for archives (fixes #651) --- tools/migration/prosody-migrator.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua index 3cc10ecf..5c4800ad 100644 --- a/tools/migration/prosody-migrator.lua +++ b/tools/migration/prosody-migrator.lua @@ -174,6 +174,12 @@ local migration_runner = async.runner(function (job) local data, err = origin:get(user); assert(not err, err); assert(destination:set(user, data)); + elseif typ == "archive" then + local iter, err = origin:find(user); + assert(iter, err); + for id, item, when, with in iter do + assert(destination:append(user, id, item, when, with)); + end else error("Don't know how to migrate data of type '"..typ.."'."); end -- cgit v1.2.3 From b61b308c569dda7ca5e214466503e9cc7ea4a7e7 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 13 May 2019 10:03:46 +0100 Subject: util.hashring: Implementation of hashring data structure --- util/hashring.lua | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 util/hashring.lua diff --git a/util/hashring.lua b/util/hashring.lua new file mode 100644 index 00000000..322bc005 --- /dev/null +++ b/util/hashring.lua @@ -0,0 +1,88 @@ +local function generate_ring(nodes, num_replicas, hash) + local new_ring = {}; + for _, node_name in ipairs(nodes) do + for replica = 1, num_replicas do + local replica_hash = hash(node_name..":"..replica); + new_ring[replica_hash] = node_name; + table.insert(new_ring, replica_hash); + end + end + table.sort(new_ring); + return new_ring; +end + +local hashring_methods = {}; +local hashring_mt = { + __index = function (self, k) + -- Automatically build self.ring if it's missing + if k == "ring" then + local ring = generate_ring(self.nodes, self.num_replicas, self.hash); + rawset(self, "ring", ring); + return ring; + end + return rawget(hashring_methods, k); + end +}; + +local function new(num_replicas, hash_function) + return setmetatable({ nodes = {}, num_replicas = num_replicas, hash = hash_function }, hashring_mt); +end; + +function hashring_methods:add_node(name) + self.ring = nil; + self.nodes[name] = true; + table.insert(self.nodes, name); + return true; +end + +function hashring_methods:add_nodes(nodes) + self.ring = nil; + for _, node_name in ipairs(nodes) do + if not self.nodes[node_name] then + self.nodes[node_name] = true; + table.insert(self.nodes, node_name); + end + end + return true; +end + +function hashring_methods:remove_node(node_name) + self.ring = nil; + if self.nodes[node_name] then + for i, stored_node_name in ipairs(self.nodes) do + if node_name == stored_node_name then + self.nodes[node_name] = nil; + table.remove(self.nodes, i); + return true; + end + end + end + return false; +end + +function hashring_methods:remove_nodes(nodes) + self.ring = nil; + for _, node_name in ipairs(nodes) do + self:remove_node(node_name); + end +end + +function hashring_methods:clone() + local clone_hashring = new(self.num_replicas, self.hash); + clone_hashring:add_nodes(self.nodes); + return clone_hashring; +end + +function hashring_methods:get_node(key) + local key_hash = self.hash(key); + for _, replica_hash in ipairs(self.ring) do + if key_hash < replica_hash then + return self.ring[replica_hash]; + end + end + return self.ring[self.ring[1]]; +end + +return { + new = new; +} -- cgit v1.2.3 From e81589a45be00cb02e8577d3eff16a358a3f9080 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 13 May 2019 11:30:45 +0200 Subject: util.encodings: Declare absence of arguments [-Wstrict-prototypes] --- util-src/encodings.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util-src/encodings.c b/util-src/encodings.c index 3b7f322d..5e7032cf 100644 --- a/util-src/encodings.c +++ b/util-src/encodings.c @@ -330,7 +330,7 @@ USpoofChecker *icu_spoofcheck; #endif /* initialize global ICU stringprep profiles */ -void init_icu() { +void init_icu(void) { UErrorCode err = U_ZERO_ERROR; utrace_setLevel(UTRACE_VERBOSE); icu_nameprep = usprep_openByType(USPREP_RFC3491_NAMEPREP, &err); -- cgit v1.2.3 From fa03622e56e1520f4e0f87558d5e375f581fcad0 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 13 May 2019 10:36:03 +0100 Subject: util.hashring: Add tests --- spec/util_hashring_spec.lua | 85 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 spec/util_hashring_spec.lua diff --git a/spec/util_hashring_spec.lua b/spec/util_hashring_spec.lua new file mode 100644 index 00000000..d8801774 --- /dev/null +++ b/spec/util_hashring_spec.lua @@ -0,0 +1,85 @@ +local hashring = require "util.hashring"; + +describe("util.hashring", function () + + local sha256 = require "util.hashes".sha256; + + local ring = hashring.new(128, sha256); + + it("should fail to get a node that does not exist", function () + assert.is_nil(ring:get_node("foo")) + end); + + it("should support adding nodes", function () + ring:add_node("node1"); + end); + + it("should return a single node for all keys if only one node exists", function () + for i = 1, 100 do + assert.is_equal("node1", ring:get_node(tostring(i))) + end + end); + + it("should support adding a second node", function () + ring:add_node("node2"); + end); + + it("should fail to remove a non-existent node", function () + assert.is_falsy(ring:remove_node("node3")); + end); + + it("should succeed to remove a node", function () + assert.is_truthy(ring:remove_node("node1")); + end); + + it("should return the only node for all keys", function () + for i = 1, 100 do + assert.is_equal("node2", ring:get_node(tostring(i))) + end + end); + + it("should support adding multiple nodes", function () + ring:add_nodes({ "node1", "node3", "node4", "node5" }); + end); + + it("should disrupt a minimal number of keys on node removal", function () + local orig_ring = ring:clone(); + local node_tallies = {}; + + local n = 1000; + + for i = 1, n do + local key = tostring(i); + local node = ring:get_node(key); + node_tallies[node] = (node_tallies[node] or 0) + 1; + end + + --[[ + for node, key_count in pairs(node_tallies) do + print(node, key_count, ("%.2f%%"):format((key_count/n)*100)); + end + ]] + + ring:remove_node("node5"); + + local disrupted_keys = 0; + for i = 1, n do + local key = tostring(i); + if orig_ring:get_node(key) ~= ring:get_node(key) then + disrupted_keys = disrupted_keys + 1; + end + end + assert.is_equal(node_tallies["node5"], disrupted_keys); + end); + + it("should support removing multiple nodes", function () + ring:remove_nodes({"node2", "node3", "node4", "node5"}); + end); + + it("should return a single node for all keys if only one node remains", function () + for i = 1, 100 do + assert.is_equal("node1", ring:get_node(tostring(i))) + end + end); + +end); -- cgit v1.2.3 From a49e94f20f276be733be3eab4f0ae8a57d28b1e2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 13 May 2019 11:52:16 +0200 Subject: configure: Handle ostype preset after argument processing --- configure | 138 ++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/configure b/configure index c8ad1115..33d86909 100755 --- a/configure +++ b/configure @@ -153,74 +153,8 @@ do SYSCONFDIR_SET=yes ;; --ostype) - # TODO make this a switch? OSPRESET="$value" - if [ "$OSPRESET" = "debian" ]; then - if [ "$LUA_SUFFIX_SET" != "yes" ]; then - LUA_SUFFIX="5.1"; - LUA_SUFFIX_SET=yes - fi - if [ "$RUNWITH_SET" != "yes" ]; then - RUNWITH="lua$LUA_SUFFIX"; - RUNWITH_SET=yes - fi - LUA_INCDIR="/usr/include/lua$LUA_SUFFIX" - LUA_INCDIR_SET=yes - CFLAGS="$CFLAGS -ggdb" - fi - if [ "$OSPRESET" = "macosx" ]; then - LUA_INCDIR=/usr/local/include; - LUA_INCDIR_SET=yes - LUA_LIBDIR=/usr/local/lib - LUA_LIBDIR_SET=yes - CFLAGS="$CFLAGS -mmacosx-version-min=10.3" - LDFLAGS="-bundle -undefined dynamic_lookup" - fi - if [ "$OSPRESET" = "linux" ]; then - LUA_INCDIR=/usr/local/include; - LUA_INCDIR_SET=yes - LUA_LIBDIR=/usr/local/lib - LUA_LIBDIR_SET=yes - CFLAGS="$CFLAGS -ggdb" - fi - if [ "$OSPRESET" = "freebsd" ] || [ "$OSPRESET" = "openbsd" ]; then - LUA_INCDIR="/usr/local/include/lua51" - LUA_INCDIR_SET=yes - CFLAGS="-Wall -fPIC -I/usr/local/include" - LDFLAGS="-I/usr/local/include -L/usr/local/lib -shared" - LUA_SUFFIX="51" - LUA_SUFFIX_SET=yes - LUA_DIR=/usr/local - LUA_DIR_SET=yes - CC=cc - LD=ld - fi - if [ "$OSPRESET" = "openbsd" ]; then - LUA_INCDIR="/usr/local/include"; - LUA_INCDIR_SET="yes" - fi - if [ "$OSPRESET" = "netbsd" ]; then - LUA_INCDIR="/usr/pkg/include/lua-5.1" - LUA_INCDIR_SET=yes - LUA_LIBDIR="/usr/pkg/lib/lua/5.1" - LUA_LIBDIR_SET=yes - CFLAGS="-Wall -fPIC -I/usr/pkg/include" - LDFLAGS="-L/usr/pkg/lib -Wl,-rpath,/usr/pkg/lib -shared" - fi - if [ "$OSPRESET" = "pkg-config" ]; then - if [ "$LUA_SUFFIX_SET" != "yes" ]; then - LUA_SUFFIX="5.1"; - LUA_SUFFIX_SET=yes - fi - LUA_CF="$(pkg-config --cflags-only-I lua$LUA_SUFFIX)" - LUA_CF="${LUA_CF#*-I}" - LUA_CF="${LUA_CF%% *}" - if [ "$LUA_CF" != "" ]; then - LUA_INCDIR="$LUA_CF" - LUA_INCDIR_SET=yes - fi - CFLAGS="$CFLAGS" - fi + OSPRESET_SET="yes" ;; --libdir) LIBDIR="$value" @@ -319,6 +253,76 @@ do shift done +if [ "$OSPRESET_SET" = "yes" ]; then + # TODO make this a switch? + if [ "$OSPRESET" = "debian" ]; then + if [ "$LUA_SUFFIX_SET" != "yes" ]; then + LUA_SUFFIX="5.1"; + LUA_SUFFIX_SET=yes + fi + if [ "$RUNWITH_SET" != "yes" ]; then + RUNWITH="lua$LUA_SUFFIX"; + RUNWITH_SET=yes + fi + LUA_INCDIR="/usr/include/lua$LUA_SUFFIX" + LUA_INCDIR_SET=yes + CFLAGS="$CFLAGS -ggdb" + fi + if [ "$OSPRESET" = "macosx" ]; then + LUA_INCDIR=/usr/local/include; + LUA_INCDIR_SET=yes + LUA_LIBDIR=/usr/local/lib + LUA_LIBDIR_SET=yes + CFLAGS="$CFLAGS -mmacosx-version-min=10.3" + LDFLAGS="-bundle -undefined dynamic_lookup" + fi + if [ "$OSPRESET" = "linux" ]; then + LUA_INCDIR=/usr/local/include; + LUA_INCDIR_SET=yes + LUA_LIBDIR=/usr/local/lib + LUA_LIBDIR_SET=yes + CFLAGS="$CFLAGS -ggdb" + fi + if [ "$OSPRESET" = "freebsd" ] || [ "$OSPRESET" = "openbsd" ]; then + LUA_INCDIR="/usr/local/include/lua51" + LUA_INCDIR_SET=yes + CFLAGS="-Wall -fPIC -I/usr/local/include" + LDFLAGS="-I/usr/local/include -L/usr/local/lib -shared" + LUA_SUFFIX="51" + LUA_SUFFIX_SET=yes + LUA_DIR=/usr/local + LUA_DIR_SET=yes + CC=cc + LD=ld + fi + if [ "$OSPRESET" = "openbsd" ]; then + LUA_INCDIR="/usr/local/include"; + LUA_INCDIR_SET="yes" + fi + if [ "$OSPRESET" = "netbsd" ]; then + LUA_INCDIR="/usr/pkg/include/lua-5.1" + LUA_INCDIR_SET=yes + LUA_LIBDIR="/usr/pkg/lib/lua/5.1" + LUA_LIBDIR_SET=yes + CFLAGS="-Wall -fPIC -I/usr/pkg/include" + LDFLAGS="-L/usr/pkg/lib -Wl,-rpath,/usr/pkg/lib -shared" + fi + if [ "$OSPRESET" = "pkg-config" ]; then + if [ "$LUA_SUFFIX_SET" != "yes" ]; then + LUA_SUFFIX="5.1"; + LUA_SUFFIX_SET=yes + fi + LUA_CF="$(pkg-config --cflags-only-I lua$LUA_SUFFIX)" + LUA_CF="${LUA_CF#*-I}" + LUA_CF="${LUA_CF%% *}" + if [ "$LUA_CF" != "" ]; then + LUA_INCDIR="$LUA_CF" + LUA_INCDIR_SET=yes + fi + CFLAGS="$CFLAGS" + fi +fi + if [ "$PREFIX_SET" = "yes" ] && [ ! "$SYSCONFDIR_SET" = "yes" ] then if [ "$PREFIX" = "/usr" ] -- cgit v1.2.3 From 617d39783136dc8405dcdeded238ec42a6dc8468 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 13 May 2019 11:59:00 +0200 Subject: configure: Remove preset settings that are autodiscovered These are likely wrong if other flags have been given. --- configure | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/configure b/configure index 33d86909..c9737ae6 100755 --- a/configure +++ b/configure @@ -256,16 +256,6 @@ done if [ "$OSPRESET_SET" = "yes" ]; then # TODO make this a switch? if [ "$OSPRESET" = "debian" ]; then - if [ "$LUA_SUFFIX_SET" != "yes" ]; then - LUA_SUFFIX="5.1"; - LUA_SUFFIX_SET=yes - fi - if [ "$RUNWITH_SET" != "yes" ]; then - RUNWITH="lua$LUA_SUFFIX"; - RUNWITH_SET=yes - fi - LUA_INCDIR="/usr/include/lua$LUA_SUFFIX" - LUA_INCDIR_SET=yes CFLAGS="$CFLAGS -ggdb" fi if [ "$OSPRESET" = "macosx" ]; then @@ -277,10 +267,6 @@ if [ "$OSPRESET_SET" = "yes" ]; then LDFLAGS="-bundle -undefined dynamic_lookup" fi if [ "$OSPRESET" = "linux" ]; then - LUA_INCDIR=/usr/local/include; - LUA_INCDIR_SET=yes - LUA_LIBDIR=/usr/local/lib - LUA_LIBDIR_SET=yes CFLAGS="$CFLAGS -ggdb" fi if [ "$OSPRESET" = "freebsd" ] || [ "$OSPRESET" = "openbsd" ]; then -- cgit v1.2.3 From 5c927a37de1ce01b20970e5245959c1c41a85b3f Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Mon, 13 May 2019 12:00:28 +0200 Subject: configure: Respect previously set paths in macosx preset --- configure | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/configure b/configure index c9737ae6..2b58efe5 100755 --- a/configure +++ b/configure @@ -259,10 +259,14 @@ if [ "$OSPRESET_SET" = "yes" ]; then CFLAGS="$CFLAGS -ggdb" fi if [ "$OSPRESET" = "macosx" ]; then - LUA_INCDIR=/usr/local/include; - LUA_INCDIR_SET=yes - LUA_LIBDIR=/usr/local/lib - LUA_LIBDIR_SET=yes + if [ "$LUA_INCDIR_SET" != "yes" ]; then + LUA_INCDIR=/usr/local/include; + LUA_INCDIR_SET=yes + fi + if [ "$LUA_LIBDIR_SET" != "yes" ]; then + LUA_LIBDIR=/usr/local/lib + LUA_LIBDIR_SET=yes + fi CFLAGS="$CFLAGS -mmacosx-version-min=10.3" LDFLAGS="-bundle -undefined dynamic_lookup" fi -- cgit v1.2.3 From cf5c64cd690c92ae7705fe6e7acce2ed81610ec2 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 5 Mar 2019 00:12:30 +0100 Subject: mod_storage_internal: Return error if 'before' or 'after' are not found (partial fix for #1325) --- plugins/mod_storage_internal.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index 96780b33..fdce3c98 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -161,20 +161,30 @@ function archive:find(username, query) if query.reverse then items:reverse(); if query.before then + local found = false; for j = 1, #items do if (items[j].key or tostring(j)) == query.before then + found = true; i = j; break; end end + if not found then + return nil, "item-not-found"; + end end elseif query.after then + local found = false; for j = 1, #items do if (items[j].key or tostring(j)) == query.after then + found = true; i = j; break; end end + if not found then + return nil, "item-not-found"; + end end if query.limit and #items - i > query.limit then items[i+query.limit+1] = nil; -- cgit v1.2.3 From 59b4972c34a6887caf3ecb8b96d088fef831d9e9 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 5 Mar 2019 00:16:41 +0100 Subject: mod_storage_memory: Return error if 'before' or 'after' are not found (partial fix for #1325) --- plugins/mod_storage_memory.lua | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua index 4655cb3a..2fae8828 100644 --- a/plugins/mod_storage_memory.lua +++ b/plugins/mod_storage_memory.lua @@ -127,20 +127,30 @@ function archive_store:find(username, query) if query.reverse then items:reverse(); if query.before then + local found = false; for j = 1, #items do if (items[j].key or tostring(j)) == query.before then + found = true; i = j; break; end end + if not found then + return nil, "item-not-found"; + end end elseif query.after then + local found = false; for j = 1, #items do if (items[j].key or tostring(j)) == query.after then + found = true; i = j; break; end end + if not found then + return nil, "item-not-found"; + end end if query.limit and #items - i > query.limit then items[i+query.limit+1] = nil; -- cgit v1.2.3 From 7e32666c2b78f5bebe46bcf336f66a3e7d2eff36 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sat, 23 Mar 2019 00:51:10 +0100 Subject: mod_storage_sql: Look up archive IDs in separate queries (fixes #1325) This is probably not good for performance. --- plugins/mod_storage_sql.lua | 58 ++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index f0a8fee0..5da5e448 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -327,38 +327,36 @@ local function archive_where(query, args, where) end end local function archive_where_id_range(query, args, where) - local args_len = #args -- Before or after specific item, exclusive + local id_lookup_sql = [[ + SELECT "sort_id" + FROM "prosodyarchive" + WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ? + LIMIT 1; + ]]; if query.after then -- keys better be unique! - where[#where+1] = [[ - "sort_id" > COALESCE( - ( - SELECT "sort_id" - FROM "prosodyarchive" - WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ? - LIMIT 1 - ), 0) - ]]; - args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.after, args[1], args[2], args[3]; - args_len = args_len + 4 + local after_id = nil; + for row in engine:select(id_lookup_sql, query.after, host, user or "", store) do + after_id = row[1]; + end + if not after_id then + return nil, "item-not-found"; + end + where[#where+1] = '"sort_id" > ?'; + args[#args+1] = after_id; end if query.before then - where[#where+1] = [[ - "sort_id" < COALESCE( - ( - SELECT "sort_id" - FROM "prosodyarchive" - WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ? - LIMIT 1 - ), - ( - SELECT MAX("sort_id")+1 - FROM "prosodyarchive" - ) - ) - ]] - args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.before, args[1], args[2], args[3]; + local before_id = nil; + for row in engine:select(id_lookup_sql, query.after, host, user or "", store) do + before_id = row[1]; + end + if not before_id then + return nil, "item-not-found"; + end + where[#where+1] = '"sort_id" < ?'; + args[#args+1] = before_id; end + return true; end function archive_store:find(username, query) @@ -398,7 +396,8 @@ function archive_store:find(username, query) end end - archive_where_id_range(query, args, where); + local ok, err = archive_where_id_range(query, args, where); + if not ok then return ok, err; end if query.limit then args[#args+1] = query.limit; @@ -466,7 +465,8 @@ function archive_store:delete(username, query) table.remove(where, 2); end archive_where(query, args, where); - archive_where_id_range(query, args, where); + local ok, err = archive_where_id_range(query, args, where); + if not ok then return ok, err; end if query.truncate == nil then sql_query = sql_query:format(t_concat(where, " AND ")); else -- cgit v1.2.3 From ffd7ca3f8daaa3d0137012759f7cfe52a835ea89 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 07:16:05 +0200 Subject: mod_mam: Propagate item-not-found to client (fixes #1325) --- plugins/mod_mam/mod_mam.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua index 632de9ea..317ddac1 100644 --- a/plugins/mod_mam/mod_mam.lua +++ b/plugins/mod_mam/mod_mam.lua @@ -142,7 +142,11 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event) }); if not data then - origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err)); + if err == "item-not-found" then + origin.send(st.error_reply(stanza, "modify", "item-not-found")); + else + origin.send(st.error_reply(stanza, "cancel", "internal-server-error")); + end return true; end local total = tonumber(err); -- cgit v1.2.3 From 43f98c242d4a212a8de0d8965733c206109c4d23 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 07:16:03 +0200 Subject: mod_muc_mam: Propagate item-not-found to client (fixes #1325) --- plugins/mod_muc_mam.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/mod_muc_mam.lua b/plugins/mod_muc_mam.lua index fffe23e7..bba7f422 100644 --- a/plugins/mod_muc_mam.lua +++ b/plugins/mod_muc_mam.lua @@ -189,7 +189,11 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event) }); if not data then - origin.send(st.error_reply(stanza, "cancel", "internal-server-error")); + if err == "item-not-found" then + origin.send(st.error_reply(stanza, "modify", "item-not-found")); + else + origin.send(st.error_reply(stanza, "cancel", "internal-server-error")); + end return true; end local total = tonumber(err); -- cgit v1.2.3 From faf1282ee2dfacfba3a947edd9142b93ca1f4162 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 07:22:18 +0200 Subject: mod_storage_memory: Return correct error even if no archive data available --- plugins/mod_storage_memory.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua index 2fae8828..376ae277 100644 --- a/plugins/mod_storage_memory.lua +++ b/plugins/mod_storage_memory.lua @@ -91,6 +91,9 @@ function archive_store:find(username, query) local items = self.store[username or NULL]; if not items then if query then + if query.before or query.after then + return nil, "item-not-found"; + end if query.total then return function () end, 0; end -- cgit v1.2.3 From fa0d394f9b0ab4f503558d9c944fe4052d5eb324 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 May 2019 07:24:12 +0200 Subject: mod_storage_internal: Return appropriate error even with empty archive --- plugins/mod_storage_internal.lua | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua index fdce3c98..2556224d 100644 --- a/plugins/mod_storage_internal.lua +++ b/plugins/mod_storage_internal.lua @@ -125,6 +125,9 @@ function archive:find(username, query) if err then return items, err; elseif query then + if query.before or query.after then + return nil, "item-not-found"; + end if query.total then return function () end, 0; end -- cgit v1.2.3 From 2dbd1528bb118e77463db85403d535d307ec050d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 26 May 2019 15:04:16 +0200 Subject: mod_csi_simple: Disable optimizations on disconnect (fixes #1358) The connection becomes invalid here, regardless of 3rd party modules that might keep the session alive. --- plugins/mod_csi_simple.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua index a9148618..13002ea8 100644 --- a/plugins/mod_csi_simple.lua +++ b/plugins/mod_csi_simple.lua @@ -108,6 +108,10 @@ module:hook("csi-client-active", function (event) disable_optimizations(session); end); +module:hook("pre-resource-unbind", function (event) + local session = event.session; + disable_optimizations(session); +end); module:hook("c2s-ondrain", function (event) local session = event.session; -- cgit v1.2.3 From 8799bf12c702335a32d1d220269090f653230b0d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 28 May 2019 00:46:24 +0200 Subject: mod_storage_sql: Correctly return item-not-found error `return ok, err` comes out as `transaction_ok, ok, err` --- plugins/mod_storage_sql.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index 5da5e448..cfc8450c 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -367,7 +367,7 @@ function archive_store:find(username, query) if total ~= nil and query.limit == 0 and query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then return noop, total; end - local ok, result = engine:transaction(function() + local ok, result, err = engine:transaction(function() local sql_query = [[ SELECT "key", "type", "value", "when", "with" FROM "prosodyarchive" @@ -407,7 +407,8 @@ function archive_store:find(username, query) and "DESC" or "ASC", query.limit and " LIMIT ?" or ""); return engine:select(sql_query, unpack(args)); end); - if not ok then return ok, result end + if not ok then return ok, result; end + if not result then return nil, err; end return function() local row = result(); if row ~= nil then -- cgit v1.2.3 From a52c045f4a8b7e11eac9b6ba4cf72e0a920594e5 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 28 May 2019 00:47:50 +0200 Subject: mod_storage_sql: Fix to use correct arguments to archive id lookup --- plugins/mod_storage_sql.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index cfc8450c..ad2de840 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -336,7 +336,7 @@ local function archive_where_id_range(query, args, where) ]]; if query.after then -- keys better be unique! local after_id = nil; - for row in engine:select(id_lookup_sql, query.after, host, user or "", store) do + for row in engine:select(id_lookup_sql, query.after, args[1], args[2], args[3]) do after_id = row[1]; end if not after_id then @@ -347,7 +347,7 @@ local function archive_where_id_range(query, args, where) end if query.before then local before_id = nil; - for row in engine:select(id_lookup_sql, query.after, host, user or "", store) do + for row in engine:select(id_lookup_sql, query.after, args[1], args[2], args[3]) do before_id = row[1]; end if not before_id then -- cgit v1.2.3 From 411d4c02a1315eb778e2ee5613723b74b6cfd00d Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 28 May 2019 00:56:30 +0200 Subject: mod_storage_sql: Ignore shadowed error variable [luacheck] --- plugins/mod_storage_sql.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua index ad2de840..518e2654 100644 --- a/plugins/mod_storage_sql.lua +++ b/plugins/mod_storage_sql.lua @@ -153,7 +153,7 @@ end local archive_item_limit = module:get_option_number("storage_archive_item_limit"); local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000)); --- luacheck: ignore 512 431/user 431/store +-- luacheck: ignore 512 431/user 431/store 431/err local map_store = {}; map_store.__index = map_store; map_store.remove = {}; -- cgit v1.2.3 From be249b1a12e22c3276598555bedf7beb8386f550 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 30 May 2019 13:41:05 +0200 Subject: util.format: Handle formats expecting an integer in Lua 5.3+ (fixes #1371) --- spec/util_format_spec.lua | 1 + util/format.lua | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/spec/util_format_spec.lua b/spec/util_format_spec.lua index b9652d19..82b70205 100644 --- a/spec/util_format_spec.lua +++ b/spec/util_format_spec.lua @@ -12,6 +12,7 @@ describe("util.format", function() assert.equal("[true]", format("%d", true)); assert.equal("% [true]", format("%%", true)); assert.equal("{ }", format("%q", { })); + assert.equal("[1.5]", format("%d", 1.5)); end); end); end); diff --git a/util/format.lua b/util/format.lua index c31f599f..857bb694 100644 --- a/util/format.lua +++ b/util/format.lua @@ -7,6 +7,9 @@ local unpack = table.unpack or unpack; -- luacheck: ignore 113/unpack local pack = require "util.table".pack; -- TODO table.pack in 5.2+ local type = type; local dump = require "util.serialization".new("debug"); +local num_type = math.type; + +local expects_integer = num_type and { c = true, d = true, i = true, o = true, u = true, X = true, x = true, } or {}; local function format(formatstring, ...) local args = pack(...); @@ -43,6 +46,9 @@ local function format(formatstring, ...) elseif type(arg) ~= "number" then -- arg isn't number as expected? args[i] = tostring(arg); spec = "[%s]"; + elseif expects_integer[option] and num_type(arg) ~= "integer" then + args[i] = tostring(arg); + spec = "[%s]"; end end return spec; -- cgit v1.2.3 From 9991f8892518b2d48cf27b3667a2fe33d25dcd67 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Thu, 30 May 2019 13:54:11 +0200 Subject: util.format: Handle integer formats the same way on Lua versions without integer support --- spec/util_format_spec.lua | 1 + util/format.lua | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spec/util_format_spec.lua b/spec/util_format_spec.lua index 82b70205..50509630 100644 --- a/spec/util_format_spec.lua +++ b/spec/util_format_spec.lua @@ -13,6 +13,7 @@ describe("util.format", function() assert.equal("% [true]", format("%%", true)); assert.equal("{ }", format("%q", { })); assert.equal("[1.5]", format("%d", 1.5)); + assert.equal("[7.3786976294838e+19]", format("%d", 73786976294838206464)); end); end); end); diff --git a/util/format.lua b/util/format.lua index 857bb694..1ce670f3 100644 --- a/util/format.lua +++ b/util/format.lua @@ -7,9 +7,12 @@ local unpack = table.unpack or unpack; -- luacheck: ignore 113/unpack local pack = require "util.table".pack; -- TODO table.pack in 5.2+ local type = type; local dump = require "util.serialization".new("debug"); -local num_type = math.type; +local num_type = math.type or function (n) + return n % 1 == 0 and n <= 9007199254740992 and n >= -9007199254740992 and "integer" or "float"; +end -local expects_integer = num_type and { c = true, d = true, i = true, o = true, u = true, X = true, x = true, } or {}; +-- In Lua 5.3+ these formats throw an error if given a float +local expects_integer = { c = true, d = true, i = true, o = true, u = true, X = true, x = true, }; local function format(formatstring, ...) local args = pack(...); -- cgit v1.2.3 From 4c7de2595bc229710b3985f96bea96a25c053300 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 31 May 2019 17:01:22 +0200 Subject: prosody: Log shutdown reason --- prosody | 1 + 1 file changed, 1 insertion(+) diff --git a/prosody b/prosody index 204fb36d..f0061bf1 100755 --- a/prosody +++ b/prosody @@ -90,6 +90,7 @@ end loop(); prosody.log("info", "Shutting down..."); +prosody.log("debug", "Shutdown reason is: %s", prosody.shutdown_reason or "not specified"); cleanup(); prosody.events.fire_event("server-stopped"); prosody.log("info", "Shutdown complete"); -- cgit v1.2.3 From c3c7aaae0f69ab4b2dfcef94f6224f46613fb191 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 31 May 2019 18:50:13 +0200 Subject: prosody: Log shutdown reason (in past tense) as the very last thing --- prosody | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prosody b/prosody index f0061bf1..ced319e0 100755 --- a/prosody +++ b/prosody @@ -90,9 +90,9 @@ end loop(); prosody.log("info", "Shutting down..."); -prosody.log("debug", "Shutdown reason is: %s", prosody.shutdown_reason or "not specified"); cleanup(); prosody.events.fire_event("server-stopped"); prosody.log("info", "Shutdown complete"); +prosody.log("debug", "Shutdown reason was: %s", prosody.shutdown_reason or "not specified"); os.exit(prosody.shutdown_code); -- cgit v1.2.3 From 9bca9eb263362d4f6c4f4e4a790b33676c509e03 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 31 May 2019 18:50:34 +0200 Subject: prosody: Also log status code passed to exit() Sometimes you're just too lazy to `echo $?` --- prosody | 1 + 1 file changed, 1 insertion(+) diff --git a/prosody b/prosody index ced319e0..e82318d1 100755 --- a/prosody +++ b/prosody @@ -95,4 +95,5 @@ prosody.events.fire_event("server-stopped"); prosody.log("info", "Shutdown complete"); prosody.log("debug", "Shutdown reason was: %s", prosody.shutdown_reason or "not specified"); +prosody.log("debug", "Exiting with status code: %d", prosody.shutdown_code or 0); os.exit(prosody.shutdown_code); -- cgit v1.2.3 From cd4878033fd3f3b6b7dd67d66f8f02423ee11b94 Mon Sep 17 00:00:00 2001 From: Maxime ?pep? Buquet Date: Sat, 1 Jun 2019 15:00:35 +0200 Subject: core/sessionmanager: Remove unnecessary fallback in make_authenticated --- core/sessionmanager.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/sessionmanager.lua b/core/sessionmanager.lua index f5af1185..55f096b9 100644 --- a/core/sessionmanager.lua +++ b/core/sessionmanager.lua @@ -129,7 +129,7 @@ local function make_authenticated(session, username) if session.type == "c2s_unauthed" then session.type = "c2s_unbound"; end - session.log("info", "Authenticated as %s@%s", username or "(unknown)", session.host or "(unknown)"); + session.log("info", "Authenticated as %s@%s", username, session.host or "(unknown)"); return true; end -- cgit v1.2.3 From 9a5a3fd0f10693e6cba2779095398614eb94e68c Mon Sep 17 00:00:00 2001 From: Michel Le Bihan Date: Mon, 3 Jun 2019 20:51:15 +0200 Subject: mod_admin_telnet: Collect array from Bosh connections when appending to connection list Fixes #1356 --- plugins/mod_admin_telnet.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua index 0fbd3ff9..fa03840b 100644 --- a/plugins/mod_admin_telnet.lua +++ b/plugins/mod_admin_telnet.lua @@ -597,7 +597,7 @@ end local function show_c2s(callback) local c2s = array.collect(values(module:shared"/*/c2s/sessions")); - c2s:append(values(module:shared"/*/bosh/sessions")); + c2s:append(array.collect(values(module:shared"/*/bosh/sessions"))); c2s:sort(function(a, b) if a.host == b.host then if a.username == b.username then -- cgit v1.2.3