Merge 0.11->trunk

73 files changed, 4493 insertions, 2719 deletions

diff --git a/plugins/adhoc/adhoc.lib.lua b/plugins/adhoc/adhoc.lib.lua
index 0b910299..0c61a636 100644
--- a/plugins/adhoc/adhoc.lib.lua
+++ b/plugins/adhoc/adhoc.lib.lua
@@ -21,7 +21,13 @@ local function _cmdtag(desc, status, sessionid, action)
 end
 
 function _M.new(name, node, handler, permission)
-	return { name = name, node = node, handler = handler, cmdtag = _cmdtag, permission = (permission or "user") };
+	if not permission then
+		error "adhoc.new() expects a permission argument, none given"
+	end
+	if permission == "user" then
+		error "the permission mode 'user' has been renamed 'any', please update your code"
+	end
+	return { name = name, node = node, handler = handler, cmdtag = _cmdtag, permission = permission };
 end
 
 function _M.handle_cmd(command, origin, stanza)
diff --git a/plugins/adhoc/mod_adhoc.lua b/plugins/adhoc/mod_adhoc.lua
index bf1775b4..188d05e8 100644
--- a/plugins/adhoc/mod_adhoc.lua
+++ b/plugins/adhoc/mod_adhoc.lua
@@ -8,7 +8,7 @@
 local it = require "util.iterators";
 local st = require "util.stanza";
 local is_admin = require "core.usermanager".is_admin;
-local jid_split = require "util.jid".split;
+local jid_host = require "util.jid".host;
 local adhoc_handle_cmd = module:require "adhoc".handle_cmd;
 local xmlns_cmd = "http://jabber.org/protocol/commands";
 local commands = {};
@@ -21,12 +21,12 @@ module:hook("host-disco-info-node", function (event)
 		local from = stanza.attr.from;
 		local privileged = is_admin(from, stanza.attr.to);
 		local global_admin = is_admin(from);
-		local username, hostname = jid_split(from);
+		local hostname = jid_host(from);
 		local command = commands[node];
 		if (command.permission == "admin" and privileged)
 		    or (command.permission == "global_admin" and global_admin)
 		    or (command.permission == "local_user" and hostname == module.host)
-		    or (command.permission == "user") then
+		    or (command.permission == "any") then
 			reply:tag("identity", { name = command.name,
 			    category = "automation", type = "command-node" }):up();
 			reply:tag("feature", { var = xmlns_cmd }):up();
@@ -52,12 +52,12 @@ module:hook("host-disco-items-node", function (event)
 	local from = stanza.attr.from;
 	local admin = is_admin(from, stanza.attr.to);
 	local global_admin = is_admin(from);
-	local username, hostname = jid_split(from);
+	local hostname = jid_host(from);
 	for node, command in it.sorted_pairs(commands) do
 		if (command.permission == "admin" and admin)
 		    or (command.permission == "global_admin" and global_admin)
 		    or (command.permission == "local_user" and hostname == module.host)
-		    or (command.permission == "user") then
+		    or (command.permission == "any") then
 			reply:tag("item", { name = command.name,
 			    node = node, jid = module:get_host() });
 			reply:up();
@@ -74,7 +74,7 @@ module:hook("iq-set/host/"..xmlns_cmd..":command", function (event)
 		local from = stanza.attr.from;
 		local admin = is_admin(from, stanza.attr.to);
 		local global_admin = is_admin(from);
-		local username, hostname = jid_split(from);
+		local hostname = jid_host(from);
 		if (command.permission == "admin" and not admin)
 		    or (command.permission == "global_admin" and not global_admin)
 		    or (command.permission == "local_user" and hostname ~= module.host) then
diff --git a/plugins/mod_admin_adhoc.lua b/plugins/mod_admin_adhoc.lua
index 37e77ab0..674b3339 100644
--- a/plugins/mod_admin_adhoc.lua
+++ b/plugins/mod_admin_adhoc.lua
@@ -59,7 +59,7 @@ local add_user_command_handler = adhoc_simple(add_user_layout, function(fields,
 	if err then
 		return generate_error_message(err);
 	end
-	local username, host, resource = jid.split(fields.accountjid);
+	local username, host = jid.split(fields.accountjid);
 	if module_host ~= host then
 		return { status = "completed", error = { message = "Trying to add a user on " .. host .. " but command was sent to " .. module_host}};
 	end
@@ -94,7 +94,7 @@ local change_user_password_command_handler = adhoc_simple(change_user_password_l
 	if err then
 		return generate_error_message(err);
 	end
-	local username, host, resource = jid.split(fields.accountjid);
+	local username, host = jid.split(fields.accountjid);
 	if module_host ~= host then
 		return {
 			status = "completed",
@@ -136,7 +136,7 @@ local delete_user_command_handler = adhoc_simple(delete_user_layout, function(fi
 	local failed = {};
 	local succeeded = {};
 	for _, aJID in ipairs(fields.accountjids) do
-		local username, host, resource = jid.split(aJID);
+		local username, host = jid.split(aJID);
 		if (host == module_host) and  usermanager_user_exists(username, host) and usermanager_delete_user(username, host) then
 			module:log("debug", "User %s has been deleted", aJID);
 			succeeded[#succeeded+1] = aJID;
@@ -180,7 +180,7 @@ local end_user_session_handler = adhoc_simple(end_user_session_layout, function(
 	local failed = {};
 	local succeeded = {};
 	for _, aJID in ipairs(fields.accountjids) do
-		local username, host, resource = jid.split(aJID);
+		local username, host = jid.split(aJID);
 		if (host == module_host) and  usermanager_user_exists(username, host) and disconnect_user(aJID) then
 			succeeded[#succeeded+1] = aJID;
 		else
@@ -212,7 +212,7 @@ local get_user_password_handler = adhoc_simple(get_user_password_layout, functio
 	if err then
 		return generate_error_message(err);
 	end
-	local user, host, resource = jid.split(fields.accountjid);
+	local user, host = jid.split(fields.accountjid);
 	local accountjid;
 	local password;
 	if host ~= module_host then
@@ -243,7 +243,7 @@ local get_user_roster_handler = adhoc_simple(get_user_roster_layout, function(fi
 		return generate_error_message(err);
 	end
 
-	local user, host, resource = jid.split(fields.accountjid);
+	local user, host = jid.split(fields.accountjid);
 	if host ~= module_host then
 		return { status = "completed", error = { message = "Tried to get roster for a user on " .. host .. " but command was sent to " .. module_host } };
 	elseif not usermanager_user_exists(user, host) then
@@ -392,6 +392,12 @@ local function session_flags(session, line)
 	if session.cert_identity_status == "valid" then
 		flags[#flags+1] = "authenticated";
 	end
+	if session.dialback_key then
+		flags[#flags+1] = "dialback";
+	end
+	if session.external_auth then
+		flags[#flags+1] = "SASL";
+	end
 	if session.secure then
 		flags[#flags+1] = "encrypted";
 	end
@@ -404,6 +410,12 @@ local function session_flags(session, line)
 	if session.ip and session.ip:match(":") then
 		flags[#flags+1] = "IPv6";
 	end
+	if session.incoming and session.outgoing then
+		flags[#flags+1] = "bidi";
+	elseif session.is_bidi or session.bidi_session then
+		flags[#flags+1] = "bidi";
+	end
+
 	line[#line+1] = "("..t_concat(flags, ", ")..")";
 
 	return t_concat(line, " ");
diff --git a/plugins/mod_admin_shell.lua b/plugins/mod_admin_shell.lua
new file mode 100644
index 00000000..5b4c2a61
--- /dev/null
+++ b/plugins/mod_admin_shell.lua
@@ -0,0 +1,1665 @@
+-- Prosody IM
+-- Copyright (C) 2008-2010 Matthew Wild
+-- Copyright (C) 2008-2010 Waqas Hussain
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+-- luacheck: ignore 212/self
+
+module:set_global();
+module:depends("admin_socket");
+
+local hostmanager = require "core.hostmanager";
+local modulemanager = require "core.modulemanager";
+local s2smanager = require "core.s2smanager";
+local portmanager = require "core.portmanager";
+local helpers = require "util.helpers";
+local server = require "net.server";
+local st = require "util.stanza";
+
+local _G = _G;
+
+local prosody = _G.prosody;
+
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
+local iterators = require "util.iterators";
+local keys, values = iterators.keys, iterators.values;
+local jid_bare, jid_split, jid_join = import("util.jid", "bare", "prepped_split", "join");
+local set, array = require "util.set", require "util.array";
+local cert_verify_identity = require "util.x509".verify_identity;
+local envload = require "util.envload".envload;
+local envloadfile = require "util.envload".envloadfile;
+local has_pposix, pposix = pcall(require, "util.pposix");
+local async = require "util.async";
+local serialization = require "util.serialization";
+local serialize_config = serialization.new ({ fatal = false, unquoted = true});
+local time = require "util.time";
+
+local format_number = require "util.human.units".format;
+
+local commands = module:shared("commands")
+local def_env = module:shared("env");
+local default_env_mt = { __index = def_env };
+
+local function redirect_output(target, session)
+	local env = setmetatable({ print = session.print }, { __index = function (_, k) return rawget(target, k); end });
+	env.dofile = function(name)
+		local f, err = envloadfile(name, env);
+		if not f then return f, err; end
+		return f();
+	end;
+	return env;
+end
+
+console = {};
+
+local runner_callbacks = {};
+
+function runner_callbacks:error(err)
+	module:log("error", "Traceback[shell]: %s", err);
+
+	self.data.print("Fatal error while running command, it did not complete");
+	self.data.print("Error: "..tostring(err));
+end
+
+local function send_repl_output(session, line)
+	return session.send(st.stanza("repl-output"):text(tostring(line)));
+end
+
+function console:new_session(admin_session)
+	local session = {
+			send = function (t)
+				return send_repl_output(admin_session, t);
+			end;
+			print = function (...)
+				local t = {};
+				for i=1,select("#", ...) do
+					t[i] = tostring(select(i, ...));
+				end
+				return send_repl_output(admin_session, table.concat(t, "\t"));
+			end;
+			serialize = tostring;
+			disconnect = function () admin_session:close(); end;
+			};
+	session.env = setmetatable({}, default_env_mt);
+
+	session.thread = async.runner(function (line)
+		console:process_line(session, line);
+	end, runner_callbacks, session);
+
+	-- Load up environment with helper objects
+	for name, t in pairs(def_env) do
+		if type(t) == "table" then
+			session.env[name] = setmetatable({ session = session }, { __index = t });
+		end
+	end
+
+	session.env.output:configure();
+
+	return session;
+end
+
+local function handle_line(event)
+	local session = event.origin.shell_session;
+	if not session then
+		session = console:new_session(event.origin);
+		event.origin.shell_session = session;
+	end
+	local line = event.stanza:get_text();
+	local useglobalenv;
+
+	local result = st.stanza("repl-result");
+
+	if line:match("^>") then
+		line = line:gsub("^>", "");
+		useglobalenv = true;
+	else
+		local command = line:match("^%w+") or line:match("%p");
+		if commands[command] then
+			commands[command](session, line);
+			event.origin.send(result);
+			return;
+		end
+	end
+
+	session.env._ = line;
+
+	if not useglobalenv and commands[line:lower()] then
+		commands[line:lower()](session, line);
+		event.origin.send(result);
+		return;
+	end
+
+	local chunkname = "=console";
+	local env = (useglobalenv and redirect_output(_G, session)) or session.env or nil
+	-- luacheck: ignore 311/err
+	local chunk, err = envload("return "..line, chunkname, env);
+	if not chunk then
+		chunk, err = envload(line, chunkname, env);
+		if not chunk then
+			err = err:gsub("^%[string .-%]:%d+: ", "");
+			err = err:gsub("^:%d+: ", "");
+			err = err:gsub("'<eof>'", "the end of the line");
+			result.attr.type = "error";
+			result:text("Sorry, I couldn't understand that... "..err);
+			event.origin.send(result);
+			return;
+		end
+	end
+
+	local taskok, message = chunk();
+
+	if not message then
+		if type(taskok) ~= "string" and useglobalenv then
+			taskok = session.serialize(taskok);
+		end
+		result:text("Result: "..tostring(taskok));
+	elseif (not taskok) and message then
+		result.attr.type = "error";
+		result:text("Error: "..tostring(message));
+	else
+		result:text("OK: "..tostring(message));
+	end
+
+	event.origin.send(result);
+end
+
+module:hook("admin/repl-input", function (event)
+	local ok, err = pcall(handle_line, event);
+	if not ok then
+		event.origin.send(st.stanza("repl-result", { type = "error" }):text(err));
+	end
+end);
+
+-- Console commands --
+-- These are simple commands, not valid standalone in Lua
+
+function commands.help(session, data)
+	local print = session.print;
+	local section = data:match("^help (%w+)");
+	if not section then
+		print [[Commands are divided into multiple sections. For help on a particular section, ]]
+		print [[type: help SECTION (for example, 'help c2s'). Sections are: ]]
+		print [[]]
+		print [[c2s - Commands to manage local client-to-server sessions]]
+		print [[s2s - Commands to manage sessions between this server and others]]
+		print [[http - Commands to inspect HTTP services]] -- XXX plural but there is only one so far
+		print [[module - Commands to load/reload/unload modules/plugins]]
+		print [[host - Commands to activate, deactivate and list virtual hosts]]
+		print [[user - Commands to create and delete users, and change their passwords]]
+		print [[server - Uptime, version, shutting down, etc.]]
+		print [[port - Commands to manage ports the server is listening on]]
+		print [[dns - Commands to manage and inspect the internal DNS resolver]]
+		print [[xmpp - Commands for sending XMPP stanzas]]
+		print [[debug - Commands for debugging the server]]
+		print [[config - Reloading the configuration, etc.]]
+		print [[console - Help regarding the console itself]]
+	elseif section == "c2s" then
+		print [[c2s:show(jid) - Show all client sessions with the specified JID (or all if no JID given)]]
+		print [[c2s:show_insecure() - Show all unencrypted client connections]]
+		print [[c2s:show_secure() - Show all encrypted client connections]]
+		print [[c2s:show_tls() - Show TLS cipher info for encrypted sessions]]
+		print [[c2s:count() - Count sessions without listing them]]
+		print [[c2s:close(jid) - Close all sessions for the specified JID]]
+		print [[c2s:closeall() - Close all active c2s connections ]]
+	elseif section == "s2s" then
+		print [[s2s:show(domain) - Show all s2s connections for the given domain (or all if no domain given)]]
+		print [[s2s:show_tls(domain) - Show TLS cipher info for encrypted sessions]]
+		print [[s2s:close(from, to) - Close a connection from one domain to another]]
+		print [[s2s:closeall(host) - Close all the incoming/outgoing s2s sessions to specified host]]
+	elseif section == "http" then
+		print [[http:list(hosts) - Show HTTP endpoints]]
+	elseif section == "module" then
+		print [[module:load(module, host) - Load the specified module on the specified host (or all hosts if none given)]]
+		print [[module:reload(module, host) - The same, but unloads and loads the module (saving state if the module supports it)]]
+		print [[module:unload(module, host) - The same, but just unloads the module from memory]]
+		print [[module:list(host) - List the modules loaded on the specified host]]
+	elseif section == "host" then
+		print [[host:activate(hostname) - Activates the specified host]]
+		print [[host:deactivate(hostname) - Disconnects all clients on this host and deactivates]]
+		print [[host:list() - List the currently-activated hosts]]
+	elseif section == "user" then
+		print [[user:create(jid, password) - Create the specified user account]]
+		print [[user:password(jid, password) - Set the password for the specified user account]]
+		print [[user:delete(jid) - Permanently remove the specified user account]]
+		print [[user:list(hostname, pattern) - List users on the specified host, optionally filtering with a pattern]]
+	elseif section == "server" then
+		print [[server:version() - Show the server's version number]]
+		print [[server:uptime() - Show how long the server has been running]]
+		print [[server:memory() - Show details about the server's memory usage]]
+		print [[server:shutdown(reason) - Shut down the server, with an optional reason to be broadcast to all connections]]
+	elseif section == "port" then
+		print [[port:list() - Lists all network ports prosody currently listens on]]
+		print [[port:close(port, interface) - Close a port]]
+	elseif section == "dns" then
+		print [[dns:lookup(name, type, class) - Do a DNS lookup]]
+		print [[dns:addnameserver(nameserver) - Add a nameserver to the list]]
+		print [[dns:setnameserver(nameserver) - Replace the list of name servers with the supplied one]]
+		print [[dns:purge() - Clear the DNS cache]]
+		print [[dns:cache() - Show cached records]]
+	elseif section == "xmpp" then
+		print [[xmpp:ping(localhost, remotehost) -- Sends a ping to a remote XMPP server and reports the response]]
+	elseif section == "config" then
+		print [[config:reload() - Reload the server configuration. Modules may need to be reloaded for changes to take effect.]]
+		print [[config:get([host,] option) - Show the value of a config option.]]
+	elseif section == "stats" then -- luacheck: ignore 542
+		-- TODO describe how stats:show() works
+	elseif section == "debug" then
+		print [[debug:logevents(host) - Enable logging of fired events on host]]
+		print [[debug:events(host, event) - Show registered event handlers]]
+		print [[debug:timers() - Show information about scheduled timers]]
+	elseif section == "console" then
+		print [[Hey! Welcome to Prosody's admin console.]]
+		print [[First thing, if you're ever wondering how to get out, simply type 'quit'.]]
+		print [[Secondly, note that we don't support the full telnet protocol yet (it's coming)]]
+		print [[so you may have trouble using the arrow keys, etc. depending on your system.]]
+		print [[]]
+		print [[For now we offer a couple of handy shortcuts:]]
+		print [[!! - Repeat the last command]]
+		print [[!old!new! - repeat the last command, but with 'old' replaced by 'new']]
+		print [[]]
+		print [[For those well-versed in Prosody's internals, or taking instruction from those who are,]]
+		print [[you can prefix a command with > to escape the console sandbox, and access everything in]]
+		print [[the running server. Great fun, but be careful not to break anything :)]]
+	end
+end
+
+-- Session environment --
+-- Anything in def_env will be accessible within the session as a global variable
+
+--luacheck: ignore 212/self
+local serialize_defaults = module:get_option("console_prettyprint_settings", { fatal = false, unquoted = true, maxdepth = 2})
+
+def_env.output = {};
+function def_env.output:configure(opts)
+	if type(opts) ~= "table" then
+		opts = { preset = opts };
+	end
+	if not opts.fallback then
+		-- XXX Error message passed to fallback is lost, does it matter?
+		opts.fallback = tostring;
+	end
+	for k,v in pairs(serialize_defaults) do
+		if opts[k] == nil then
+			opts[k] = v;
+		end
+	end
+	self.session.serialize = serialization.new(opts);
+end
+
+def_env.server = {};
+
+function def_env.server:insane_reload()
+	prosody.unlock_globals();
+	dofile "prosody"
+	prosody = _G.prosody;
+	return true, "Server reloaded";
+end
+
+function def_env.server:version()
+	return true, tostring(prosody.version or "unknown");
+end
+
+function def_env.server:uptime()
+	local t = os.time()-prosody.start_time;
+	local seconds = t%60;
+	t = (t - seconds)/60;
+	local minutes = t%60;
+	t = (t - minutes)/60;
+	local hours = t%24;
+	t = (t - hours)/24;
+	local days = t;
+	return true, string.format("This server has been running for %d day%s, %d hour%s and %d minute%s (since %s)",
+		days, (days ~= 1 and "s") or "", hours, (hours ~= 1 and "s") or "",
+		minutes, (minutes ~= 1 and "s") or "", os.date("%c", prosody.start_time));
+end
+
+function def_env.server:shutdown(reason)
+	prosody.shutdown(reason);
+	return true, "Shutdown initiated";
+end
+
+local function human(kb)
+	return format_number(kb*1024, "B", "b");
+end
+
+function def_env.server:memory()
+	if not has_pposix or not pposix.meminfo then
+		return true, "Lua is using "..human(collectgarbage("count"));
+	end
+	local mem, lua_mem = pposix.meminfo(), collectgarbage("count");
+	local print = self.session.print;
+	print("Process: "..human((mem.allocated+mem.allocated_mmap)/1024));
+	print("   Used: "..human(mem.used/1024).." ("..human(lua_mem).." by Lua)");
+	print("   Free: "..human(mem.unused/1024).." ("..human(mem.returnable/1024).." returnable)");
+	return true, "OK";
+end
+
+def_env.module = {};
+
+local function get_hosts_set(hosts)
+	if type(hosts) == "table" then
+		if hosts[1] then
+			return set.new(hosts);
+		elseif hosts._items then
+			return hosts;
+		end
+	elseif type(hosts) == "string" then
+		return set.new { hosts };
+	elseif hosts == nil then
+		return set.new(array.collect(keys(prosody.hosts)));
+	end
+end
+
+-- Hosts with a module or all virtualhosts if no module given
+-- matching modules_enabled in the global section
+local function get_hosts_with_module(hosts, module)
+	local hosts_set = get_hosts_set(hosts)
+	/ function (host)
+			if module then
+				-- Module given, filter in hosts with this module loaded
+				if modulemanager.is_loaded(host, module) then
+					return host;
+				else
+					return nil;
+				end
+			end
+			if not hosts then
+				-- No hosts given, filter in VirtualHosts
+				if prosody.hosts[host].type == "local" then
+					return host;
+				else
+					return nil
+				end
+			end;
+			-- No module given, but hosts are, don't filter at all
+			return host;
+		end;
+	if module and modulemanager.get_module("*", module) then
+		hosts_set:add("*");
+	end
+	return hosts_set;
+end
+
+function def_env.module:load(name, hosts, config)
+	hosts = get_hosts_with_module(hosts);
+
+	-- Load the module for each host
+	local ok, err, count, mod = true, nil, 0;
+	for host in hosts do
+		if (not modulemanager.is_loaded(host, name)) then
+			mod, err = modulemanager.load(host, name, config);
+			if not mod then
+				ok = false;
+				if err == "global-module-already-loaded" then
+					if count > 0 then
+						ok, err, count = true, nil, 1;
+					end
+					break;
+				end
+				self.session.print(err or "Unknown error loading module");
+			else
+				count = count + 1;
+				self.session.print("Loaded for "..mod.module.host);
+			end
+		end
+	end
+
+	return ok, (ok and "Module loaded onto "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
+end
+
+function def_env.module:unload(name, hosts)
+	hosts = get_hosts_with_module(hosts, name);
+
+	-- Unload the module for each host
+	local ok, err, count = true, nil, 0;
+	for host in hosts do
+		if modulemanager.is_loaded(host, name) then
+			ok, err = modulemanager.unload(host, name);
+			if not ok then
+				ok = false;
+				self.session.print(err or "Unknown error unloading module");
+			else
+				count = count + 1;
+				self.session.print("Unloaded from "..host);
+			end
+		end
+	end
+	return ok, (ok and "Module unloaded from "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
+end
+
+local function _sort_hosts(a, b)
+	if a == "*" then return true
+	elseif b == "*" then return false
+	else return a:gsub("[^.]+", string.reverse):reverse() < b:gsub("[^.]+", string.reverse):reverse(); end
+end
+
+function def_env.module:reload(name, hosts)
+	hosts = array.collect(get_hosts_with_module(hosts, name)):sort(_sort_hosts)
+
+	-- Reload the module for each host
+	local ok, err, count = true, nil, 0;
+	for _, host in ipairs(hosts) do
+		if modulemanager.is_loaded(host, name) then
+			ok, err = modulemanager.reload(host, name);
+			if not ok then
+				ok = false;
+				self.session.print(err or "Unknown error reloading module");
+			else
+				count = count + 1;
+				if ok == nil then
+					ok = true;
+				end
+				self.session.print("Reloaded on "..host);
+			end
+		end
+	end
+	return ok, (ok and "Module reloaded on "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
+end
+
+function def_env.module:list(hosts)
+	hosts = array.collect(set.new({ not hosts and "*" or nil }) + get_hosts_set(hosts)):sort(_sort_hosts);
+
+	local print = self.session.print;
+	for _, host in ipairs(hosts) do
+		print((host == "*" and "Global" or host)..":");
+		local modules = array.collect(keys(modulemanager.get_modules(host) or {})):sort();
+		if #modules == 0 then
+			if prosody.hosts[host] then
+				print("    No modules loaded");
+			else
+				print("    Host not found");
+			end
+		else
+			for _, name in ipairs(modules) do
+				local status, status_text = modulemanager.get_module(host, name).module:get_status();
+				local status_summary = "";
+				if status == "warn" or status == "error" then
+					status_summary = (" (%s: %s)"):format(status, status_text);
+				end
+				print(("    %s%s"):format(name, status_summary));
+			end
+		end
+	end
+end
+
+def_env.config = {};
+function def_env.config:load(filename, format)
+	local config_load = require "core.configmanager".load;
+	local ok, err = config_load(filename, format);
+	if not ok then
+		return false, err or "Unknown error loading config";
+	end
+	return true, "Config loaded";
+end
+
+function def_env.config:get(host, key)
+	if key == nil then
+		host, key = "*", host;
+	end
+	local config_get = require "core.configmanager".get
+	return true, serialize_config(config_get(host, key));
+end
+
+function def_env.config:reload()
+	local ok, err = prosody.reload_config();
+	return ok, (ok and "Config reloaded (you may need to reload modules to take effect)") or tostring(err);
+end
+
+local function common_info(session, line)
+	if session.id then
+		line[#line+1] = "["..session.id.."]"
+	else
+		line[#line+1] = "["..session.type..(tostring(session):match("%x*$")).."]"
+	end
+end
+
+local function session_flags(session, line)
+	line = line or {};
+	common_info(session, line);
+	if session.type == "c2s" then
+		local status, priority = "unavailable", tostring(session.priority or "-");
+		if session.presence then
+			status = session.presence:get_child_text("show") or "available";
+		end
+		line[#line+1] = status.."("..priority..")";
+	end
+	if session.cert_identity_status == "valid" then
+		line[#line+1] = "(authenticated)";
+	end
+	if session.dialback_key then
+		line[#line+1] = "(dialback)";
+	end
+	if session.external_auth then
+		line[#line+1] = "(SASL)";
+	end
+	if session.secure then
+		line[#line+1] = "(encrypted)";
+	end
+	if session.compressed then
+		line[#line+1] = "(compressed)";
+	end
+	if session.smacks then
+		line[#line+1] = "(sm)";
+	end
+	if session.state then
+		if type(session.csi_counter) == "number" then
+			line[#line+1] = string.format("(csi:%s queue #%d)", session.state, session.csi_counter);
+		else
+			line[#line+1] = string.format("(csi:%s)", session.state);
+		end
+	end
+	if session.ip and session.ip:match(":") then
+		line[#line+1] = "(IPv6)";
+	end
+	if session.remote then
+		line[#line+1] = "(remote)";
+	end
+	if session.incoming and session.outgoing then
+		line[#line+1] = "(bidi)";
+	elseif session.is_bidi or session.bidi_session then
+		line[#line+1] = "(bidi)";
+	end
+	if session.bosh_version then
+		line[#line+1] = "(bosh)";
+	end
+	if session.websocket_request then
+		line[#line+1] = "(websocket)";
+	end
+	return table.concat(line, " ");
+end
+
+local function tls_info(session, line)
+	line = line or {};
+	common_info(session, line);
+	if session.secure then
+		local sock = session.conn and session.conn.socket and session.conn:socket();
+		if sock then
+			local info = sock.info and sock:info();
+			if info then
+				line[#line+1] = ("(%s with %s)"):format(info.protocol, info.cipher);
+			else
+				-- TLS session might not be ready yet
+				line[#line+1] = "(cipher info unavailable)";
+			end
+			if sock.getsniname then
+				local name = sock:getsniname();
+				if name then
+					line[#line+1] = ("(SNI:%q)"):format(name);
+				end
+			end
+			if sock.getalpn then
+				local proto = sock:getalpn();
+				if proto then
+					line[#line+1] = ("(ALPN:%q)"):format(proto);
+				end
+			end
+		end
+	else
+		line[#line+1] = "(insecure)";
+	end
+	return table.concat(line, " ");
+end
+
+def_env.c2s = {};
+
+local function get_jid(session)
+	if session.username then
+		return session.full_jid or jid_join(session.username, session.host, session.resource);
+	end
+
+	local conn = session.conn;
+	local ip = session.ip or "?";
+	local clientport = conn and conn:clientport() or "?";
+	local serverip = conn and conn.server and conn:server():ip() or "?";
+	local serverport = conn and conn:serverport() or "?"
+	return jid_join("["..ip.."]:"..clientport, session.host or "["..serverip.."]:"..serverport);
+end
+
+local function get_c2s()
+	local c2s = array.collect(values(prosody.full_sessions));
+	c2s:append(array.collect(values(module:shared"/*/c2s/sessions")));
+	c2s:append(array.collect(values(module:shared"/*/bosh/sessions")));
+	c2s:unique();
+	return c2s;
+end
+
+local function show_c2s(callback)
+	get_c2s():sort(function(a, b)
+		if a.host == b.host then
+			if a.username == b.username then
+				return (a.resource or "") > (b.resource or "");
+			end
+			return (a.username or "") > (b.username or "");
+		end
+		return _sort_hosts(a.host or "", b.host or "");
+	end):map(function (session)
+		callback(get_jid(session), session)
+	end);
+end
+
+function def_env.c2s:count()
+	local c2s = get_c2s();
+	return true, "Total: "..  #c2s .." clients";
+end
+
+function def_env.c2s:show(match_jid, annotate)
+	local print, count = self.session.print, 0;
+	annotate = annotate or session_flags;
+	local curr_host = false;
+	show_c2s(function (jid, session)
+		if curr_host ~= session.host then
+			curr_host = session.host;
+			print(curr_host or "(not connected to any host yet)");
+		end
+		if (not match_jid) or jid:match(match_jid) then
+			count = count + 1;
+			print(annotate(session, { "  ", jid }));
+		end
+	end);
+	return true, "Total: "..count.." clients";
+end
+
+function def_env.c2s:show_insecure(match_jid)
+	local print, count = self.session.print, 0;
+	show_c2s(function (jid, session)
+		if ((not match_jid) or jid:match(match_jid)) and not session.secure then
+			count = count + 1;
+			print(jid);
+		end
+	end);
+	return true, "Total: "..count.." insecure client connections";
+end
+
+function def_env.c2s:show_secure(match_jid)
+	local print, count = self.session.print, 0;
+	show_c2s(function (jid, session)
+		if ((not match_jid) or jid:match(match_jid)) and session.secure then
+			count = count + 1;
+			print(jid);
+		end
+	end);
+	return true, "Total: "..count.." secure client connections";
+end
+
+function def_env.c2s:show_tls(match_jid)
+	return self:show(match_jid, tls_info);
+end
+
+local function build_reason(text, condition)
+	if text or condition then
+		return {
+			text = text,
+			condition = condition or "undefined-condition",
+		};
+	end
+end
+
+function def_env.c2s:close(match_jid, text, condition)
+	local count = 0;
+	show_c2s(function (jid, session)
+		if jid == match_jid or jid_bare(jid) == match_jid then
+			count = count + 1;
+			session:close(build_reason(text, condition));
+		end
+	end);
+	return true, "Total: "..count.." sessions closed";
+end
+
+function def_env.c2s:closeall(text, condition)
+	local count = 0;
+	--luacheck: ignore 212/jid
+	show_c2s(function (jid, session)
+		count = count + 1;
+		session:close(build_reason(text, condition));
+	end);
+	return true, "Total: "..count.." sessions closed";
+end
+
+
+def_env.s2s = {};
+function def_env.s2s:show(match_jid, annotate)
+	local print = self.session.print;
+	annotate = annotate or session_flags;
+
+	local count_in, count_out = 0,0;
+	local s2s_list = { };
+
+	local s2s_sessions = module:shared"/*/s2s/sessions";
+	for _, session in pairs(s2s_sessions) do
+		local remotehost, localhost, direction;
+		if session.direction == "outgoing" then
+			direction = "->";
+			count_out = count_out + 1;
+			remotehost, localhost = session.to_host or "?", session.from_host or "?";
+		else
+			direction = "<-";
+			count_in = count_in + 1;
+			remotehost, localhost = session.from_host or "?", session.to_host or "?";
+		end
+		local sess_lines = { l = localhost, r = remotehost,
+			annotate(session, { "", direction, remotehost or "?" })};
+
+		if (not match_jid) or remotehost:match(match_jid) or localhost:match(match_jid) then
+			table.insert(s2s_list, sess_lines);
+			-- luacheck: ignore 421/print
+			local print = function (s) table.insert(sess_lines, "        "..s); end
+			if session.sendq then
+				print("There are "..#session.sendq.." queued outgoing stanzas for this connection");
+			end
+			if session.type == "s2sout_unauthed" then
+				if session.connecting then
+					print("Connection not yet established");
+					if not session.srv_hosts then
+						if not session.conn then
+							print("We do not yet have a DNS answer for this host's SRV records");
+						else
+							print("This host has no SRV records, using A record instead");
+						end
+					elseif session.srv_choice then
+						print("We are on SRV record "..session.srv_choice.." of "..#session.srv_hosts);
+						local srv_choice = session.srv_hosts[session.srv_choice];
+						print("Using "..(srv_choice.target or ".")..":"..(srv_choice.port or 5269));
+					end
+				elseif session.notopen then
+					print("The <stream> has not yet been opened");
+				elseif not session.dialback_key then
+					print("Dialback has not been initiated yet");
+				elseif session.dialback_key then
+					print("Dialback has been requested, but no result received");
+				end
+			end
+			if session.type == "s2sin_unauthed" then
+				print("Connection not yet authenticated");
+			elseif session.type == "s2sin" then
+				for name in pairs(session.hosts) do
+					if name ~= session.from_host then
+						print("also hosts "..tostring(name));
+					end
+				end
+			end
+		end
+	end
+
+	-- Sort by local host, then remote host
+	table.sort(s2s_list, function(a,b)
+		if a.l == b.l then return _sort_hosts(a.r, b.r); end
+		return _sort_hosts(a.l, b.l);
+	end);
+	local lasthost;
+	for _, sess_lines in ipairs(s2s_list) do
+		if sess_lines.l ~= lasthost then print(sess_lines.l); lasthost=sess_lines.l end
+		for _, line in ipairs(sess_lines) do print(line); end
+	end
+	return true, "Total: "..count_out.." outgoing, "..count_in.." incoming connections";
+end
+
+function def_env.s2s:show_tls(match_jid)
+	return self:show(match_jid, tls_info);
+end
+
+local function print_subject(print, subject)
+	for _, entry in ipairs(subject) do
+		print(
+			("    %s: %q"):format(
+				entry.name or entry.oid,
+				entry.value:gsub("[\r\n%z%c]", " ")
+			)
+		);
+	end
+end
+
+-- As much as it pains me to use the 0-based depths that OpenSSL does,
+-- I think there's going to be more confusion among operators if we
+-- break from that.
+local function print_errors(print, errors)
+	for depth, t in pairs(errors) do
+		print(
+			("    %d: %s"):format(
+				depth-1,
+				table.concat(t, "\n|        ")
+			)
+		);
+	end
+end
+
+function def_env.s2s:showcert(domain)
+	local print = self.session.print;
+	local s2s_sessions = module:shared"/*/s2s/sessions";
+	local domain_sessions = set.new(array.collect(values(s2s_sessions)))
+		/function(session) return (session.to_host == domain or session.from_host == domain) and session or nil; end;
+	local cert_set = {};
+	for session in domain_sessions do
+		local conn = session.conn;
+		conn = conn and conn:socket();
+		if not conn.getpeerchain then
+			if conn.dohandshake then
+				error("This version of LuaSec does not support certificate viewing");
+			end
+		else
+			local cert = conn:getpeercertificate();
+			if cert then
+				local certs = conn:getpeerchain();
+				local digest = cert:digest("sha1");
+				if not cert_set[digest] then
+					local chain_valid, chain_errors = conn:getpeerverification();
+					cert_set[digest] = {
+						{
+						  from = session.from_host,
+						  to = session.to_host,
+						  direction = session.direction
+						};
+						chain_valid = chain_valid;
+						chain_errors = chain_errors;
+						certs = certs;
+					};
+				else
+					table.insert(cert_set[digest], {
+						from = session.from_host,
+						to = session.to_host,
+						direction = session.direction
+					});
+				end
+			end
+		end
+	end
+	local domain_certs = array.collect(values(cert_set));
+	-- Phew. We now have a array of unique certificates presented by domain.
+	local n_certs = #domain_certs;
+
+	if n_certs == 0 then
+		return "No certificates found for "..domain;
+	end
+
+	local function _capitalize_and_colon(byte)
+		return string.upper(byte)..":";
+	end
+	local function pretty_fingerprint(hash)
+		return hash:gsub("..", _capitalize_and_colon):sub(1, -2);
+	end
+
+	for cert_info in values(domain_certs) do
+		local certs = cert_info.certs;
+		local cert = certs[1];
+		print("---")
+		print("Fingerprint (SHA1): "..pretty_fingerprint(cert:digest("sha1")));
+		print("");
+		local n_streams = #cert_info;
+		print("Currently used on "..n_streams.." stream"..(n_streams==1 and "" or "s")..":");
+		for _, stream in ipairs(cert_info) do
+			if stream.direction == "incoming" then
+				print("    "..stream.to.." <- "..stream.from);
+			else
+				print("    "..stream.from.." -> "..stream.to);
+			end
+		end
+		print("");
+		local chain_valid, errors = cert_info.chain_valid, cert_info.chain_errors;
+		local valid_identity = cert_verify_identity(domain, "xmpp-server", cert);
+		if chain_valid then
+			print("Trusted certificate: Yes");
+		else
+			print("Trusted certificate: No");
+			print_errors(print, errors);
+		end
+		print("");
+		print("Issuer: ");
+		print_subject(print, cert:issuer());
+		print("");
+		print("Valid for "..domain..": "..(valid_identity and "Yes" or "No"));
+		print("Subject:");
+		print_subject(print, cert:subject());
+	end
+	print("---");
+	return ("Showing "..n_certs.." certificate"
+		..(n_certs==1 and "" or "s")
+		.." presented by "..domain..".");
+end
+
+function def_env.s2s:close(from, to, text, condition)
+	local print, count = self.session.print, 0;
+	local s2s_sessions = module:shared"/*/s2s/sessions";
+
+	local match_id;
+	if from and not to then
+		match_id, from = from, nil;
+	elseif not to then
+		return false, "Syntax: s2s:close('from', 'to') - Closes all s2s sessions from 'from' to 'to'";
+	elseif from == to then
+		return false, "Both from and to are the same... you can't do that :)";
+	end
+
+	for _, session in pairs(s2s_sessions) do
+		local id = session.id or (session.type..tostring(session):match("[a-f0-9]+$"));
+		if (match_id and match_id == id)
+		or (session.from_host == from and session.to_host == to) then
+			print(("Closing connection from %s to %s [%s]"):format(session.from_host, session.to_host, id));
+			(session.close or s2smanager.destroy_session)(session, build_reason(text, condition));
+			count = count + 1 ;
+		end
+	end
+	return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s");
+end
+
+function def_env.s2s:closeall(host, text, condition)
+	local count = 0;
+	local s2s_sessions = module:shared"/*/s2s/sessions";
+	for _,session in pairs(s2s_sessions) do
+		if not host or session.from_host == host or session.to_host == host then
+			session:close(build_reason(text, condition));
+			count = count + 1;
+		end
+	end
+	if count == 0 then return false, "No sessions to close.";
+	else return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s"); end
+end
+
+def_env.host = {}; def_env.hosts = def_env.host;
+
+function def_env.host:activate(hostname, config)
+	return hostmanager.activate(hostname, config);
+end
+function def_env.host:deactivate(hostname, reason)
+	return hostmanager.deactivate(hostname, reason);
+end
+
+function def_env.host:list()
+	local print = self.session.print;
+	local i = 0;
+	local type;
+	for host, host_session in iterators.sorted_pairs(prosody.hosts, _sort_hosts) do
+		i = i + 1;
+		type = host_session.type;
+		if type == "local" then
+			print(host);
+		else
+			type = module:context(host):get_option_string("component_module", type);
+			if type ~= "component" then
+				type = type .. " component";
+			end
+			print(("%s (%s)"):format(host, type));
+		end
+	end
+	return true, i.." hosts";
+end
+
+def_env.port = {};
+
+function def_env.port:list()
+	local print = self.session.print;
+	local services = portmanager.get_active_services().data;
+	local n_services, n_ports = 0, 0;
+	for service, interfaces in iterators.sorted_pairs(services) do
+		n_services = n_services + 1;
+		local ports_list = {};
+		for interface, ports in pairs(interfaces) do
+			for port in pairs(ports) do
+				table.insert(ports_list, "["..interface.."]:"..port);
+			end
+		end
+		n_ports = n_ports + #ports_list;
+		print(service..": "..table.concat(ports_list, ", "));
+	end
+	return true, n_services.." services listening on "..n_ports.." ports";
+end
+
+function def_env.port:close(close_port, close_interface)
+	close_port = assert(tonumber(close_port), "Invalid port number");
+	local n_closed = 0;
+	local services = portmanager.get_active_services().data;
+	for service, interfaces in pairs(services) do -- luacheck: ignore 213
+		for interface, ports in pairs(interfaces) do
+			if not close_interface or close_interface == interface then
+				if ports[close_port] then
+					self.session.print("Closing ["..interface.."]:"..close_port.."...");
+					local ok, err = portmanager.close(interface, close_port)
+					if not ok then
+						self.session.print("Failed to close "..interface.." "..close_port..": "..err);
+					else
+						n_closed = n_closed + 1;
+					end
+				end
+			end
+		end
+	end
+	return true, "Closed "..n_closed.." ports";
+end
+
+def_env.muc = {};
+
+local console_room_mt = {
+	__index = function (self, k) return self.room[k]; end;
+	__tostring = function (self)
+		return "MUC room <"..self.room.jid..">";
+	end;
+};
+
+local function check_muc(jid)
+	local room_name, host = jid_split(jid);
+	if not prosody.hosts[host] then
+		return nil, "No such host: "..host;
+	elseif not prosody.hosts[host].modules.muc then
+		return nil, "Host '"..host.."' is not a MUC service";
+	end
+	return room_name, host;
+end
+
+function def_env.muc:create(room_jid, config)
+	local room_name, host = check_muc(room_jid);
+	if not room_name then
+		return room_name, host;
+	end
+	if not room_name then return nil, host end
+	if config ~= nil and type(config) ~= "table" then return nil, "Config must be a table"; end
+	if prosody.hosts[host].modules.muc.get_room_from_jid(room_jid) then return nil, "Room exists already" end
+	return prosody.hosts[host].modules.muc.create_room(room_jid, config);
+end
+
+function def_env.muc:room(room_jid)
+	local room_name, host = check_muc(room_jid);
+	if not room_name then
+		return room_name, host;
+	end
+	local room_obj = prosody.hosts[host].modules.muc.get_room_from_jid(room_jid);
+	if not room_obj then
+		return nil, "No such room: "..room_jid;
+	end
+	return setmetatable({ room = room_obj }, console_room_mt);
+end
+
+function def_env.muc:list(host)
+	local host_session = prosody.hosts[host];
+	if not host_session or not host_session.modules.muc then
+		return nil, "Please supply the address of a local MUC component";
+	end
+	local print = self.session.print;
+	local c = 0;
+	for room in host_session.modules.muc.each_room() do
+		print(room.jid);
+		c = c + 1;
+	end
+	return true, c.." rooms";
+end
+
+local um = require"core.usermanager";
+
+def_env.user = {};
+function def_env.user:create(jid, password)
+	local username, host = jid_split(jid);
+	if not prosody.hosts[host] then
+		return nil, "No such host: "..host;
+	elseif um.user_exists(username, host) then
+		return nil, "User exists";
+	end
+	local ok, err = um.create_user(username, password, host);
+	if ok then
+		return true, "User created";
+	else
+		return nil, "Could not create user: "..err;
+	end
+end
+
+function def_env.user:delete(jid)
+	local username, host = jid_split(jid);
+	if not prosody.hosts[host] then
+		return nil, "No such host: "..host;
+	elseif not um.user_exists(username, host) then
+		return nil, "No such user";
+	end
+	local ok, err = um.delete_user(username, host);
+	if ok then
+		return true, "User deleted";
+	else
+		return nil, "Could not delete user: "..err;
+	end
+end
+
+function def_env.user:password(jid, password)
+	local username, host = jid_split(jid);
+	if not prosody.hosts[host] then
+		return nil, "No such host: "..host;
+	elseif not um.user_exists(username, host) then
+		return nil, "No such user";
+	end
+	local ok, err = um.set_password(username, password, host, nil);
+	if ok then
+		return true, "User password changed";
+	else
+		return nil, "Could not change password for user: "..err;
+	end
+end
+
+function def_env.user:list(host, pat)
+	if not host then
+		return nil, "No host given";
+	elseif not prosody.hosts[host] then
+		return nil, "No such host";
+	end
+	local print = self.session.print;
+	local total, matches = 0, 0;
+	for user in um.users(host) do
+		if not pat or user:match(pat) then
+			print(user.."@"..host);
+			matches = matches + 1;
+		end
+		total = total + 1;
+	end
+	return true, "Showing "..(pat and (matches.." of ") or "all " )..total.." users";
+end
+
+def_env.xmpp = {};
+
+local new_id = require "util.id".medium;
+function def_env.xmpp:ping(localhost, remotehost, timeout)
+	localhost = select(2, jid_split(localhost));
+	remotehost = select(2, jid_split(remotehost));
+	if not localhost then
+		return nil, "Invalid sender hostname";
+	elseif not prosody.hosts[localhost] then
+		return nil, "No such local host";
+	end
+	if not remotehost then
+		return nil, "Invalid destination hostname";
+	elseif prosody.hosts[remotehost] then
+		return nil, "Both hosts are local";
+	end
+	local iq = st.iq{ from=localhost, to=remotehost, type="get", id=new_id()}
+			:tag("ping", {xmlns="urn:xmpp:ping"});
+	local time_start = time.now();
+	local ret, err = async.wait_for(module:context(localhost):send_iq(iq, nil, timeout));
+	if ret then
+		return true, ("pong from %s in %gs"):format(ret.stanza.attr.from, time.now() - time_start);
+	else
+		return false, tostring(err);
+	end
+end
+
+def_env.dns = {};
+local adns = require"net.adns";
+
+local function get_resolver(session)
+	local resolver = session.dns_resolver;
+	if not resolver then
+		resolver = adns.resolver();
+		session.dns_resolver = resolver;
+	end
+	return resolver;
+end
+
+function def_env.dns:lookup(name, typ, class)
+	local resolver = get_resolver(self.session);
+	local ret, err = async.wait_for(resolver:lookup_promise(name, typ, class));
+	if ret then
+		return true, ret;
+	elseif err then
+		return false, err;
+	end
+end
+
+function def_env.dns:addnameserver(...)
+	local resolver = get_resolver(self.session);
+	resolver._resolver:addnameserver(...)
+	return true
+end
+
+function def_env.dns:setnameserver(...)
+	local resolver = get_resolver(self.session);
+	resolver._resolver:setnameserver(...)
+	return true
+end
+
+function def_env.dns:purge()
+	local resolver = get_resolver(self.session);
+	resolver._resolver:purge()
+	return true
+end
+
+function def_env.dns:cache()
+	local resolver = get_resolver(self.session);
+	return true, "Cache:\n"..tostring(resolver._resolver.cache)
+end
+
+def_env.http = {};
+
+function def_env.http:list(hosts)
+	local print = self.session.print;
+
+	for host in get_hosts_set(hosts) do
+		local http_apps = modulemanager.get_items("http-provider", host);
+		if #http_apps > 0 then
+			local http_host = module:context(host):get_option_string("http_host");
+			print("HTTP endpoints on "..host..(http_host and (" (using "..http_host.."):") or ":"));
+			for _, provider in ipairs(http_apps) do
+				local url = module:context(host):http_url(provider.name, provider.default_path);
+				print("", url);
+			end
+			print("");
+		end
+	end
+
+	local default_host = module:get_option_string("http_default_host");
+	if not default_host then
+		print("HTTP requests to unknown hosts will return 404 Not Found");
+	else
+		print("HTTP requests to unknown hosts will be handled by "..default_host);
+	end
+	return true;
+end
+
+def_env.debug = {};
+
+function def_env.debug:logevents(host)
+	helpers.log_host_events(host);
+	return true;
+end
+
+function def_env.debug:events(host, event)
+	local events_obj;
+	if host and host ~= "*" then
+		if host == "http" then
+			events_obj = require "net.http.server"._events;
+		elseif not prosody.hosts[host] then
+			return false, "Unknown host: "..host;
+		else
+			events_obj = prosody.hosts[host].events;
+		end
+	else
+		events_obj = prosody.events;
+	end
+	return true, helpers.show_events(events_obj, event);
+end
+
+function def_env.debug:timers()
+	local print = self.session.print;
+	local add_task = require"util.timer".add_task;
+	local h, params = add_task.h, add_task.params;
+	local function normalize_time(t)
+		return t;
+	end
+	local function format_time(t)
+		return os.date("%F %T", math.floor(normalize_time(t)));
+	end
+	if h then
+		print("-- util.timer");
+	elseif server.timer then
+		print("-- net.server.timer");
+		h = server.timer.add_task.timers;
+		normalize_time = server.timer.to_absolute_time or normalize_time;
+	end
+	if h then
+		for i, id in ipairs(h.ids) do
+			local t, cb = h.priorities[i], h.items[id];
+			if not params then
+				local param = cb.param;
+				if param then
+					cb = param.callback;
+				else
+					cb = cb.timer_callback or cb;
+				end
+			elseif params[id] then
+				cb = params[id].callback or cb;
+			end
+			print(format_time(t), cb);
+		end
+	end
+	if server.event_base then
+		local count = 0;
+		for _, v in pairs(debug.getregistry()) do
+			if type(v) == "function" and v.callback and v.callback == add_task._on_timer then
+				count = count + 1;
+			end
+		end
+		print(count .. " libevent callbacks");
+	end
+	if h then
+		local next_time = h:peek();
+		if next_time then
+			return true, ("Next event at %s (in %.6fs)"):format(format_time(next_time), normalize_time(next_time) - time.now());
+		end
+	end
+	return true;
+end
+
+-- COMPAT: debug:timers() was timer:info() for some time in trunk
+def_env.timer = { info = def_env.debug.timers };
+
+def_env.stats = {};
+
+local short_units = {
+	seconds = "s",
+	bytes = "B",
+};
+
+local function format_stat(type, unit, value, ref_value)
+	ref_value = ref_value or value;
+	--do return tostring(value) end
+	if not unit then
+		if type == "duration" then
+			unit = "seconds"
+		elseif type == "size" then
+			unit = "bytes";
+		elseif type == "rate" then
+			unit = " events/sec"
+			if ref_value < 0.9 then
+				unit = "events/min"
+				value = value*60;
+				if ref_value < 0.6/60 then
+					unit = "events/h"
+					value = value*60;
+				end
+			end
+			return ("%.3g %s"):format(value, unit);
+		end
+	end
+	return format_number(value, short_units[unit] or unit or "", unit == "bytes" and 'b' or nil);
+end
+
+local stats_methods = {};
+function stats_methods:bounds(_lower, _upper)
+	for _, stat_info in ipairs(self) do
+		local data = stat_info[4];
+		if data then
+			local lower = _lower or data.min;
+			local upper = _upper or data.max;
+			local new_data = {
+				min = lower;
+				max = upper;
+				samples = {};
+				sample_count = 0;
+				count = data.count;
+				units = data.units;
+			};
+			local sum = 0;
+			for _, v in ipairs(data.samples) do
+				if v > upper then
+					break;
+				elseif v>=lower then
+					table.insert(new_data.samples, v);
+					sum = sum + v;
+				end
+			end
+			new_data.sample_count = #new_data.samples;
+			stat_info[4] = new_data;
+			stat_info[3] = sum/new_data.sample_count;
+		end
+	end
+	return self;
+end
+
+function stats_methods:trim(lower, upper)
+	upper = upper or (100-lower);
+	local statistics = require "util.statistics";
+	for _, stat_info in ipairs(self) do
+		-- Strip outliers
+		local data = stat_info[4];
+		if data then
+			local new_data = {
+				min = statistics.get_percentile(data, lower);
+				max = statistics.get_percentile(data, upper);
+				samples = {};
+				sample_count = 0;
+				count = data.count;
+				units = data.units;
+			};
+			local sum = 0;
+			for _, v in ipairs(data.samples) do
+				if v > new_data.max then
+					break;
+				elseif v>=new_data.min then
+					table.insert(new_data.samples, v);
+					sum = sum + v;
+				end
+			end
+			new_data.sample_count = #new_data.samples;
+			stat_info[4] = new_data;
+			stat_info[3] = sum/new_data.sample_count;
+		end
+	end
+	return self;
+end
+
+function stats_methods:max(upper)
+	return self:bounds(nil, upper);
+end
+
+function stats_methods:min(lower)
+	return self:bounds(lower, nil);
+end
+
+function stats_methods:summary()
+	local statistics = require "util.statistics";
+	for _, stat_info in ipairs(self) do
+		local type, value, data = stat_info[2], stat_info[3], stat_info[4];
+		if data and data.samples then
+			table.insert(stat_info.output, string.format("Count: %d (%d captured)",
+				data.count,
+				data.sample_count
+			));
+			table.insert(stat_info.output, string.format("Min: %s  Mean: %s  Max: %s",
+				format_stat(type, data.units, data.min),
+				format_stat(type, data.units, value),
+				format_stat(type, data.units, data.max)
+			));
+			table.insert(stat_info.output, string.format("Q1: %s  Median: %s  Q3: %s",
+				format_stat(type, data.units, statistics.get_percentile(data, 25)),
+				format_stat(type, data.units, statistics.get_percentile(data, 50)),
+				format_stat(type, data.units, statistics.get_percentile(data, 75))
+			));
+		end
+	end
+	return self;
+end
+
+function stats_methods:cfgraph()
+	for _, stat_info in ipairs(self) do
+		local name, type, value, data = unpack(stat_info, 1, 4); -- luacheck: ignore 211
+		local function print(s)
+			table.insert(stat_info.output, s);
+		end
+
+		if data and data.sample_count and data.sample_count > 0 then
+			local raw_histogram = require "util.statistics".get_histogram(data);
+
+			local graph_width, graph_height = 50, 10;
+			local eighth_chars = "   ▁▂▃▄▅▆▇█";
+
+			local range = data.max - data.min;
+
+			if range > 0 then
+				local x_scaling = #raw_histogram/graph_width;
+				local histogram = {};
+				for i = 1, graph_width do
+					histogram[i] = math.max(raw_histogram[i*x_scaling-1] or 0, raw_histogram[i*x_scaling] or 0);
+				end
+
+				print("");
+				print(("_"):rep(52)..format_stat(type, data.units, data.max));
+				for row = graph_height, 1, -1 do
+					local row_chars = {};
+					local min_eighths, max_eighths = 8, 0;
+					for i = 1, #histogram do
+						local char_eighths = math.ceil(math.max(math.min((graph_height/(data.max/histogram[i]))-(row-1), 1), 0)*8);
+						if char_eighths < min_eighths then
+							min_eighths = char_eighths;
+						end
+						if char_eighths > max_eighths then
+							max_eighths = char_eighths;
+						end
+						if char_eighths == 0 then
+							row_chars[i] = "-";
+						else
+							local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
+							row_chars[i] = char;
+						end
+					end
+					print(table.concat(row_chars).."|-"..format_stat(type, data.units, data.max/(graph_height/(row-0.5))));
+				end
+				print(("\\    "):rep(11));
+				local x_labels = {};
+				for i = 1, 11 do
+					local s = ("%-4s"):format((i-1)*10);
+					if #s > 4 then
+						s = s:sub(1, 3).."…";
+					end
+					x_labels[i] = s;
+				end
+				print(" "..table.concat(x_labels, " "));
+				local units = "%";
+				local margin = math.floor((graph_width-#units)/2);
+				print((" "):rep(margin)..units);
+			else
+				print("[range too small to graph]");
+			end
+			print("");
+		end
+	end
+	return self;
+end
+
+function stats_methods:histogram()
+	for _, stat_info in ipairs(self) do
+		local name, type, value, data = unpack(stat_info, 1, 4); -- luacheck: ignore 211
+		local function print(s)
+			table.insert(stat_info.output, s);
+		end
+
+		if not data then
+			print("[no data]");
+			return self;
+		elseif not data.sample_count then
+			print("[not a sampled metric type]");
+			return self;
+		end
+
+		local graph_width, graph_height = 50, 10;
+		local eighth_chars = "   ▁▂▃▄▅▆▇█";
+
+		local range = data.max - data.min;
+
+		if range > 0 then
+			local n_buckets = graph_width;
+
+			local histogram = {};
+			for i = 1, n_buckets do
+				histogram[i] = 0;
+			end
+			local max_bin_samples = 0;
+			for _, d in ipairs(data.samples) do
+				local bucket = math.floor(1+(n_buckets-1)/(range/(d-data.min)));
+				histogram[bucket] = histogram[bucket] + 1;
+				if histogram[bucket] > max_bin_samples then
+					max_bin_samples = histogram[bucket];
+				end
+			end
+
+			print("");
+			print(("_"):rep(52)..max_bin_samples);
+			for row = graph_height, 1, -1 do
+				local row_chars = {};
+				local min_eighths, max_eighths = 8, 0;
+				for i = 1, #histogram do
+					local char_eighths = math.ceil(math.max(math.min((graph_height/(max_bin_samples/histogram[i]))-(row-1), 1), 0)*8);
+					if char_eighths < min_eighths then
+						min_eighths = char_eighths;
+					end
+					if char_eighths > max_eighths then
+						max_eighths = char_eighths;
+					end
+					if char_eighths == 0 then
+						row_chars[i] = "-";
+					else
+						local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
+						row_chars[i] = char;
+					end
+				end
+				print(table.concat(row_chars).."|-"..math.ceil((max_bin_samples/graph_height)*(row-0.5)));
+			end
+			print(("\\    "):rep(11));
+			local x_labels = {};
+			for i = 1, 11 do
+				local s = ("%-4s"):format(format_stat(type, data.units, data.min+range*i/11, data.min):match("^%S+"));
+				if #s > 4 then
+					s = s:sub(1, 3).."…";
+				end
+				x_labels[i] = s;
+			end
+			print(" "..table.concat(x_labels, " "));
+			local units = format_stat(type, data.units, data.min):match("%s+(.+)$") or data.units or "";
+			local margin = math.floor((graph_width-#units)/2);
+			print((" "):rep(margin)..units);
+		else
+			print("[range too small to graph]");
+		end
+		print("");
+	end
+	return self;
+end
+
+local function stats_tostring(stats)
+	local print = stats.session.print;
+	for _, stat_info in ipairs(stats) do
+		if #stat_info.output > 0 then
+			print("\n#"..stat_info[1]);
+			print("");
+			for _, v in ipairs(stat_info.output) do
+				print(v);
+			end
+			print("");
+		else
+			print(("%-50s %s"):format(stat_info[1], format_stat(stat_info[2], (stat_info[4] or {}).units, stat_info[3])));
+		end
+	end
+	return #stats.." statistics displayed";
+end
+
+local stats_mt = {__index = stats_methods, __tostring = stats_tostring }
+local function new_stats_context(self)
+	return setmetatable({ session = self.session, stats = true }, stats_mt);
+end
+
+function def_env.stats:show(filter)
+	-- luacheck: ignore 211/changed
+	local stats, changed, extra = require "core.statsmanager".get_stats();
+	local available, displayed = 0, 0;
+	local displayed_stats = new_stats_context(self);
+	for name, value in iterators.sorted_pairs(stats) do
+		available = available + 1;
+		if not filter or name:match(filter) then
+			displayed = displayed + 1;
+			local type = name:match(":(%a+)$");
+			table.insert(displayed_stats, {
+				name, type, value, extra[name];
+				output = {};
+			});
+		end
+	end
+	return displayed_stats;
+end
+
+
+
+-------------
+
+function printbanner(session)
+	local option = module:get_option_string("console_banner", "full");
+	if option == "full" or option == "graphic" then
+		session.print [[
+                   ____                \   /     _
+                    |  _ \ _ __ ___  ___  _-_   __| |_   _
+                    | |_) | '__/ _ \/ __|/ _ \ / _` | | | |
+                    |  __/| | | (_) \__ \ |_| | (_| | |_| |
+                    |_|   |_|  \___/|___/\___/ \__,_|\__, |
+                    A study in simplicity            |___/
+
+]]
+	end
+	if option == "short" or option == "full" then
+	session.print("Welcome to the Prosody administration console. For a list of commands, type: help");
+	session.print("You may find more help on using this console in our online documentation at ");
+	session.print("https://prosody.im/doc/console\n");
+	end
+	if option ~= "short" and option ~= "full" and option ~= "graphic" then
+		session.print(option);
+	end
+end
diff --git a/plugins/mod_admin_socket.lua b/plugins/mod_admin_socket.lua
new file mode 100644
index 00000000..b197adae
--- /dev/null
+++ b/plugins/mod_admin_socket.lua
@@ -0,0 +1,73 @@
+module:set_global();
+
+local have_unix, unix = pcall(require, "socket.unix");
+
+if not have_unix or type(unix) ~= "table" then
+	module:log_status("error", "LuaSocket unix socket support not available or incompatible, ensure it is up to date");
+	return;
+end
+
+local server = require "net.server";
+
+local adminstream = require "util.adminstream";
+
+local socket_path = module:get_option_path("admin_socket", "prosody.sock", "data");
+
+local sessions = module:shared("sessions");
+
+local function fire_admin_event(session, stanza)
+	local event_data = {
+		origin = session, stanza = stanza;
+	};
+	local event_name;
+	if stanza.attr.xmlns then
+		event_name = "admin/"..stanza.attr.xmlns..":"..stanza.name;
+	else
+		event_name = "admin/"..stanza.name;
+	end
+	module:log("debug", "Firing %s", event_name);
+	return module:fire_event(event_name, event_data);
+end
+
+module:hook("server-stopping", function ()
+	for _, session in pairs(sessions) do
+		session:close("system-shutdown");
+	end
+	os.remove(socket_path);
+end);
+
+--- Unix domain socket management
+
+local conn, sock;
+
+local listeners = adminstream.server(sessions, fire_admin_event).listeners;
+
+local function accept_connection()
+	module:log("debug", "accepting...");
+	local client = sock:accept();
+	if not client then return; end
+	server.wrapclient(client, "unix", 0, listeners, "*a");
+end
+
+function module.load()
+	sock = unix.stream();
+	sock:settimeout(0);
+	os.remove(socket_path);
+	assert(sock:bind(socket_path));
+	assert(sock:listen());
+	if server.wrapserver then
+		conn = server.wrapserver(sock, socket_path, 0, listeners);
+	else
+		conn = server.watchfd(sock:getfd(), accept_connection);
+	end
+end
+
+function module.unload()
+	if conn then
+		conn:close();
+	end
+	if sock then
+		sock:close();
+	end
+	os.remove(socket_path);
+end
diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua
index b0e349da..15220ec9 100644
--- a/plugins/mod_admin_telnet.lua
+++ b/plugins/mod_admin_telnet.lua
@@ -8,49 +8,68 @@
 -- luacheck: ignore 212/self
 
 module:set_global();
-
-local hostmanager = require "core.hostmanager";
-local modulemanager = require "core.modulemanager";
-local s2smanager = require "core.s2smanager";
-local portmanager = require "core.portmanager";
-local helpers = require "util.helpers";
-local server = require "net.server";
-
-local _G = _G;
-
-local prosody = _G.prosody;
+module:depends("admin_shell");
 
 local console_listener = { default_port = 5582; default_mode = "*a"; interface = "127.0.0.1" };
 
-local iterators = require "util.iterators";
-local keys, values = iterators.keys, iterators.values;
-local jid_bare, jid_split, jid_join = import("util.jid", "bare", "prepped_split", "join");
-local set, array = require "util.set", require "util.array";
-local cert_verify_identity = require "util.x509".verify_identity;
-local envload = require "util.envload".envload;
-local envloadfile = require "util.envload".envloadfile;
-local has_pposix, pposix = pcall(require, "util.pposix");
+local async = require "util.async";
+local st = require "util.stanza";
 
-local commands = module:shared("commands")
-local def_env = module:shared("env");
+local def_env = module:shared("admin_shell/env");
 local default_env_mt = { __index = def_env };
 
-local function redirect_output(target, session)
-	local env = setmetatable({ print = session.print }, { __index = function (_, k) return rawget(target, k); end });
-	env.dofile = function(name)
-		local f, err = envloadfile(name, env);
-		if not f then return f, err; end
-		return f();
-	end;
-	return env;
+local function printbanner(session)
+	local option = module:get_option_string("console_banner", "full");
+	if option == "full" or option == "graphic" then
+		session.print [[
+                   ____                \   /     _
+                    |  _ \ _ __ ___  ___  _-_   __| |_   _
+                    | |_) | '__/ _ \/ __|/ _ \ / _` | | | |
+                    |  __/| | | (_) \__ \ |_| | (_| | |_| |
+                    |_|   |_|  \___/|___/\___/ \__,_|\__, |
+                    A study in simplicity            |___/
+
+]]
+	end
+	if option == "short" or option == "full" then
+	session.print("Welcome to the Prosody administration console. For a list of commands, type: help");
+	session.print("You may find more help on using this console in our online documentation at ");
+	session.print("https://prosody.im/doc/console\n");
+	end
+	if option ~= "short" and option ~= "full" and option ~= "graphic" then
+		session.print(option);
+	end
 end
 
 console = {};
 
+local runner_callbacks = {};
+
+function runner_callbacks:ready()
+	self.data.conn:resume();
+end
+
+function runner_callbacks:waiting()
+	self.data.conn:pause();
+end
+
+function runner_callbacks:error(err)
+	module:log("error", "Traceback[telnet]: %s", err);
+
+	self.data.print("Fatal error while running command, it did not complete");
+	self.data.print("Error: "..tostring(err));
+end
+
+
 function console:new_session(conn)
 	local w = function(s) conn:write(s:gsub("\n", "\r\n")); end;
 	local session = { conn = conn;
-			send = function (t) w(tostring(t)); end;
+			send = function (t)
+				if st.is_stanza(t) and (t.name == "repl-result" or t.name == "repl-output") then
+					t = "| "..t:get_text().."\n";
+				end
+				w(tostring(t));
+			end;
 			print = function (...)
 				local t = {};
 				for i=1,select("#", ...) do
@@ -58,10 +77,16 @@ function console:new_session(conn)
 				end
 				w("| "..table.concat(t, "\t").."\n");
 			end;
+			serialize = tostring;
 			disconnect = function () conn:close(); end;
 			};
 	session.env = setmetatable({}, default_env_mt);
 
+	session.thread = async.runner(function (line)
+		console:process_line(session, line);
+		session.send(string.char(0));
+	end, runner_callbacks, session);
+
 	-- Load up environment with helper objects
 	for name, t in pairs(def_env) do
 		if type(t) == "table" then
@@ -69,69 +94,37 @@ function console:new_session(conn)
 		end
 	end
 
+	session.env.output:configure();
+
 	return session;
 end
 
 function console:process_line(session, line)
-	local useglobalenv;
-
-	if line:match("^>") then
-		line = line:gsub("^>", "");
-		useglobalenv = true;
-	elseif line == "\004" then
-		commands["bye"](session, line);
+	line = line:gsub("\r?\n$", "");
+	if line == "bye" or line == "quit" or line == "exit" or line:byte() == 4 then
+		session.print("See you!");
+		session:disconnect();
 		return;
-	else
-		local command = line:match("^%w+") or line:match("%p");
-		if commands[command] then
-			commands[command](session, line);
-			return;
-		end
-	end
-
-	session.env._ = line;
-
-	local chunkname = "=console";
-	local env = (useglobalenv and redirect_output(_G, session)) or session.env or nil
-	local chunk, err = envload("return "..line, chunkname, env);
-	if not chunk then
-		chunk, err = envload(line, chunkname, env);
-		if not chunk then
-			err = err:gsub("^%[string .-%]:%d+: ", "");
-			err = err:gsub("^:%d+: ", "");
-			err = err:gsub("'<eof>'", "the end of the line");
-			session.print("Sorry, I couldn't understand that... "..err);
-			return;
-		end
 	end
+	return module:fire_event("admin/repl-input", { origin = session, stanza = st.stanza("repl-input"):text(line) });
+end
 
-	local ranok, taskok, message = pcall(chunk);
-
-	if not (ranok or message or useglobalenv) and commands[line:lower()] then
-		commands[line:lower()](session, line);
-		return;
-	end
+local sessions = {};
 
-	if not ranok then
-		session.print("Fatal error while running command, it did not complete");
-		session.print("Error: "..taskok);
-		return;
-	end
+function module.save()
+	return { sessions = sessions }
+end
 
-	if not message then
-		session.print("Result: "..tostring(taskok));
-		return;
-	elseif (not taskok) and message then
-		session.print("Command completed with a problem");
-		session.print("Message: "..tostring(message));
-		return;
+function module.restore(data)
+	if data.sessions then
+		for conn in pairs(data.sessions) do
+			conn:setlistener(console_listener);
+			local session = console:new_session(conn);
+			sessions[conn] = session;
+		end
 	end
-
-	session.print("OK: "..tostring(message));
 end
 
-local sessions = {};
-
 function console_listener.onconnect(conn)
 	-- Handle new connection
 	local session = console:new_session(conn);
@@ -150,8 +143,7 @@ function console_listener.onincoming(conn, data)
 
 	for line in data:gmatch("[^\n]*[\n\004]") do
 		if session.closed then return end
-		console:process_line(session, line);
-		session.send(string.char(0));
+		session.thread:run(line);
 	end
 	session.partial_data = data:match("[^\n]+$");
 end
@@ -176,1382 +168,6 @@ function console_listener.ondetach(conn)
 	sessions[conn] = nil;
 end
 
--- Console commands --
--- These are simple commands, not valid standalone in Lua
-
-function commands.bye(session)
-	session.print("See you! :)");
-	session.closed = true;
-	session.disconnect();
-end
-commands.quit, commands.exit = commands.bye, commands.bye;
-
-commands["!"] = function (session, data)
-	if data:match("^!!") and session.env._ then
-		session.print("!> "..session.env._);
-		return console_listener.onincoming(session.conn, session.env._);
-	end
-	local old, new = data:match("^!(.-[^\\])!(.-)!$");
-	if old and new then
-		local ok, res = pcall(string.gsub, session.env._, old, new);
-		if not ok then
-			session.print(res)
-			return;
-		end
-		session.print("!> "..res);
-		return console_listener.onincoming(session.conn, res);
-	end
-	session.print("Sorry, not sure what you want");
-end
-
-
-function commands.help(session, data)
-	local print = session.print;
-	local section = data:match("^help (%w+)");
-	if not section then
-		print [[Commands are divided into multiple sections. For help on a particular section, ]]
-		print [[type: help SECTION (for example, 'help c2s'). Sections are: ]]
-		print [[]]
-		print [[c2s - Commands to manage local client-to-server sessions]]
-		print [[s2s - Commands to manage sessions between this server and others]]
-		print [[module - Commands to load/reload/unload modules/plugins]]
-		print [[host - Commands to activate, deactivate and list virtual hosts]]
-		print [[user - Commands to create and delete users, and change their passwords]]
-		print [[server - Uptime, version, shutting down, etc.]]
-		print [[port - Commands to manage ports the server is listening on]]
-		print [[dns - Commands to manage and inspect the internal DNS resolver]]
-		print [[config - Reloading the configuration, etc.]]
-		print [[console - Help regarding the console itself]]
-	elseif section == "c2s" then
-		print [[c2s:show(jid) - Show all client sessions with the specified JID (or all if no JID given)]]
-		print [[c2s:show_insecure() - Show all unencrypted client connections]]
-		print [[c2s:show_secure() - Show all encrypted client connections]]
-		print [[c2s:show_tls() - Show TLS cipher info for encrypted sessions]]
-		print [[c2s:close(jid) - Close all sessions for the specified JID]]
-	elseif section == "s2s" then
-		print [[s2s:show(domain) - Show all s2s connections for the given domain (or all if no domain given)]]
-		print [[s2s:show_tls(domain) - Show TLS cipher info for encrypted sessions]]
-		print [[s2s:close(from, to) - Close a connection from one domain to another]]
-		print [[s2s:closeall(host) - Close all the incoming/outgoing s2s sessions to specified host]]
-	elseif section == "module" then
-		print [[module:load(module, host) - Load the specified module on the specified host (or all hosts if none given)]]
-		print [[module:reload(module, host) - The same, but unloads and loads the module (saving state if the module supports it)]]
-		print [[module:unload(module, host) - The same, but just unloads the module from memory]]
-		print [[module:list(host) - List the modules loaded on the specified host]]
-	elseif section == "host" then
-		print [[host:activate(hostname) - Activates the specified host]]
-		print [[host:deactivate(hostname) - Disconnects all clients on this host and deactivates]]
-		print [[host:list() - List the currently-activated hosts]]
-	elseif section == "user" then
-		print [[user:create(jid, password) - Create the specified user account]]
-		print [[user:password(jid, password) - Set the password for the specified user account]]
-		print [[user:delete(jid) - Permanently remove the specified user account]]
-		print [[user:list(hostname, pattern) - List users on the specified host, optionally filtering with a pattern]]
-	elseif section == "server" then
-		print [[server:version() - Show the server's version number]]
-		print [[server:uptime() - Show how long the server has been running]]
-		print [[server:memory() - Show details about the server's memory usage]]
-		print [[server:shutdown(reason) - Shut down the server, with an optional reason to be broadcast to all connections]]
-	elseif section == "port" then
-		print [[port:list() - Lists all network ports prosody currently listens on]]
-		print [[port:close(port, interface) - Close a port]]
-	elseif section == "dns" then
-		print [[dns:lookup(name, type, class) - Do a DNS lookup]]
-		print [[dns:addnameserver(nameserver) - Add a nameserver to the list]]
-		print [[dns:setnameserver(nameserver) - Replace the list of name servers with the supplied one]]
-		print [[dns:purge() - Clear the DNS cache]]
-		print [[dns:cache() - Show cached records]]
-	elseif section == "config" then
-		print [[config:reload() - Reload the server configuration. Modules may need to be reloaded for changes to take effect.]]
-	elseif section == "console" then
-		print [[Hey! Welcome to Prosody's admin console.]]
-		print [[First thing, if you're ever wondering how to get out, simply type 'quit'.]]
-		print [[Secondly, note that we don't support the full telnet protocol yet (it's coming)]]
-		print [[so you may have trouble using the arrow keys, etc. depending on your system.]]
-		print [[]]
-		print [[For now we offer a couple of handy shortcuts:]]
-		print [[!! - Repeat the last command]]
-		print [[!old!new! - repeat the last command, but with 'old' replaced by 'new']]
-		print [[]]
-		print [[For those well-versed in Prosody's internals, or taking instruction from those who are,]]
-		print [[you can prefix a command with > to escape the console sandbox, and access everything in]]
-		print [[the running server. Great fun, but be careful not to break anything :)]]
-	end
-	print [[]]
-end
-
--- Session environment --
--- Anything in def_env will be accessible within the session as a global variable
-
---luacheck: ignore 212/self
-
-def_env.server = {};
-
-function def_env.server:insane_reload()
-	prosody.unlock_globals();
-	dofile "prosody"
-	prosody = _G.prosody;
-	return true, "Server reloaded";
-end
-
-function def_env.server:version()
-	return true, tostring(prosody.version or "unknown");
-end
-
-function def_env.server:uptime()
-	local t = os.time()-prosody.start_time;
-	local seconds = t%60;
-	t = (t - seconds)/60;
-	local minutes = t%60;
-	t = (t - minutes)/60;
-	local hours = t%24;
-	t = (t - hours)/24;
-	local days = t;
-	return true, string.format("This server has been running for %d day%s, %d hour%s and %d minute%s (since %s)",
-		days, (days ~= 1 and "s") or "", hours, (hours ~= 1 and "s") or "",
-		minutes, (minutes ~= 1 and "s") or "", os.date("%c", prosody.start_time));
-end
-
-function def_env.server:shutdown(reason)
-	prosody.shutdown(reason);
-	return true, "Shutdown initiated";
-end
-
-local function human(kb)
-	local unit = "K";
-	if kb > 1024 then
-		kb, unit = kb/1024, "M";
-	end
-	return ("%0.2f%sB"):format(kb, unit);
-end
-
-function def_env.server:memory()
-	if not has_pposix or not pposix.meminfo then
-		return true, "Lua is using "..human(collectgarbage("count"));
-	end
-	local mem, lua_mem = pposix.meminfo(), collectgarbage("count");
-	local print = self.session.print;
-	print("Process: "..human((mem.allocated+mem.allocated_mmap)/1024));
-	print("   Used: "..human(mem.used/1024).." ("..human(lua_mem).." by Lua)");
-	print("   Free: "..human(mem.unused/1024).." ("..human(mem.returnable/1024).." returnable)");
-	return true, "OK";
-end
-
-def_env.module = {};
-
-local function get_hosts_set(hosts, module)
-	if type(hosts) == "table" then
-		if hosts[1] then
-			return set.new(hosts);
-		elseif hosts._items then
-			return hosts;
-		end
-	elseif type(hosts) == "string" then
-		return set.new { hosts };
-	elseif hosts == nil then
-		local hosts_set = set.new(array.collect(keys(prosody.hosts)))
-			/ function (host) return (prosody.hosts[host].type == "local" or module and modulemanager.is_loaded(host, module)) and host or nil; end;
-		if module and modulemanager.get_module("*", module) then
-			hosts_set:add("*");
-		end
-		return hosts_set;
-	end
-end
-
-function def_env.module:load(name, hosts, config)
-	hosts = get_hosts_set(hosts);
-
-	-- Load the module for each host
-	local ok, err, count, mod = true, nil, 0;
-	for host in hosts do
-		if (not modulemanager.is_loaded(host, name)) then
-			mod, err = modulemanager.load(host, name, config);
-			if not mod then
-				ok = false;
-				if err == "global-module-already-loaded" then
-					if count > 0 then
-						ok, err, count = true, nil, 1;
-					end
-					break;
-				end
-				self.session.print(err or "Unknown error loading module");
-			else
-				count = count + 1;
-				self.session.print("Loaded for "..mod.module.host);
-			end
-		end
-	end
-
-	return ok, (ok and "Module loaded onto "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
-end
-
-function def_env.module:unload(name, hosts)
-	hosts = get_hosts_set(hosts, name);
-
-	-- Unload the module for each host
-	local ok, err, count = true, nil, 0;
-	for host in hosts do
-		if modulemanager.is_loaded(host, name) then
-			ok, err = modulemanager.unload(host, name);
-			if not ok then
-				ok = false;
-				self.session.print(err or "Unknown error unloading module");
-			else
-				count = count + 1;
-				self.session.print("Unloaded from "..host);
-			end
-		end
-	end
-	return ok, (ok and "Module unloaded from "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
-end
-
-local function _sort_hosts(a, b)
-	if a == "*" then return true
-	elseif b == "*" then return false
-	else return a < b; end
-end
-
-function def_env.module:reload(name, hosts)
-	hosts = array.collect(get_hosts_set(hosts, name)):sort(_sort_hosts)
-
-	-- Reload the module for each host
-	local ok, err, count = true, nil, 0;
-	for _, host in ipairs(hosts) do
-		if modulemanager.is_loaded(host, name) then
-			ok, err = modulemanager.reload(host, name);
-			if not ok then
-				ok = false;
-				self.session.print(err or "Unknown error reloading module");
-			else
-				count = count + 1;
-				if ok == nil then
-					ok = true;
-				end
-				self.session.print("Reloaded on "..host);
-			end
-		end
-	end
-	return ok, (ok and "Module reloaded on "..count.." host"..(count ~= 1 and "s" or "")) or ("Last error: "..tostring(err));
-end
-
-function def_env.module:list(hosts)
-	if hosts == nil then
-		hosts = array.collect(keys(prosody.hosts));
-		table.insert(hosts, 1, "*");
-	end
-	if type(hosts) == "string" then
-		hosts = { hosts };
-	end
-	if type(hosts) ~= "table" then
-		return false, "Please supply a host or a list of hosts you would like to see";
-	end
-
-	local print = self.session.print;
-	for _, host in ipairs(hosts) do
-		print((host == "*" and "Global" or host)..":");
-		local modules = array.collect(keys(modulemanager.get_modules(host) or {})):sort();
-		if #modules == 0 then
-			if prosody.hosts[host] then
-				print("    No modules loaded");
-			else
-				print("    Host not found");
-			end
-		else
-			for _, name in ipairs(modules) do
-				print("    "..name);
-			end
-		end
-	end
-end
-
-def_env.config = {};
-function def_env.config:load(filename, format)
-	local config_load = require "core.configmanager".load;
-	local ok, err = config_load(filename, format);
-	if not ok then
-		return false, err or "Unknown error loading config";
-	end
-	return true, "Config loaded";
-end
-
-function def_env.config:get(host, section, key)
-	local config_get = require "core.configmanager".get
-	return true, tostring(config_get(host, section, key));
-end
-
-function def_env.config:reload()
-	local ok, err = prosody.reload_config();
-	return ok, (ok and "Config reloaded (you may need to reload modules to take effect)") or tostring(err);
-end
-
-local function common_info(session, line)
-	if session.id then
-		line[#line+1] = "["..session.id.."]"
-	else
-		line[#line+1] = "["..session.type..(tostring(session):match("%x*$")).."]"
-	end
-end
-
-local function session_flags(session, line)
-	line = line or {};
-	common_info(session, line);
-	if session.type == "c2s" then
-		local status, priority = "unavailable", tostring(session.priority or "-");
-		if session.presence then
-			status = session.presence:get_child_text("show") or "available";
-		end
-		line[#line+1] = status.."("..priority..")";
-	end
-	if session.cert_identity_status == "valid" then
-		line[#line+1] = "(authenticated)";
-	end
-	if session.secure then
-		line[#line+1] = "(encrypted)";
-	end
-	if session.compressed then
-		line[#line+1] = "(compressed)";
-	end
-	if session.smacks then
-		line[#line+1] = "(sm)";
-	end
-	if session.ip and session.ip:match(":") then
-		line[#line+1] = "(IPv6)";
-	end
-	if session.remote then
-		line[#line+1] = "(remote)";
-	end
-	return table.concat(line, " ");
-end
-
-local function tls_info(session, line)
-	line = line or {};
-	common_info(session, line);
-	if session.secure then
-		local sock = session.conn and session.conn.socket and session.conn:socket();
-		if sock then
-			local info = sock.info and sock:info();
-			if info then
-				line[#line+1] = ("(%s with %s)"):format(info.protocol, info.cipher);
-			else
-				-- TLS session might not be ready yet
-				line[#line+1] = "(cipher info unavailable)";
-			end
-		end
-	else
-		line[#line+1] = "(insecure)";
-	end
-	return table.concat(line, " ");
-end
-
-def_env.c2s = {};
-
-local function get_jid(session)
-	if session.username then
-		return session.full_jid or jid_join(session.username, session.host, session.resource);
-	end
-
-	local conn = session.conn;
-	local ip = session.ip or "?";
-	local clientport = conn and conn:clientport() or "?";
-	local serverip = conn and conn.server and conn:server():ip() or "?";
-	local serverport = conn and conn:serverport() or "?"
-	return jid_join("["..ip.."]:"..clientport, session.host or "["..serverip.."]:"..serverport);
-end
-
-local function show_c2s(callback)
-	local c2s = array.collect(values(module:shared"/*/c2s/sessions"));
-	c2s:sort(function(a, b)
-		if a.host == b.host then
-			if a.username == b.username then
-				return (a.resource or "") > (b.resource or "");
-			end
-			return (a.username or "") > (b.username or "");
-		end
-		return (a.host or "") > (b.host or "");
-	end):map(function (session)
-		callback(get_jid(session), session)
-	end);
-end
-
-function def_env.c2s:count()
-	return true, "Total: "..  iterators.count(values(module:shared"/*/c2s/sessions")) .." clients";
-end
-
-function def_env.c2s:show(match_jid, annotate)
-	local print, count = self.session.print, 0;
-	annotate = annotate or session_flags;
-	local curr_host = false;
-	show_c2s(function (jid, session)
-		if curr_host ~= session.host then
-			curr_host = session.host;
-			print(curr_host or "(not connected to any host yet)");
-		end
-		if (not match_jid) or jid:match(match_jid) then
-			count = count + 1;
-			print(annotate(session, { "  ", jid }));
-		end
-	end);
-	return true, "Total: "..count.." clients";
-end
-
-function def_env.c2s:show_insecure(match_jid)
-	local print, count = self.session.print, 0;
-	show_c2s(function (jid, session)
-		if ((not match_jid) or jid:match(match_jid)) and not session.secure then
-			count = count + 1;
-			print(jid);
-		end
-	end);
-	return true, "Total: "..count.." insecure client connections";
-end
-
-function def_env.c2s:show_secure(match_jid)
-	local print, count = self.session.print, 0;
-	show_c2s(function (jid, session)
-		if ((not match_jid) or jid:match(match_jid)) and session.secure then
-			count = count + 1;
-			print(jid);
-		end
-	end);
-	return true, "Total: "..count.." secure client connections";
-end
-
-function def_env.c2s:show_tls(match_jid)
-	return self:show(match_jid, tls_info);
-end
-
-function def_env.c2s:close(match_jid)
-	local count = 0;
-	show_c2s(function (jid, session)
-		if jid == match_jid or jid_bare(jid) == match_jid then
-			count = count + 1;
-			session:close();
-		end
-	end);
-	return true, "Total: "..count.." sessions closed";
-end
-
-
-def_env.s2s = {};
-function def_env.s2s:show(match_jid, annotate)
-	local print = self.session.print;
-	annotate = annotate or session_flags;
-
-	local count_in, count_out = 0,0;
-	local s2s_list = { };
-
-	local s2s_sessions = module:shared"/*/s2s/sessions";
-	for _, session in pairs(s2s_sessions) do
-		local remotehost, localhost, direction;
-		if session.direction == "outgoing" then
-			direction = "->";
-			count_out = count_out + 1;
-			remotehost, localhost = session.to_host or "?", session.from_host or "?";
-		else
-			direction = "<-";
-			count_in = count_in + 1;
-			remotehost, localhost = session.from_host or "?", session.to_host or "?";
-		end
-		local sess_lines = { l = localhost, r = remotehost,
-			annotate(session, { "", direction, remotehost or "?" })};
-
-		if (not match_jid) or remotehost:match(match_jid) or localhost:match(match_jid) then
-			table.insert(s2s_list, sess_lines);
-			-- luacheck: ignore 421/print
-			local print = function (s) table.insert(sess_lines, "        "..s); end
-			if session.sendq then
-				print("There are "..#session.sendq.." queued outgoing stanzas for this connection");
-			end
-			if session.type == "s2sout_unauthed" then
-				if session.connecting then
-					print("Connection not yet established");
-					if not session.srv_hosts then
-						if not session.conn then
-							print("We do not yet have a DNS answer for this host's SRV records");
-						else
-							print("This host has no SRV records, using A record instead");
-						end
-					elseif session.srv_choice then
-						print("We are on SRV record "..session.srv_choice.." of "..#session.srv_hosts);
-						local srv_choice = session.srv_hosts[session.srv_choice];
-						print("Using "..(srv_choice.target or ".")..":"..(srv_choice.port or 5269));
-					end
-				elseif session.notopen then
-					print("The <stream> has not yet been opened");
-				elseif not session.dialback_key then
-					print("Dialback has not been initiated yet");
-				elseif session.dialback_key then
-					print("Dialback has been requested, but no result received");
-				end
-			end
-			if session.type == "s2sin_unauthed" then
-				print("Connection not yet authenticated");
-			elseif session.type == "s2sin" then
-				for name in pairs(session.hosts) do
-					if name ~= session.from_host then
-						print("also hosts "..tostring(name));
-					end
-				end
-			end
-		end
-	end
-
-	-- Sort by local host, then remote host
-	table.sort(s2s_list, function(a,b)
-		if a.l == b.l then return a.r < b.r; end
-		return a.l < b.l;
-	end);
-	local lasthost;
-	for _, sess_lines in ipairs(s2s_list) do
-		if sess_lines.l ~= lasthost then print(sess_lines.l); lasthost=sess_lines.l end
-		for _, line in ipairs(sess_lines) do print(line); end
-	end
-	return true, "Total: "..count_out.." outgoing, "..count_in.." incoming connections";
-end
-
-function def_env.s2s:show_tls(match_jid)
-	return self:show(match_jid, tls_info);
-end
-
-local function print_subject(print, subject)
-	for _, entry in ipairs(subject) do
-		print(
-			("    %s: %q"):format(
-				entry.name or entry.oid,
-				entry.value:gsub("[\r\n%z%c]", " ")
-			)
-		);
-	end
-end
-
--- As much as it pains me to use the 0-based depths that OpenSSL does,
--- I think there's going to be more confusion among operators if we
--- break from that.
-local function print_errors(print, errors)
-	for depth, t in pairs(errors) do
-		print(
-			("    %d: %s"):format(
-				depth-1,
-				table.concat(t, "\n|        ")
-			)
-		);
-	end
-end
-
-function def_env.s2s:showcert(domain)
-	local print = self.session.print;
-	local s2s_sessions = module:shared"/*/s2s/sessions";
-	local domain_sessions = set.new(array.collect(values(s2s_sessions)))
-		/function(session) return (session.to_host == domain or session.from_host == domain) and session or nil; end;
-	local cert_set = {};
-	for session in domain_sessions do
-		local conn = session.conn;
-		conn = conn and conn:socket();
-		if not conn.getpeerchain then
-			if conn.dohandshake then
-				error("This version of LuaSec does not support certificate viewing");
-			end
-		else
-			local cert = conn:getpeercertificate();
-			if cert then
-				local certs = conn:getpeerchain();
-				local digest = cert:digest("sha1");
-				if not cert_set[digest] then
-					local chain_valid, chain_errors = conn:getpeerverification();
-					cert_set[digest] = {
-						{
-						  from = session.from_host,
-						  to = session.to_host,
-						  direction = session.direction
-						};
-						chain_valid = chain_valid;
-						chain_errors = chain_errors;
-						certs = certs;
-					};
-				else
-					table.insert(cert_set[digest], {
-						from = session.from_host,
-						to = session.to_host,
-						direction = session.direction
-					});
-				end
-			end
-		end
-	end
-	local domain_certs = array.collect(values(cert_set));
-	-- Phew. We now have a array of unique certificates presented by domain.
-	local n_certs = #domain_certs;
-
-	if n_certs == 0 then
-		return "No certificates found for "..domain;
-	end
-
-	local function _capitalize_and_colon(byte)
-		return string.upper(byte)..":";
-	end
-	local function pretty_fingerprint(hash)
-		return hash:gsub("..", _capitalize_and_colon):sub(1, -2);
-	end
-
-	for cert_info in values(domain_certs) do
-		local certs = cert_info.certs;
-		local cert = certs[1];
-		print("---")
-		print("Fingerprint (SHA1): "..pretty_fingerprint(cert:digest("sha1")));
-		print("");
-		local n_streams = #cert_info;
-		print("Currently used on "..n_streams.." stream"..(n_streams==1 and "" or "s")..":");
-		for _, stream in ipairs(cert_info) do
-			if stream.direction == "incoming" then
-				print("    "..stream.to.." <- "..stream.from);
-			else
-				print("    "..stream.from.." -> "..stream.to);
-			end
-		end
-		print("");
-		local chain_valid, errors = cert_info.chain_valid, cert_info.chain_errors;
-		local valid_identity = cert_verify_identity(domain, "xmpp-server", cert);
-		if chain_valid then
-			print("Trusted certificate: Yes");
-		else
-			print("Trusted certificate: No");
-			print_errors(print, errors);
-		end
-		print("");
-		print("Issuer: ");
-		print_subject(print, cert:issuer());
-		print("");
-		print("Valid for "..domain..": "..(valid_identity and "Yes" or "No"));
-		print("Subject:");
-		print_subject(print, cert:subject());
-	end
-	print("---");
-	return ("Showing "..n_certs.." certificate"
-		..(n_certs==1 and "" or "s")
-		.." presented by "..domain..".");
-end
-
-function def_env.s2s:close(from, to)
-	local print, count = self.session.print, 0;
-	local s2s_sessions = module:shared"/*/s2s/sessions";
-
-	local match_id;
-	if from and not to then
-		match_id, from = from, nil;
-	elseif not to then
-		return false, "Syntax: s2s:close('from', 'to') - Closes all s2s sessions from 'from' to 'to'";
-	elseif from == to then
-		return false, "Both from and to are the same... you can't do that :)";
-	end
-
-	for _, session in pairs(s2s_sessions) do
-		local id = session.type..tostring(session):match("[a-f0-9]+$");
-		if (match_id and match_id == id)
-		or (session.from_host == from and session.to_host == to) then
-			print(("Closing connection from %s to %s [%s]"):format(session.from_host, session.to_host, id));
-			(session.close or s2smanager.destroy_session)(session);
-			count = count + 1 ;
-		end
-	end
-	return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s");
-end
-
-function def_env.s2s:closeall(host)
-	local count = 0;
-	local s2s_sessions = module:shared"/*/s2s/sessions";
-	for _,session in pairs(s2s_sessions) do
-		if not host or session.from_host == host or session.to_host == host then
-			session:close();
-			count = count + 1;
-		end
-	end
-	if count == 0 then return false, "No sessions to close.";
-	else return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s"); end
-end
-
-def_env.host = {}; def_env.hosts = def_env.host;
-
-function def_env.host:activate(hostname, config)
-	return hostmanager.activate(hostname, config);
-end
-function def_env.host:deactivate(hostname, reason)
-	return hostmanager.deactivate(hostname, reason);
-end
-
-function def_env.host:list()
-	local print = self.session.print;
-	local i = 0;
-	local type;
-	for host, host_session in iterators.sorted_pairs(prosody.hosts) do
-		i = i + 1;
-		type = host_session.type;
-		if type == "local" then
-			print(host);
-		else
-			type = module:context(host):get_option_string("component_module", type);
-			if type ~= "component" then
-				type = type .. " component";
-			end
-			print(("%s (%s)"):format(host, type));
-		end
-	end
-	return true, i.." hosts";
-end
-
-def_env.port = {};
-
-function def_env.port:list()
-	local print = self.session.print;
-	local services = portmanager.get_active_services().data;
-	local n_services, n_ports = 0, 0;
-	for service, interfaces in iterators.sorted_pairs(services) do
-		n_services = n_services + 1;
-		local ports_list = {};
-		for interface, ports in pairs(interfaces) do
-			for port in pairs(ports) do
-				table.insert(ports_list, "["..interface.."]:"..port);
-			end
-		end
-		n_ports = n_ports + #ports_list;
-		print(service..": "..table.concat(ports_list, ", "));
-	end
-	return true, n_services.." services listening on "..n_ports.." ports";
-end
-
-function def_env.port:close(close_port, close_interface)
-	close_port = assert(tonumber(close_port), "Invalid port number");
-	local n_closed = 0;
-	local services = portmanager.get_active_services().data;
-	for service, interfaces in pairs(services) do -- luacheck: ignore 213
-		for interface, ports in pairs(interfaces) do
-			if not close_interface or close_interface == interface then
-				if ports[close_port] then
-					self.session.print("Closing ["..interface.."]:"..close_port.."...");
-					local ok, err = portmanager.close(interface, close_port)
-					if not ok then
-						self.session.print("Failed to close "..interface.." "..close_port..": "..err);
-					else
-						n_closed = n_closed + 1;
-					end
-				end
-			end
-		end
-	end
-	return true, "Closed "..n_closed.." ports";
-end
-
-def_env.muc = {};
-
-local console_room_mt = {
-	__index = function (self, k) return self.room[k]; end;
-	__tostring = function (self)
-		return "MUC room <"..self.room.jid..">";
-	end;
-};
-
-local function check_muc(jid)
-	local room_name, host = jid_split(jid);
-	if not prosody.hosts[host] then
-		return nil, "No such host: "..host;
-	elseif not prosody.hosts[host].modules.muc then
-		return nil, "Host '"..host.."' is not a MUC service";
-	end
-	return room_name, host;
-end
-
-function def_env.muc:create(room_jid, config)
-	local room_name, host = check_muc(room_jid);
-	if not room_name then
-		return room_name, host;
-	end
-	if not room_name then return nil, host end
-	if config ~= nil and type(config) ~= "table" then return nil, "Config must be a table"; end
-	if prosody.hosts[host].modules.muc.get_room_from_jid(room_jid) then return nil, "Room exists already" end
-	return prosody.hosts[host].modules.muc.create_room(room_jid, config);
-end
-
-function def_env.muc:room(room_jid)
-	local room_name, host = check_muc(room_jid);
-	if not room_name then
-		return room_name, host;
-	end
-	local room_obj = prosody.hosts[host].modules.muc.get_room_from_jid(room_jid);
-	if not room_obj then
-		return nil, "No such room: "..room_jid;
-	end
-	return setmetatable({ room = room_obj }, console_room_mt);
-end
-
-function def_env.muc:list(host)
-	local host_session = prosody.hosts[host];
-	if not host_session or not host_session.modules.muc then
-		return nil, "Please supply the address of a local MUC component";
-	end
-	local print = self.session.print;
-	local c = 0;
-	for room in host_session.modules.muc.each_room() do
-		print(room.jid);
-		c = c + 1;
-	end
-	return true, c.." rooms";
-end
-
-local um = require"core.usermanager";
-
-def_env.user = {};
-function def_env.user:create(jid, password)
-	local username, host = jid_split(jid);
-	if not prosody.hosts[host] then
-		return nil, "No such host: "..host;
-	elseif um.user_exists(username, host) then
-		return nil, "User exists";
-	end
-	local ok, err = um.create_user(username, password, host);
-	if ok then
-		return true, "User created";
-	else
-		return nil, "Could not create user: "..err;
-	end
-end
-
-function def_env.user:delete(jid)
-	local username, host = jid_split(jid);
-	if not prosody.hosts[host] then
-		return nil, "No such host: "..host;
-	elseif not um.user_exists(username, host) then
-		return nil, "No such user";
-	end
-	local ok, err = um.delete_user(username, host);
-	if ok then
-		return true, "User deleted";
-	else
-		return nil, "Could not delete user: "..err;
-	end
-end
-
-function def_env.user:password(jid, password)
-	local username, host = jid_split(jid);
-	if not prosody.hosts[host] then
-		return nil, "No such host: "..host;
-	elseif not um.user_exists(username, host) then
-		return nil, "No such user";
-	end
-	local ok, err = um.set_password(username, password, host, nil);
-	if ok then
-		return true, "User password changed";
-	else
-		return nil, "Could not change password for user: "..err;
-	end
-end
-
-function def_env.user:list(host, pat)
-	if not host then
-		return nil, "No host given";
-	elseif not prosody.hosts[host] then
-		return nil, "No such host";
-	end
-	local print = self.session.print;
-	local total, matches = 0, 0;
-	for user in um.users(host) do
-		if not pat or user:match(pat) then
-			print(user.."@"..host);
-			matches = matches + 1;
-		end
-		total = total + 1;
-	end
-	return true, "Showing "..(pat and (matches.." of ") or "all " )..total.." users";
-end
-
-def_env.xmpp = {};
-
-local st = require "util.stanza";
-function def_env.xmpp:ping(localhost, remotehost)
-	if prosody.hosts[localhost] then
-		module:send(st.iq{ from=localhost, to=remotehost, type="get", id="ping" }
-				:tag("ping", {xmlns="urn:xmpp:ping"}), prosody.hosts[localhost]);
-		return true, "Sent ping";
-	else
-		return nil, "No such host";
-	end
-end
-
-def_env.dns = {};
-local adns = require"net.adns";
-
-local function get_resolver(session)
-	local resolver = session.dns_resolver;
-	if not resolver then
-		resolver = adns.resolver();
-		session.dns_resolver = resolver;
-	end
-	return resolver;
-end
-
-function def_env.dns:lookup(name, typ, class)
-	local resolver = get_resolver(self.session);
-	local ret = "Query sent";
-	local print = self.session.print;
-	local function handler(...)
-		ret = "Got response";
-		print(...);
-	end
-	resolver:lookup(handler, name, typ, class);
-	return true, ret;
-end
-
-function def_env.dns:addnameserver(...)
-	local resolver = get_resolver(self.session);
-	resolver._resolver:addnameserver(...)
-	return true
-end
-
-function def_env.dns:setnameserver(...)
-	local resolver = get_resolver(self.session);
-	resolver._resolver:setnameserver(...)
-	return true
-end
-
-function def_env.dns:purge()
-	local resolver = get_resolver(self.session);
-	resolver._resolver:purge()
-	return true
-end
-
-function def_env.dns:cache()
-	local resolver = get_resolver(self.session);
-	return true, "Cache:\n"..tostring(resolver._resolver.cache)
-end
-
-def_env.http = {};
-
-function def_env.http:list()
-	local print = self.session.print;
-
-	for host in pairs(prosody.hosts) do
-		local http_apps = modulemanager.get_items("http-provider", host);
-		if #http_apps > 0 then
-			local http_host = module:context(host):get_option_string("http_host");
-			print("HTTP endpoints on "..host..(http_host and (" (using "..http_host.."):") or ":"));
-			for _, provider in ipairs(http_apps) do
-				local url = module:context(host):http_url(provider.name, provider.default_path);
-				print("", url);
-			end
-			print("");
-		end
-	end
-
-	local default_host = module:get_option_string("http_default_host");
-	if not default_host then
-		print("HTTP requests to unknown hosts will return 404 Not Found");
-	else
-		print("HTTP requests to unknown hosts will be handled by "..default_host);
-	end
-	return true;
-end
-
-def_env.debug = {};
-
-function def_env.debug:logevents(host)
-	helpers.log_host_events(host);
-	return true;
-end
-
-function def_env.debug:events(host, event)
-	local events_obj;
-	if host and host ~= "*" then
-		if host == "http" then
-			events_obj = require "net.http.server"._events;
-		elseif not prosody.hosts[host] then
-			return false, "Unknown host: "..host;
-		else
-			events_obj = prosody.hosts[host].events;
-		end
-	else
-		events_obj = prosody.events;
-	end
-	return true, helpers.show_events(events_obj, event);
-end
-
-function def_env.debug:timers()
-	local socket = require "socket";
-	local print = self.session.print;
-	local add_task = require"util.timer".add_task;
-	local h, params = add_task.h, add_task.params;
-	if h then
-		print("-- util.timer");
-		for i, id in ipairs(h.ids) do
-			if not params[id] then
-				print(os.date("%F %T", h.priorities[i]), h.items[id]);
-			elseif not params[id].callback then
-				print(os.date("%F %T", h.priorities[i]), h.items[id], unpack(params[id]));
-			else
-				print(os.date("%F %T", h.priorities[i]), params[id].callback, unpack(params[id]));
-			end
-		end
-	end
-	if server.event_base then
-		local count = 0;
-		for _, v in pairs(debug.getregistry()) do
-			if type(v) == "function" and v.callback and v.callback == add_task._on_timer then
-				count = count + 1;
-			end
-		end
-		print(count .. " libevent callbacks");
-	end
-	if h then
-		local next_time = h:peek();
-		if next_time then
-			return true, os.date("Next event at %F %T (in %%.6fs)", next_time):format(next_time - socket.gettime());
-		end
-	end
-	return true;
-end
-
--- COMPAT: debug:timers() was timer:info() for some time in trunk
-def_env.timer = { info = def_env.debug.timers };
-
-module:hook("server-stopping", function(event)
-	for _, session in pairs(sessions) do
-		session.print("Shutting down: "..(event.reason or "unknown reason"));
-	end
-end);
-
-def_env.stats = {};
-
-local function format_stat(type, value, ref_value)
-	ref_value = ref_value or value;
-	--do return tostring(value) end
-	if type == "duration" then
-		if ref_value < 0.001 then
-			return ("%d µs"):format(value*1000000);
-		elseif ref_value < 0.9 then
-			return ("%0.2f ms"):format(value*1000);
-		end
-		return ("%0.2f"):format(value);
-	elseif type == "size" then
-		if ref_value > 1048576 then
-			return ("%d MB"):format(value/1048576);
-		elseif ref_value > 1024 then
-			return ("%d KB"):format(value/1024);
-		end
-		return ("%d bytes"):format(value);
-	elseif type == "rate" then
-		if ref_value < 0.9 then
-			return ("%0.2f/min"):format(value*60);
-		end
-		return ("%0.2f/sec"):format(value);
-	end
-	return tostring(value);
-end
-
-local stats_methods = {};
-function stats_methods:bounds(_lower, _upper)
-	for _, stat_info in ipairs(self) do
-		local data = stat_info[4];
-		if data then
-			local lower = _lower or data.min;
-			local upper = _upper or data.max;
-			local new_data = {
-				min = lower;
-				max = upper;
-				samples = {};
-				sample_count = 0;
-				count = data.count;
-				units = data.units;
-			};
-			local sum = 0;
-			for _, v in ipairs(data.samples) do
-				if v > upper then
-					break;
-				elseif v>=lower then
-					table.insert(new_data.samples, v);
-					sum = sum + v;
-				end
-			end
-			new_data.sample_count = #new_data.samples;
-			stat_info[4] = new_data;
-			stat_info[3] = sum/new_data.sample_count;
-		end
-	end
-	return self;
-end
-
-function stats_methods:trim(lower, upper)
-	upper = upper or (100-lower);
-	local statistics = require "util.statistics";
-	for _, stat_info in ipairs(self) do
-		-- Strip outliers
-		local data = stat_info[4];
-		if data then
-			local new_data = {
-				min = statistics.get_percentile(data, lower);
-				max = statistics.get_percentile(data, upper);
-				samples = {};
-				sample_count = 0;
-				count = data.count;
-				units = data.units;
-			};
-			local sum = 0;
-			for _, v in ipairs(data.samples) do
-				if v > new_data.max then
-					break;
-				elseif v>=new_data.min then
-					table.insert(new_data.samples, v);
-					sum = sum + v;
-				end
-			end
-			new_data.sample_count = #new_data.samples;
-			stat_info[4] = new_data;
-			stat_info[3] = sum/new_data.sample_count;
-		end
-	end
-	return self;
-end
-
-function stats_methods:max(upper)
-	return self:bounds(nil, upper);
-end
-
-function stats_methods:min(lower)
-	return self:bounds(lower, nil);
-end
-
-function stats_methods:summary()
-	local statistics = require "util.statistics";
-	for _, stat_info in ipairs(self) do
-		local type, value, data = stat_info[2], stat_info[3], stat_info[4];
-		if data and data.samples then
-			table.insert(stat_info.output, string.format("Count: %d (%d captured)",
-				data.count,
-				data.sample_count
-			));
-			table.insert(stat_info.output, string.format("Min: %s  Mean: %s  Max: %s",
-				format_stat(type, data.min),
-				format_stat(type, value),
-				format_stat(type, data.max)
-			));
-			table.insert(stat_info.output, string.format("Q1: %s  Median: %s  Q3: %s",
-				format_stat(type, statistics.get_percentile(data, 25)),
-				format_stat(type, statistics.get_percentile(data, 50)),
-				format_stat(type, statistics.get_percentile(data, 75))
-			));
-		end
-	end
-	return self;
-end
-
-function stats_methods:cfgraph()
-	for _, stat_info in ipairs(self) do
-		local name, type, value, data = unpack(stat_info, 1, 4);
-		local function print(s)
-			table.insert(stat_info.output, s);
-		end
-
-		if data and data.sample_count and data.sample_count > 0 then
-			local raw_histogram = require "util.statistics".get_histogram(data);
-
-			local graph_width, graph_height = 50, 10;
-			local eighth_chars = "   ▁▂▃▄▅▆▇█";
-
-			local range = data.max - data.min;
-
-			if range > 0 then
-				local x_scaling = #raw_histogram/graph_width;
-				local histogram = {};
-				for i = 1, graph_width do
-					histogram[i] = math.max(raw_histogram[i*x_scaling-1] or 0, raw_histogram[i*x_scaling] or 0);
-				end
-
-				print("");
-				print(("_"):rep(52)..format_stat(type, data.max));
-				for row = graph_height, 1, -1 do
-					local row_chars = {};
-					local min_eighths, max_eighths = 8, 0;
-					for i = 1, #histogram do
-						local char_eighths = math.ceil(math.max(math.min((graph_height/(data.max/histogram[i]))-(row-1), 1), 0)*8);
-						if char_eighths < min_eighths then
-							min_eighths = char_eighths;
-						end
-						if char_eighths > max_eighths then
-							max_eighths = char_eighths;
-						end
-						if char_eighths == 0 then
-							row_chars[i] = "-";
-						else
-							local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
-							row_chars[i] = char;
-						end
-					end
-					print(table.concat(row_chars).."|-"..format_stat(type, data.max/(graph_height/(row-0.5))));
-				end
-				print(("\\    "):rep(11));
-				local x_labels = {};
-				for i = 1, 11 do
-					local s = ("%-4s"):format((i-1)*10);
-					if #s > 4 then
-						s = s:sub(1, 3).."…";
-					end
-					x_labels[i] = s;
-				end
-				print(" "..table.concat(x_labels, " "));
-				local units = "%";
-				local margin = math.floor((graph_width-#units)/2);
-				print((" "):rep(margin)..units);
-			else
-				print("[range too small to graph]");
-			end
-			print("");
-		end
-	end
-	return self;
-end
-
-function stats_methods:histogram()
-	for _, stat_info in ipairs(self) do
-		local name, type, value, data = unpack(stat_info, 1, 4);
-		local function print(s)
-			table.insert(stat_info.output, s);
-		end
-
-		if not data then
-			print("[no data]");
-			return self;
-		elseif not data.sample_count then
-			print("[not a sampled metric type]");
-			return self;
-		end
-
-		local graph_width, graph_height = 50, 10;
-		local eighth_chars = "   ▁▂▃▄▅▆▇█";
-
-		local range = data.max - data.min;
-
-		if range > 0 then
-			local n_buckets = graph_width;
-
-			local histogram = {};
-			for i = 1, n_buckets do
-				histogram[i] = 0;
-			end
-			local max_bin_samples = 0;
-			for _, d in ipairs(data.samples) do
-				local bucket = math.floor(1+(n_buckets-1)/(range/(d-data.min)));
-				histogram[bucket] = histogram[bucket] + 1;
-				if histogram[bucket] > max_bin_samples then
-					max_bin_samples = histogram[bucket];
-				end
-			end
-
-			print("");
-			print(("_"):rep(52)..max_bin_samples);
-			for row = graph_height, 1, -1 do
-				local row_chars = {};
-				local min_eighths, max_eighths = 8, 0;
-				for i = 1, #histogram do
-					local char_eighths = math.ceil(math.max(math.min((graph_height/(max_bin_samples/histogram[i]))-(row-1), 1), 0)*8);
-					if char_eighths < min_eighths then
-						min_eighths = char_eighths;
-					end
-					if char_eighths > max_eighths then
-						max_eighths = char_eighths;
-					end
-					if char_eighths == 0 then
-						row_chars[i] = "-";
-					else
-						local char = eighth_chars:sub(char_eighths*3+1, char_eighths*3+3);
-						row_chars[i] = char;
-					end
-				end
-				print(table.concat(row_chars).."|-"..math.ceil((max_bin_samples/graph_height)*(row-0.5)));
-			end
-			print(("\\    "):rep(11));
-			local x_labels = {};
-			for i = 1, 11 do
-				local s = ("%-4s"):format(format_stat(type, data.min+range*i/11, data.min):match("^%S+"));
-				if #s > 4 then
-					s = s:sub(1, 3).."…";
-				end
-				x_labels[i] = s;
-			end
-			print(" "..table.concat(x_labels, " "));
-			local units = format_stat(type, data.min):match("%s+(.+)$") or data.units or "";
-			local margin = math.floor((graph_width-#units)/2);
-			print((" "):rep(margin)..units);
-		else
-			print("[range too small to graph]");
-		end
-		print("");
-	end
-	return self;
-end
-
-local function stats_tostring(stats)
-	local print = stats.session.print;
-	for _, stat_info in ipairs(stats) do
-		if #stat_info.output > 0 then
-			print("\n#"..stat_info[1]);
-			print("");
-			for _, v in ipairs(stat_info.output) do
-				print(v);
-			end
-			print("");
-		else
-			print(("%-50s %s"):format(stat_info[1], format_stat(stat_info[2], stat_info[3])));
-		end
-	end
-	return #stats.." statistics displayed";
-end
-
-local stats_mt = {__index = stats_methods, __tostring = stats_tostring }
-local function new_stats_context(self)
-	return setmetatable({ session = self.session, stats = true }, stats_mt);
-end
-
-function def_env.stats:show(filter)
-	local stats, changed, extra = require "core.statsmanager".get_stats();
-	local available, displayed = 0, 0;
-	local displayed_stats = new_stats_context(self);
-	for name, value in pairs(stats) do
-		available = available + 1;
-		if not filter or name:match(filter) then
-			displayed = displayed + 1;
-			local type = name:match(":(%a+)$");
-			table.insert(displayed_stats, {
-				name, type, value, extra[name];
-				output = {};
-			});
-		end
-	end
-	return displayed_stats;
-end
-
-
-
--------------
-
-function printbanner(session)
-	local option = module:get_option_string("console_banner", "full");
-	if option == "full" or option == "graphic" then
-		session.print [[
-                   ____                \   /     _
-                    |  _ \ _ __ ___  ___  _-_   __| |_   _
-                    | |_) | '__/ _ \/ __|/ _ \ / _` | | | |
-                    |  __/| | | (_) \__ \ |_| | (_| | |_| |
-                    |_|   |_|  \___/|___/\___/ \__,_|\__, |
-                    A study in simplicity            |___/
-
-]]
-	end
-	if option == "short" or option == "full" then
-	session.print("Welcome to the Prosody administration console. For a list of commands, type: help");
-	session.print("You may find more help on using this console in our online documentation at ");
-	session.print("https://prosody.im/doc/console\n");
-	end
-	if option ~= "short" and option ~= "full" and option ~= "graphic" then
-		session.print(option);
-	end
-end
-
 module:provides("net", {
 	name = "console";
 	listener = console_listener;
diff --git a/plugins/mod_announce.lua b/plugins/mod_announce.lua
index 970a273a..c742ebb8 100644
--- a/plugins/mod_announce.lua
+++ b/plugins/mod_announce.lua
@@ -38,6 +38,7 @@ end
 -- Old <message>-based jabberd-style announcement sending
 function handle_announcement(event)
 	local stanza = event.stanza;
+	-- luacheck: ignore 211/node
 	local node, host, resource = jid.split(stanza.attr.to);
 
 	if resource ~= "announce/online" then
diff --git a/plugins/mod_auth_anonymous.lua b/plugins/mod_auth_anonymous.lua
index 1f2bceb3..90646e71 100644
--- a/plugins/mod_auth_anonymous.lua
+++ b/plugins/mod_auth_anonymous.lua
@@ -11,6 +11,8 @@ local new_sasl = require "util.sasl".new;
 local datamanager = require "util.datamanager";
 local hosts = prosody.hosts;
 
+local allow_storage = module:get_option_boolean("allow_anonymous_storage", false);
+
 -- define auth provider
 local provider = {};
 
@@ -62,10 +64,14 @@ if not module:get_option_boolean("allow_anonymous_s2s", false) then
 end
 
 function module.load()
-	datamanager.add_callback(dm_callback);
+	if not allow_storage then
+		datamanager.add_callback(dm_callback);
+	end
 end
 function module.unload()
-	datamanager.remove_callback(dm_callback);
+	if not allow_storage then
+		datamanager.remove_callback(dm_callback);
+	end
 end
 
 module:provides("auth", provider);
diff --git a/plugins/mod_auth_internal_hashed.lua b/plugins/mod_auth_internal_hashed.lua
index 15058098..4fd0bd13 100644
--- a/plugins/mod_auth_internal_hashed.lua
+++ b/plugins/mod_auth_internal_hashed.lua
@@ -9,7 +9,7 @@
 
 local max = math.max;
 
-local getAuthenticationDatabaseSHA1 = require "util.sasl.scram".getAuthenticationDatabaseSHA1;
+local scram_hashers = require "util.sasl.scram".hashers;
 local usermanager = require "core.usermanager";
 local generate_uuid = require "util.uuid".generate;
 local new_sasl = require "util.sasl".new;
@@ -22,7 +22,9 @@ local host = module.host;
 
 local accounts = module:open_store("accounts");
 
-
+local hash_name = module:get_option_string("password_hash", "SHA-1");
+local get_auth_db = assert(scram_hashers[hash_name], "SCRAM-"..hash_name.." not supported by SASL library");
+local scram_name = "scram_"..hash_name:gsub("%-","_"):lower();
 
 -- Default; can be set per-user
 local default_iteration_count = 4096;
@@ -54,7 +56,7 @@ function provider.test_password(username, password)
 		return nil, "Auth failed. Stored salt and iteration count information is not complete.";
 	end
 
-	local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, credentials.salt, credentials.iteration_count);
+	local valid, stored_key, server_key = get_auth_db(password, credentials.salt, credentials.iteration_count);
 
 	local stored_key_hex = to_hex(stored_key);
 	local server_key_hex = to_hex(server_key);
@@ -72,7 +74,7 @@ function provider.set_password(username, password)
 	if account then
 		account.salt = generate_uuid();
 		account.iteration_count = max(account.iteration_count or 0, default_iteration_count);
-		local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, account.salt, account.iteration_count);
+		local valid, stored_key, server_key = get_auth_db(password, account.salt, account.iteration_count);
 		if not valid then
 			return valid, stored_key;
 		end
@@ -106,7 +108,7 @@ function provider.create_user(username, password)
 		return accounts:set(username, {});
 	end
 	local salt = generate_uuid();
-	local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, salt, default_iteration_count);
+	local valid, stored_key, server_key = get_auth_db(password, salt, default_iteration_count);
 	if not valid then
 		return valid, stored_key;
 	end
@@ -127,7 +129,7 @@ function provider.get_sasl_handler()
 		plain_test = function(_, username, password, realm)
 			return usermanager.test_password(username, realm, password), true;
 		end,
-		scram_sha_1 = function(_, username)
+		[scram_name] = function(_, username)
 			local credentials = accounts:get(username);
 			if not credentials then return; end
 			if credentials.password then
diff --git a/plugins/mod_authz_internal.lua b/plugins/mod_authz_internal.lua
new file mode 100644
index 00000000..0f6e4873
--- /dev/null
+++ b/plugins/mod_authz_internal.lua
@@ -0,0 +1,22 @@
+local normalize = require "util.jid".prep;
+local admin_jids = module:get_option_inherited_set("admins", {}) / normalize;
+local host = module.host;
+local role_store = module:open_store("roles");
+
+local admin_role = { ["prosody:admin"] = true };
+
+function get_user_roles(user)
+	if admin_jids:contains(user.."@"..host) then
+		return admin_role;
+	end
+	return role_store:get(user);
+end
+
+function get_jid_roles(jid)
+	if admin_jids:contains(jid) then
+		return admin_role;
+	end
+	return nil;
+end
+
+
diff --git a/plugins/mod_blocklist.lua b/plugins/mod_blocklist.lua
index ee48ffad..dad06b62 100644
--- a/plugins/mod_blocklist.lua
+++ b/plugins/mod_blocklist.lua
@@ -67,7 +67,7 @@ local function migrate_privacy_list(username)
 		if item.type == "jid" and item.action == "deny" then
 			local jid = jid_prep(item.value);
 			if not jid then
-				module:log("warn", "Invalid JID in privacy store for user '%s' not migrated: %s", username, tostring(item.value));
+				module:log("warn", "Invalid JID in privacy store for user '%s' not migrated: %s", username, item.value);
 			else
 				migrated_data[jid] = true;
 			end
@@ -162,7 +162,7 @@ local function edit_blocklist(event)
 	local blocklist = cache[username] or get_blocklist(username);
 
 	local new_blocklist = {
-		-- We set the [false] key to someting as a signal not to migrate privacy lists
+		-- We set the [false] key to something as a signal not to migrate privacy lists
 		[false] = blocklist[false] or { created = now; };
 	};
 	if type(blocklist[false]) == "table" then
@@ -189,6 +189,7 @@ local function edit_blocklist(event)
 
 	if is_blocking then
 		for jid in pairs(send_unavailable) do
+			-- Check that this JID isn't already blocked, i.e. this is not a change
 			if not blocklist[jid] then
 				for _, session in pairs(sessions[username].sessions) do
 					if session.presence then
diff --git a/plugins/mod_bosh.lua b/plugins/mod_bosh.lua
index 52168670..6393a73e 100644
--- a/plugins/mod_bosh.lua
+++ b/plugins/mod_bosh.lua
@@ -44,19 +44,41 @@ local bosh_max_polling = module:get_option_number("bosh_max_polling", 5);
 local bosh_max_wait = module:get_option_number("bosh_max_wait", 120);
 
 local consider_bosh_secure = module:get_option_boolean("consider_bosh_secure");
-local cross_domain = module:get_option("cross_domain_bosh", false);
+local cross_domain = module:get_option("cross_domain_bosh");
 
-if cross_domain == true then cross_domain = "*"; end
-if type(cross_domain) == "table" then cross_domain = table.concat(cross_domain, ", "); end
+if cross_domain ~= nil then
+	module:log("info", "The 'cross_domain_bosh' option has been deprecated");
+end
 
 local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat;
 
 -- All sessions, and sessions that have no requests open
 local sessions = module:shared("sessions");
 
+local measure_active = module:measure("active_sessions", "amount");
+local measure_inactive = module:measure("inactive_sessions", "amount");
+local report_bad_host = module:measure("bad_host", "rate");
+local report_bad_sid = module:measure("bad_sid", "rate");
+local report_new_sid = module:measure("new_sid", "rate");
+local report_timeout = module:measure("timeout", "rate");
+
+module:hook("stats-update", function ()
+	local active = 0;
+	local inactive = 0;
+	for _, session in pairs(sessions) do
+		if #session.requests > 0 then
+			active = active + 1;
+		else
+			inactive = inactive + 1;
+		end
+	end
+	measure_active(active);
+	measure_inactive(inactive);
+end);
+
 -- Used to respond to idle sessions (those with waiting requests)
 function on_destroy_request(request)
-	log("debug", "Request destroyed: %s", tostring(request));
+	log("debug", "Request destroyed: %s", request);
 	local session = sessions[request.context.sid];
 	if session then
 		local requests = session.requests;
@@ -73,7 +95,7 @@ function on_destroy_request(request)
 			if session.inactive_timer then
 				session.inactive_timer:stop();
 			end
-			session.inactive_timer = module:add_timer(max_inactive, check_inactive, session, request.context,
+			session.inactive_timer = module:add_timer(max_inactive, session_timeout, session, request.context,
 				"BOSH client silent for over "..max_inactive.." seconds");
 			(session.log or log)("debug", "BOSH session marked as inactive (for %ds)", max_inactive);
 		end
@@ -84,31 +106,16 @@ function on_destroy_request(request)
 	end
 end
 
-function check_inactive(now, session, context, reason) -- luacheck: ignore 212/now
+function session_timeout(now, session, context, reason) -- luacheck: ignore 212/now
 	if not session.destroyed then
+		report_timeout();
 		sessions[context.sid] = nil;
 		sm_destroy_session(session, reason);
 	end
 end
 
-local function set_cross_domain_headers(response)
-	local headers = response.headers;
-	headers.access_control_allow_methods = "GET, POST, OPTIONS";
-	headers.access_control_allow_headers = "Content-Type";
-	headers.access_control_max_age = "7200";
-	headers.access_control_allow_origin = cross_domain;
-	return response;
-end
-
-function handle_OPTIONS(event)
-	if cross_domain and event.request.headers.origin then
-		set_cross_domain_headers(event.response);
-	end
-	return "";
-end
-
 function handle_POST(event)
-	log("debug", "Handling new request %s: %s\n----------", tostring(event.request), tostring(event.request.body));
+	log("debug", "Handling new request %s: %s\n----------", event.request, event.request.body);
 
 	local request, response = event.request, event.response;
 	response.on_destroy = on_destroy_request;
@@ -121,10 +128,6 @@ function handle_POST(event)
 	local headers = response.headers;
 	headers.content_type = "text/xml; charset=utf-8";
 
-	if cross_domain and request.headers.origin then
-		set_cross_domain_headers(response);
-	end
-
 	-- stream:feed() calls the stream_callbacks, so all stanzas in
 	-- the body are processed in this next line before it returns.
 	-- In particular, the streamopened() stream callback is where
@@ -205,6 +208,7 @@ function handle_POST(event)
 		return;
 	end
 	module:log("warn", "Unable to associate request with a session (incomplete request?)");
+	report_bad_sid();
 	local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 		["xmlns:stream"] = xmlns_streams, condition = "item-not-found" });
 	return tostring(close_reply) .. "\n";
@@ -220,7 +224,7 @@ local function bosh_reset_stream(session) session.notopen = true; end
 
 local stream_xmlns_attr = { xmlns = "urn:ietf:params:xml:ns:xmpp-streams" };
 local function bosh_close_stream(session, reason)
-	(session.log or log)("info", "BOSH client disconnected: %s", tostring((reason and reason.condition or reason) or "session close"));
+	(session.log or log)("info", "BOSH client disconnected: %s", (reason and reason.condition or reason) or "session close");
 
 	local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 		["xmlns:stream"] = xmlns_streams });
@@ -245,7 +249,7 @@ local function bosh_close_stream(session, reason)
 				close_reply = reason;
 			end
 		end
-		log("info", "Disconnecting client, <stream:error> is: %s", tostring(close_reply));
+		log("info", "Disconnecting client, <stream:error> is: %s", close_reply);
 	end
 
 	local response_body = tostring(close_reply);
@@ -268,9 +272,19 @@ function stream_callbacks.streamopened(context, attr)
 		-- New session request
 		context.notopen = nil; -- Signals that we accept this opening tag
 
+		if not attr.to then
+			log("debug", "BOSH client tried to connect without specifying a host");
+			report_bad_host();
+			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
+				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
+			response:send(tostring(close_reply));
+			return;
+		end
+
 		local to_host = nameprep(attr.to);
 		if not to_host then
-			log("debug", "BOSH client tried to connect to invalid host: %s", tostring(attr.to));
+			log("debug", "BOSH client tried to connect to invalid host: %s", attr.to);
+			report_bad_host();
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
 			response:send(tostring(close_reply));
@@ -279,6 +293,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		if not prosody.hosts[to_host] then
 			log("debug", "BOSH client tried to connect to non-existant host: %s", attr.to);
+			report_bad_host();
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
 			response:send(tostring(close_reply));
@@ -287,6 +302,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		if prosody.hosts[to_host].type ~= "local" then
 			log("debug", "BOSH client tried to connect to %s host: %s", prosody.hosts[to_host].type, attr.to);
+			report_bad_host();
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
 			response:send(tostring(close_reply));
@@ -295,7 +311,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		local wait = tonumber(attr.wait);
 		if not rid or (not attr.wait or not wait or wait < 0 or wait % 1 ~= 0) then
-			log("debug", "BOSH client sent invalid rid or wait attributes: rid=%s, wait=%s", tostring(attr.rid), tostring(attr.wait));
+			log("debug", "BOSH client sent invalid rid or wait attributes: rid=%s, wait=%s", attr.rid, attr.wait);
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "bad-request" });
 			response:send(tostring(close_reply));
@@ -326,6 +342,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		session.log("debug", "BOSH session created for request from %s", session.ip);
 		log("info", "New BOSH session, assigned it sid '%s'", sid);
+		report_new_sid();
 
 		module:fire_event("bosh-session", { session = session, request = request });
 
@@ -340,7 +357,7 @@ function stream_callbacks.streamopened(context, attr)
 				s.attr.xmlns = "jabber:client";
 			end
 			s = filter("stanzas/out", s);
-			--log("debug", "Sending BOSH data: %s", tostring(s));
+			--log("debug", "Sending BOSH data: %s", s);
 			if not s then return true end
 			t_insert(session.send_buffer, tostring(s));
 
@@ -380,6 +397,7 @@ function stream_callbacks.streamopened(context, attr)
 	if not session then
 		-- Unknown sid
 		log("info", "Client tried to use sid '%s' which we don't know about", sid);
+		report_bad_sid();
 		response:send(tostring(st.stanza("body", { xmlns = xmlns_bosh, type = "terminate", condition = "item-not-found" })));
 		context.notopen = nil;
 		return;
@@ -442,7 +460,7 @@ function stream_callbacks.streamopened(context, attr)
 	end
 end
 
-local function handleerr(err) log("error", "Traceback[bosh]: %s", traceback(tostring(err), 2)); end
+local function handleerr(err) log("error", "Traceback[bosh]: %s", traceback(err, 2)); end
 
 function runner_callbacks:error(err) -- luacheck: ignore 212/self
 	return handleerr(err);
@@ -512,14 +530,16 @@ function stream_callbacks.error(context, error)
 	end
 end
 
+local GET_response_body = [[<html><body>
+	<p>It works! Now point your BOSH client to this URL to connect to Prosody.</p>
+	<p>For more information see <a href="https://prosody.im/doc/setting_up_bosh">Prosody: Setting up BOSH</a>.</p>
+	</body></html>]];
+
 local GET_response = {
 	headers = {
 		content_type = "text/html";
 	};
-	body = [[<html><body>
-	<p>It works! Now point your BOSH client to this URL to connect to Prosody.</p>
-	<p>For more information see <a href="https://prosody.im/doc/setting_up_bosh">Prosody: Setting up BOSH</a>.</p>
-	</body></html>]];
+	body = module:get_option_string("bosh_get_response_body", GET_response_body);
 };
 
 module:depends("http");
@@ -528,8 +548,6 @@ module:provides("http", {
 	route = {
 		["GET"] = GET_response;
 		["GET /"] = GET_response;
-		["OPTIONS"] = handle_OPTIONS;
-		["OPTIONS /"] = handle_OPTIONS;
 		["POST"] = handle_POST;
 		["POST /"] = handle_POST;
 	};
diff --git a/plugins/mod_c2s.lua b/plugins/mod_c2s.lua
index c648850f..ef4bd4b3 100644
--- a/plugins/mod_c2s.lua
+++ b/plugins/mod_c2s.lua
@@ -56,7 +56,17 @@ end);
 local stream_xmlns_attr = {xmlns='urn:ietf:params:xml:ns:xmpp-streams'};
 
 function stream_callbacks.streamopened(session, attr)
+	-- run _streamopened in async context
+	session.thread:run({ stream = "opened", attr = attr });
+end
+
+function stream_callbacks._streamopened(session, attr)
 	local send = session.send;
+	if not attr.to then
+		session:close{ condition = "improper-addressing",
+			text = "A 'to' attribute is required on stream headers" };
+		return;
+	end
 	local host = nameprep(attr.to);
 	if not host then
 		session:close{ condition = "improper-addressing",
@@ -98,7 +108,6 @@ function stream_callbacks.streamopened(session, attr)
 			session.compressed = info.compression;
 		else
 			(session.log or log)("info", "Stream encrypted");
-			session.compressed = sock.compression and sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -107,12 +116,23 @@ function stream_callbacks.streamopened(session, attr)
 	if features.tags[1] or session.full_jid then
 		send(features);
 	else
-		(session.log or log)("warn", "No stream features to offer");
+		if session.secure then
+			-- Here SASL should be offered
+			(session.log or log)("warn", "No stream features to offer on secure session. Check authentication settings.");
+		else
+			-- Normally STARTTLS would be offered
+			(session.log or log)("warn", "No stream features to offer on insecure session. Check encryption and security settings.");
+		end
 		session:close{ condition = "undefined-condition", text = "No stream features to proceed with" };
 	end
 end
 
-function stream_callbacks.streamclosed(session)
+function stream_callbacks.streamclosed(session, attr)
+	-- run _streamclosed in async context
+	session.thread:run({ stream = "closed", attr = attr });
+end
+
+function stream_callbacks._streamclosed(session)
 	session.log("debug", "Received </stream:stream>");
 	session:close(false);
 end
@@ -122,7 +142,7 @@ function stream_callbacks.error(session, error, data)
 		session.log("debug", "Invalid opening stream header (%s)", (data:gsub("^([^\1]+)\1", "{%1}")));
 		session:close("invalid-namespace");
 	elseif error == "parse-error" then
-		(session.log or log)("debug", "Client XML parse error: %s", tostring(data));
+		(session.log or log)("debug", "Client XML parse error: %s", data);
 		session:close("not-well-formed");
 	elseif error == "stream-error" then
 		local condition, text = "undefined-condition";
@@ -252,8 +272,6 @@ function listener.onconnect(conn)
 		local sock = conn:socket();
 		if sock.info then
 			session.compressed = sock:info"compression";
-		elseif sock.compression then
-			session.compressed = sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -273,7 +291,13 @@ function listener.onconnect(conn)
 	end
 
 	session.thread = runner(function (stanza)
-		core_process_stanza(session, stanza);
+		if st.is_stanza(stanza) then
+			core_process_stanza(session, stanza);
+		elseif stanza.stream == "opened" then
+			stream_callbacks._streamopened(session, stanza.attr);
+		elseif stanza.stream == "closed" then
+			stream_callbacks._streamclosed(session, stanza.attr);
+		end
 	end, runner_callbacks, session);
 
 	local filter = session.filter;
@@ -284,8 +308,12 @@ function listener.onconnect(conn)
 			if data then
 				local ok, err = stream:feed(data);
 				if not ok then
-					log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
-					session:close("not-well-formed");
+					log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
+					if err == "stanza-too-large" then
+						session:close({ condition = "policy-violation", text = "XML stanza is too big" });
+					else
+						session:close("not-well-formed");
+					end
 				end
 			end
 		end
@@ -328,6 +356,13 @@ function listener.onreadtimeout(conn)
 	end
 end
 
+function listener.ondrain(conn)
+	local session = sessions[conn];
+	if session then
+		return (hosts[session.host] or prosody).events.fire_event("c2s-ondrain", { session = session });
+	end
+end
+
 local function keepalive(event)
 	local session = event.session;
 	if not session.notopen then
@@ -360,6 +395,7 @@ module:provides("net", {
 	default_port = 5222;
 	encryption = "starttls";
 	multiplex = {
+		protocol = "xmpp-client";
 		pattern = "^<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:client%1.*>";
 	};
 });
diff --git a/plugins/mod_carbons.lua b/plugins/mod_carbons.lua
index 79d3e737..e0f0a711 100644
--- a/plugins/mod_carbons.lua
+++ b/plugins/mod_carbons.lua
@@ -5,10 +5,15 @@
 
 local st = require "util.stanza";
 local jid_bare = require "util.jid".bare;
+local jid_resource = require "util.jid".resource;
 local xmlns_carbons = "urn:xmpp:carbons:2";
 local xmlns_forward = "urn:xmpp:forward:0";
 local full_sessions, bare_sessions = prosody.full_sessions, prosody.bare_sessions;
 
+local function is_bare(jid)
+	return not jid_resource(jid);
+end
+
 local function toggle_carbons(event)
 	local origin, stanza = event.origin, event.stanza;
 	local state = stanza.tags[1].name;
@@ -20,6 +25,50 @@ end
 module:hook("iq-set/self/"..xmlns_carbons..":disable", toggle_carbons);
 module:hook("iq-set/self/"..xmlns_carbons..":enable", toggle_carbons);
 
+local function should_copy(stanza, c2s, user_bare) --> boolean, reason: string
+	local st_type = stanza.attr.type or "normal";
+	if stanza:get_child("private", xmlns_carbons) then
+		return false, "private";
+	end
+
+	if stanza:get_child("no-copy", "urn:xmpp:hints") then
+		return false, "hint";
+	end
+
+	if not c2s and stanza.attr.to ~= user_bare and stanza:get_child("x", "http://jabber.org/protocol/muc#user") then
+		-- MUC PMs are normally sent to full JIDs
+		return false, "muc-pm";
+	end
+
+	if st_type == "chat" then
+		return true, "type";
+	end
+
+	if st_type == "normal" and stanza:get_child("body") then
+		return true, "type";
+	end
+
+	-- Normal outgoing chat messages are sent to=bare JID. This clause should
+	-- match the error bounces from those, which would have from=bare JID and
+	-- be incoming (not c2s).
+	if st_type == "error" and not c2s and is_bare(stanza.attr.from) then
+		return true, "bounce";
+	end
+
+	if stanza:get_child(nil, "urn:xmpp:jingle-message:0") then
+		-- XXX Experimental XEP stuck in Proposed for almost a year at the time of this comment
+		return true, "jingle call";
+	end
+
+	for archived in stanza:childtags("stanza-id", "urn:xmpp:sid:0") do
+		if archived and archived.attr.by == user_bare then
+			return true, "archived";
+		end
+	end
+
+	return false, "default";
+end
+
 local function message_handler(event, c2s)
 	local origin, stanza = event.origin, event.stanza;
 	local orig_type = stanza.attr.type or "normal";
@@ -28,10 +77,6 @@ local function message_handler(event, c2s)
 	local orig_to = stanza.attr.to;
 	local bare_to = jid_bare(orig_to);
 
-	if not(orig_type == "chat" or (orig_type == "normal" and stanza:get_child("body"))) then
-		return -- Only chat type messages
-	end
-
 	-- Stanza sent by a local client
 	local bare_jid = bare_from; -- JID of the local user
 	local target_session = origin;
@@ -56,35 +101,21 @@ local function message_handler(event, c2s)
 		return -- No use in sending carbons to an offline user
 	end
 
-	if stanza:get_child("private", xmlns_carbons) then
-		if not c2s then
+	local should, why = should_copy(stanza, c2s, bare_jid);
+
+	if not should then
+		module:log("debug", "Not copying stanza: %s (%s)", stanza:top_tag(), why);
+		if why == "private" and not c2s then
 			stanza:maptags(function(tag)
 				if not ( tag.attr.xmlns == xmlns_carbons and tag.name == "private" ) then
 					return tag;
 				end
 			end);
 		end
-		module:log("debug", "Message tagged private, ignoring");
-		return
-	elseif stanza:get_child("no-copy", "urn:xmpp:hints") then
-		module:log("debug", "Message has no-copy hint, ignoring");
-		return
-	elseif not c2s and bare_jid ~= orig_to and stanza:get_child("x", "http://jabber.org/protocol/muc#user") then
-		module:log("debug", "MUC PM, ignoring");
-		return
-	end
-
-	-- Create the carbon copy and wrap it as per the Stanza Forwarding XEP
-	local copy = st.clone(stanza);
-	if c2s and not orig_to then
-		stanza.attr.to = bare_from;
+		return;
 	end
-	copy.attr.xmlns = "jabber:client";
-	local carbon = st.message{ from = bare_jid, type = orig_type, }
-		:tag(c2s and "sent" or "received", { xmlns = xmlns_carbons })
-			:tag("forwarded", { xmlns = xmlns_forward })
-				:add_child(copy):reset();
 
+	local carbon;
 	user_sessions = user_sessions and user_sessions.sessions;
 	for _, session in pairs(user_sessions) do
 		-- Carbons are sent to resources that have enabled it
@@ -93,6 +124,20 @@ local function message_handler(event, c2s)
 		and session ~= target_session
 		-- and isn't among the top resources that would receive the message per standard routing rules
 		and (c2s or session.priority ~= top_priority) then
+			if not carbon then
+				-- Create the carbon copy and wrap it as per the Stanza Forwarding XEP
+				local copy = st.clone(stanza);
+				if c2s and not orig_to then
+					stanza.attr.to = bare_from;
+				end
+				copy.attr.xmlns = "jabber:client";
+				carbon = st.message{ from = bare_jid, type = orig_type, }
+					:tag(c2s and "sent" or "received", { xmlns = xmlns_carbons })
+						:tag("forwarded", { xmlns = xmlns_forward })
+							:add_child(copy):reset();
+
+			end
+
 			carbon.attr.to = session.full_jid;
 			module:log("debug", "Sending carbon to %s", session.full_jid);
 			session.send(carbon);
diff --git a/plugins/mod_component.lua b/plugins/mod_component.lua
index b41204a2..d06df71c 100644
--- a/plugins/mod_component.lua
+++ b/plugins/mod_component.lua
@@ -49,6 +49,7 @@ function module.add_host(module)
 	local send;
 
 	local function on_destroy(session, err) --luacheck: ignore 212/err
+		module:set_status("warn", err and ("Disconnected: "..err) or "Disconnected");
 		env.connected = false;
 		env.session = false;
 		send = nil;
@@ -102,6 +103,7 @@ function module.add_host(module)
 		module:log("info", "External component successfully authenticated");
 		session.send(st.stanza("handshake"));
 		module:fire_event("component-authenticated", { session = session });
+		module:set_status("info", "Connected");
 
 		return true;
 	end
@@ -130,7 +132,8 @@ function module.add_host(module)
 			end
 			module:log("warn", "Component not connected, bouncing error for: %s", stanza:top_tag());
 			if stanza.attr.type ~= "error" and stanza.attr.type ~= "result" then
-				event.origin.send(st.error_reply(stanza, "wait", "service-unavailable", "Component unavailable"));
+				event.origin.send(st.error_reply(stanza, "wait", "remote-server-timeout", "Component unavailable", module.host)
+					:tag("not-connected", { xmlns = "xmpp:prosody.im/protocol/component" }));
 			end
 		end
 		return true;
@@ -165,11 +168,11 @@ local xmlns_xmpp_streams = "urn:ietf:params:xml:ns:xmpp-streams";
 
 function stream_callbacks.error(session, error, data)
 	if session.destroyed then return; end
-	module:log("warn", "Error processing component stream: %s", tostring(error));
+	module:log("warn", "Error processing component stream: %s", error);
 	if error == "no-stream" then
 		session:close("invalid-namespace");
 	elseif error == "parse-error" then
-		session.log("warn", "External component %s XML parse error: %s", tostring(session.host), tostring(data));
+		session.log("warn", "External component %s XML parse error: %s", session.host, data);
 		session:close("not-well-formed");
 	elseif error == "stream-error" then
 		local condition, text = "undefined-condition";
@@ -206,7 +209,7 @@ function stream_callbacks.streamclosed(session)
 	session:close();
 end
 
-local function handleerr(err) log("error", "Traceback[component]: %s", traceback(tostring(err), 2)); end
+local function handleerr(err) log("error", "Traceback[component]: %s", traceback(err, 2)); end
 function stream_callbacks.handlestanza(session, stanza)
 	-- Namespaces are icky.
 	if not stanza.attr.xmlns and stanza.name == "handshake" then
@@ -266,10 +269,10 @@ local function session_close(session, reason)
 					if reason.extra then
 						stanza:add_child(reason.extra);
 					end
-					module:log("info", "Disconnecting component, <stream:error> is: %s", tostring(stanza));
+					module:log("info", "Disconnecting component, <stream:error> is: %s", stanza);
 					session.send(stanza);
 				elseif reason.name then -- a stanza
-					module:log("info", "Disconnecting component, <stream:error> is: %s", tostring(reason));
+					module:log("info", "Disconnecting component, <stream:error> is: %s", reason);
 					session.send(reason);
 				end
 			end
@@ -310,7 +313,7 @@ function listener.onconnect(conn)
 	function session.data(_, data)
 		local ok, err = stream:feed(data);
 		if ok then return; end
-		module:log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
+		log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
 		session:close("not-well-formed");
 	end
 
@@ -325,7 +328,7 @@ end
 function listener.ondisconnect(conn, err)
 	local session = sessions[conn];
 	if session then
-		(session.log or log)("info", "component disconnected: %s (%s)", tostring(session.host), tostring(err));
+		(session.log or log)("info", "component disconnected: %s (%s)", session.host, err);
 		if session.host then
 			module:context(session.host):fire_event("component-disconnected", { session = session, reason = err });
 		end
diff --git a/plugins/mod_csi.lua b/plugins/mod_csi.lua
index 84476cac..458ff491 100644
--- a/plugins/mod_csi.lua
+++ b/plugins/mod_csi.lua
@@ -2,8 +2,9 @@ local st = require "util.stanza";
 local xmlns_csi = "urn:xmpp:csi:0";
 local csi_feature = st.stanza("csi", { xmlns = xmlns_csi });
 
+local csi_handler_available = nil;
 module:hook("stream-features", function (event)
-	if event.origin.username then
+	if event.origin.username and csi_handler_available then
 		event.features:add_child(csi_feature);
 	end
 end);
@@ -21,3 +22,14 @@ end
 module:hook("stanza/"..xmlns_csi..":active", refire_event("csi-client-active"));
 module:hook("stanza/"..xmlns_csi..":inactive", refire_event("csi-client-inactive"));
 
+function module.load()
+	if prosody.hosts[module.host].events._handlers["csi-client-active"] then
+		csi_handler_available = true;
+		module:set_status("core", "CSI handler module loaded");
+	else
+		csi_handler_available = false;
+		module:set_status("warn", "No CSI handler module loaded");
+	end
+end
+module:hook("module-loaded", module.load);
+module:hook("module-unloaded", module.load);
diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua
index 7fb6b41f..d814f083 100644
--- a/plugins/mod_csi_simple.lua
+++ b/plugins/mod_csi_simple.lua
@@ -1,4 +1,4 @@
--- Copyright (C) 2016-2018 Kim Alvefur
+-- Copyright (C) 2016-2020 Kim Alvefur
 --
 -- This project is MIT/X11 licensed. Please see the
 -- COPYING file in the source package for more information.
@@ -9,115 +9,209 @@ module:depends"csi"
 local jid = require "util.jid";
 local st = require "util.stanza";
 local dt = require "util.datetime";
-local new_queue = require "util.queue".new;
-
-local function new_pump(output, ...)
-	-- luacheck: ignore 212/self
-	local q = new_queue(...);
-	local flush = true;
-	function q:pause()
-		flush = false;
-	end
-	function q:resume()
-		flush = true;
-		return q:flush();
-	end
-	local push = q.push;
-	function q:push(item)
-		local ok = push(self, item);
-		if not ok then
-			q:flush();
-			output(item, self);
-		elseif flush then
-			return q:flush();
-		end
-		return true;
-	end
-	function q:flush()
-		local item = self:pop();
-		while item do
-			output(item, self);
-			item = self:pop();
-		end
-		return true;
-	end
-	return q;
-end
+local filters = require "util.filters";
 
 local queue_size = module:get_option_number("csi_queue_size", 256);
 
-module:hook("csi-is-stanza-important", function (event)
-	local stanza = event.stanza;
-	if not st.is_stanza(stanza) then
-		return true;
+local important_payloads = module:get_option_set("csi_important_payloads", { });
+
+function is_important(stanza) --> boolean, reason: string
+	if stanza == " " then
+		return true, "whitespace keepalive";
+	elseif type(stanza) == "string" then
+		return true, "raw data";
+	elseif not st.is_stanza(stanza) then
+		-- This should probably never happen
+		return true, type(stanza);
+	end
+	if stanza.attr.xmlns ~= nil then
+		-- stream errors, stream management etc
+		return true, "nonza";
 	end
 	local st_name = stanza.name;
 	if not st_name then return false; end
 	local st_type = stanza.attr.type;
 	if st_name == "presence" then
-		if st_type == nil or st_type == "unavailable" then
-			return false;
+		if st_type == nil or st_type == "unavailable" or st_type == "error" then
+			return false, "presence update";
 		end
-		return true;
+		-- TODO Some MUC awareness, e.g. check for the 'this relates to you' status code
+		return true, "subscription request";
 	elseif st_name == "message" then
 		if st_type == "headline" then
-			return false;
+			-- Headline messages are ephemeral by definition
+			return false, "headline";
+		end
+		if st_type == "error" then
+			return true, "delivery failure";
 		end
 		if stanza:get_child("sent", "urn:xmpp:carbons:2") then
-			return true;
+			return true, "carbon";
 		end
 		local forwarded = stanza:find("{urn:xmpp:carbons:2}received/{urn:xmpp:forward:0}/{jabber:client}message");
 		if forwarded then
 			stanza = forwarded;
 		end
 		if stanza:get_child("body") then
-			return true;
+			return true, "body";
 		end
 		if stanza:get_child("subject") then
-			return true;
+			-- Last step of a MUC join
+			return true, "subject";
 		end
 		if stanza:get_child("encryption", "urn:xmpp:eme:0") then
-			return true;
+			-- Since we can't know what an encrypted message contains, we assume it's important
+			-- XXX Experimental XEP
+			return true, "encrypted";
+		end
+		if stanza:get_child("x", "jabber:x:conference") or stanza:find("{http://jabber.org/protocol/muc#user}x/invite") then
+			return true, "invite";
 		end
 		if stanza:get_child(nil, "urn:xmpp:jingle-message:0") then
-			return true;
+			-- XXX Experimental XEP stuck in Proposed for almost a year at the time of this comment
+			return true, "jingle call";
+		end
+		for important in important_payloads do
+			if stanza:find(important) then
+				return true;
+			end
 		end
 		return false;
+	elseif st_name == "iq" then
+		return true;
 	end
-	return true;
+end
+
+module:hook("csi-is-stanza-important", function (event)
+	local important, why = is_important(event.stanza);
+	event.reason = why;
+	return important;
 end, -1);
 
-module:hook("csi-client-inactive", function (event)
-	local session = event.origin;
-	if session.pump then
-		session.pump:pause();
-	else
-		local bare_jid = jid.join(session.username, session.host);
-		local send = session.send;
-		session._orig_send = send;
-		local pump = new_pump(session.send, queue_size);
-		pump:pause();
-		session.pump = pump;
-		function session.send(stanza)
-			if session.state == "active" or module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then
-				pump:flush();
-				send(stanza);
-			else
-				if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then
-					stanza = st.clone(stanza);
-					stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = bare_jid, stamp = dt.datetime()}));
-				end
-				pump:push(stanza);
-			end
-			return true;
+local function should_flush(stanza, session, ctr) --> boolean, reason: string
+	if ctr >= queue_size then
+		return true, "queue size limit reached";
+	end
+	local event = { stanza = stanza, session = session };
+	local ret = module:fire_event("csi-is-stanza-important", event)
+	return ret, event.reason;
+end
+
+local function with_timestamp(stanza, from)
+	if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then
+		stanza = st.clone(stanza);
+		stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = from, stamp = dt.datetime()}));
+	end
+	return stanza;
+end
+
+local measure_buffer_hold = module:measure("buffer_hold", "times");
+
+local flush_reasons = setmetatable({}, {
+		__index = function (t, reason)
+			local m = module:measure("flush_reason."..reason:gsub("%W", "_"), "rate");
+			t[reason] = m;
+			return m;
+		end;
+	});
+
+
+local function manage_buffer(stanza, session)
+	local ctr = session.csi_counter or 0;
+	local flush, why = should_flush(stanza, session, ctr);
+	if flush then
+		if session.csi_measure_buffer_hold then
+			session.csi_measure_buffer_hold();
+			session.csi_measure_buffer_hold = nil;
 		end
+		flush_reasons[why or "important"]();
+		session.log("debug", "Flushing buffer (%s; queue size is %d)", why or "important", session.csi_counter);
+		session.conn:resume_writes();
+	else
+		session.log("debug", "Holding buffer (%s; queue size is %d)", why or "unimportant", session.csi_counter);
+		stanza = with_timestamp(stanza, jid.join(session.username, session.host))
+	end
+	session.csi_counter = ctr + 1;
+	return stanza;
+end
+
+local function flush_buffer(data, session)
+	session.log("debug", "Flushing buffer (%s; queue size is %d)", "client activity", session.csi_counter);
+	flush_reasons["client activity"]();
+	if session.csi_measure_buffer_hold then
+		session.csi_measure_buffer_hold();
+		session.csi_measure_buffer_hold = nil;
+	end
+	session.conn:resume_writes();
+	return data;
+end
+
+function enable_optimizations(session)
+	if session.conn and session.conn.pause_writes then
+		session.conn:pause_writes();
+		session.csi_measure_buffer_hold = measure_buffer_hold();
+		session.csi_counter = 0;
+		filters.add_filter(session, "stanzas/out", manage_buffer);
+		filters.add_filter(session, "bytes/in", flush_buffer);
+	else
+		session.log("warn", "Session connection does not support write pausing");
+	end
+end
+
+function disable_optimizations(session)
+	filters.remove_filter(session, "stanzas/out", manage_buffer);
+	filters.remove_filter(session, "bytes/in", flush_buffer);
+	session.csi_counter = nil;
+	if session.csi_measure_buffer_hold then
+		session.csi_measure_buffer_hold();
+		session.csi_measure_buffer_hold = nil;
 	end
+	if session.conn and session.conn.resume_writes then
+		session.conn:resume_writes();
+	end
+end
+
+module:hook("csi-client-inactive", function (event)
+	local session = event.origin;
+	enable_optimizations(session);
 end);
 
 module:hook("csi-client-active", function (event)
 	local session = event.origin;
-	if session.pump then
-		session.pump:resume();
+	disable_optimizations(session);
+end);
+
+module:hook("pre-resource-unbind", function (event)
+	local session = event.session;
+	disable_optimizations(session);
+end, 1);
+
+module:hook("c2s-ondrain", function (event)
+	local session = event.session;
+	if session.state == "inactive" and session.conn and session.conn.pause_writes then
+		session.conn:pause_writes();
+		session.csi_measure_buffer_hold = measure_buffer_hold();
+		session.log("debug", "Buffer flushed, resuming inactive mode (queue size was %d)", session.csi_counter);
+		session.csi_counter = 0;
 	end
 end);
 
+function module.load()
+	for _, user_session in pairs(prosody.hosts[module.host].sessions) do
+		for _, session in pairs(user_session.sessions) do
+			if session.state == "inactive" then
+				enable_optimizations(session);
+			end
+		end
+	end
+end
+
+function module.unload()
+	for _, user_session in pairs(prosody.hosts[module.host].sessions) do
+		for _, session in pairs(user_session.sessions) do
+			if session.state == "inactive" then
+				disable_optimizations(session);
+			end
+		end
+	end
+end
diff --git a/plugins/mod_dialback.lua b/plugins/mod_dialback.lua
index eddc3209..f580d948 100644
--- a/plugins/mod_dialback.lua
+++ b/plugins/mod_dialback.lua
@@ -93,6 +93,11 @@ module:hook("stanza/jabber:server:dialback:result", function(event)
 		-- he wants to be identified through dialback
 		-- We need to check the key with the Authoritative server
 		local attr = stanza.attr;
+		if not attr.to or not attr.from then
+			origin.log("debug", "Missing Dialback addressing (from=%q, to=%q)", attr.from, attr.to);
+			origin:close("improper-addressing");
+			return true;
+		end
 		local to, from = nameprep(attr.to), nameprep(attr.from);
 
 		if not hosts[to] then
@@ -102,6 +107,7 @@ module:hook("stanza/jabber:server:dialback:result", function(event)
 			return true;
 		elseif not from then
 			origin:close("improper-addressing");
+			return true;
 		end
 
 		if dwd and origin.secure then
diff --git a/plugins/mod_external_services.lua b/plugins/mod_external_services.lua
new file mode 100644
index 00000000..e18e7c3e
--- /dev/null
+++ b/plugins/mod_external_services.lua
@@ -0,0 +1,233 @@
+
+local dt = require "util.datetime";
+local base64 = require "util.encodings".base64;
+local hashes = require "util.hashes";
+local st = require "util.stanza";
+local jid = require "util.jid";
+local array = require "util.array";
+
+local default_host = module:get_option_string("external_service_host", module.host);
+local default_port = module:get_option_number("external_service_port");
+local default_secret = module:get_option_string("external_service_secret");
+local default_ttl = module:get_option_number("external_service_ttl", 86400);
+
+local configured_services = module:get_option_array("external_services", {});
+
+local access = module:get_option_set("external_service_access", {});
+
+-- https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00
+local function behave_turn_rest_credentials(srv, item, secret)
+	local ttl = default_ttl;
+	if type(item.ttl) == "number" then
+		ttl = item.ttl;
+	end
+	local expires = srv.expires or os.time() + ttl;
+	local username;
+	if type(item.username) == "string" then
+		username = string.format("%d:%s", expires, item.username);
+	else
+		username = string.format("%d", expires);
+	end
+	srv.username = username;
+	srv.password = base64.encode(hashes.hmac_sha1(secret, srv.username));
+end
+
+local algorithms = {
+	turn = behave_turn_rest_credentials;
+}
+
+-- filter config into well-defined service records
+local function prepare(item)
+	if type(item) ~= "table" then
+		module:log("error", "Service definition is not a table: %q", item);
+		return nil;
+	end
+
+	local srv = {
+		type = nil;
+		transport = nil;
+		host = default_host;
+		port = default_port;
+		username = nil;
+		password = nil;
+		restricted = nil;
+		expires = nil;
+	};
+
+	if type(item.type) == "string" then
+		srv.type = item.type;
+	else
+		module:log("error", "Service missing mandatory 'type' field: %q", item);
+		return nil;
+	end
+	if type(item.transport) == "string" then
+		srv.transport = item.transport;
+	end
+	if type(item.host) == "string" then
+		srv.host = item.host;
+	end
+	if type(item.port) == "number" then
+		srv.port = item.port;
+	end
+	if type(item.username) == "string" then
+		srv.username = item.username;
+	end
+	if type(item.password) == "string" then
+		srv.password = item.password;
+		srv.restricted = true;
+	end
+	if item.restricted == true then
+		srv.restricted = true;
+	end
+	if type(item.expires) == "number" then
+		srv.expires = item.expires;
+	elseif type(item.ttl) == "number" then
+		srv.expires = os.time() + item.ttl;
+	end
+	if (item.secret == true and default_secret) or type(item.secret) == "string" then
+		local secret_cb = item.credentials_cb or algorithms[item.algorithm] or algorithms[srv.type];
+		local secret = item.secret;
+		if secret == true then
+			secret = default_secret;
+		end
+		if secret_cb then
+			secret_cb(srv, item, secret);
+			srv.restricted = true;
+		end
+	end
+	return srv;
+end
+
+function module.load()
+	-- Trigger errors on startup
+	local services = configured_services / prepare;
+	if #services == 0 then
+		module:log("warn", "No services configured or all had errors");
+	end
+end
+
+-- Ensure only valid items are added in events
+local services_mt = {
+	__index = getmetatable(array()).__index;
+	__newindex = function (self, i, v)
+		rawset(self, i, assert(prepare(v), "Invalid service entry added"));
+	end;
+}
+
+local function handle_services(event)
+	local origin, stanza = event.origin, event.stanza;
+	local action = stanza.tags[1];
+
+	local user_bare = jid.bare(stanza.attr.from);
+	local user_host = jid.host(user_bare);
+	if not ((access:empty() and origin.type == "c2s") or access:contains(user_bare) or access:contains(user_host)) then
+		origin.send(st.error_reply(stanza, "auth", "forbidden"));
+		return true;
+	end
+
+	local reply = st.reply(stanza):tag("services", { xmlns = action.attr.xmlns });
+	local extras = module:get_host_items("external_service");
+	local services = ( configured_services + extras ) / prepare;
+
+	local requested_type = action.attr.type;
+	if requested_type then
+		services:filter(function(item)
+			return item.type == requested_type;
+		end);
+	end
+
+	setmetatable(services, services_mt);
+
+	module:fire_event("external_service/services", {
+			origin = origin;
+			stanza = stanza;
+			reply = reply;
+			requested_type = requested_type;
+			services = services;
+		});
+
+	for _, srv in ipairs(services) do
+		reply:tag("service", {
+				type = srv.type;
+				transport = srv.transport;
+				host = srv.host;
+				port = srv.port and string.format("%d", srv.port) or nil;
+				username = srv.username;
+				password = srv.password;
+				expires = srv.expires and dt.datetime(srv.expires) or nil;
+				restricted = srv.restricted and "1" or nil;
+			}):up();
+	end
+
+	origin.send(reply);
+	return true;
+end
+
+local function handle_credentials(event)
+	local origin, stanza = event.origin, event.stanza;
+	local action = stanza.tags[1];
+
+	if origin.type ~= "c2s" then
+		origin.send(st.error_reply(stanza, "auth", "forbidden"));
+		return true;
+	end
+
+	local reply = st.reply(stanza):tag("credentials", { xmlns = action.attr.xmlns });
+	local extras = module:get_host_items("external_service");
+	local services = ( configured_services + extras ) / prepare;
+	services:filter(function (item)
+		return item.restricted;
+	end)
+
+	local requested_credentials = {};
+	for service in action:childtags("service") do
+		table.insert(requested_credentials, {
+				type = service.attr.type;
+				host = service.attr.host;
+				port = tonumber(service.attr.port);
+			});
+	end
+
+	setmetatable(services, services_mt);
+	setmetatable(requested_credentials, services_mt);
+
+	module:fire_event("external_service/credentials", {
+			origin = origin;
+			stanza = stanza;
+			reply = reply;
+			requested_credentials = requested_credentials;
+			services = services;
+		});
+
+	for req_srv in action:childtags("service") do
+		for _, srv in ipairs(services) do
+			if srv.type == req_srv.attr.type and srv.host == req_srv.attr.host
+				and not req_srv.attr.port or srv.port == tonumber(req_srv.attr.port) then
+				reply:tag("service", {
+						type = srv.type;
+						transport = srv.transport;
+						host = srv.host;
+						port = srv.port and string.format("%d", srv.port) or nil;
+						username = srv.username;
+						password = srv.password;
+						expires = srv.expires and dt.datetime(srv.expires) or nil;
+						restricted = srv.restricted and "1" or nil;
+					}):up();
+			end
+		end
+	end
+
+	origin.send(reply);
+	return true;
+end
+
+-- XEP-0215 v0.7
+module:add_feature("urn:xmpp:extdisco:2");
+module:hook("iq-get/host/urn:xmpp:extdisco:2:services", handle_services);
+module:hook("iq-get/host/urn:xmpp:extdisco:2:credentials", handle_credentials);
+
+-- COMPAT XEP-0215 v0.6
+-- Those still on the old version gets to deal with undefined attributes until they upgrade.
+module:add_feature("urn:xmpp:extdisco:1");
+module:hook("iq-get/host/urn:xmpp:extdisco:1:services", handle_services);
+module:hook("iq-get/host/urn:xmpp:extdisco:1:credentials", handle_credentials);
diff --git a/plugins/mod_groups.lua b/plugins/mod_groups.lua
index 646b7408..0c44f481 100644
--- a/plugins/mod_groups.lua
+++ b/plugins/mod_groups.lua
@@ -25,7 +25,7 @@ function inject_roster_contacts(event)
 	local function import_jids_to_roster(group_name)
 		for jid in pairs(groups[group_name]) do
 			-- Add them to roster
-			--module:log("debug", "processing jid %s in group %s", tostring(jid), tostring(group_name));
+			--module:log("debug", "processing jid %s in group %s", jid, group_name);
 			if jid ~= bare_jid then
 				if not roster[jid] then roster[jid] = {}; end
 				roster[jid].subscription = "both";
@@ -99,7 +99,7 @@ function module.load()
 				end
 				members[false][#members[false]+1] = curr_group; -- Is a public group
 			end
-			module:log("debug", "New group: %s", tostring(curr_group));
+			module:log("debug", "New group: %s", curr_group);
 			groups[curr_group] = groups[curr_group] or {};
 		else
 			-- Add JID
@@ -108,7 +108,7 @@ function module.load()
 			local jid;
 			jid = jid_prep(entryjid:match("%S+"));
 			if jid then
-				module:log("debug", "New member of %s: %s", tostring(curr_group), tostring(jid));
+				module:log("debug", "New member of %s: %s", curr_group, jid);
 				groups[curr_group][jid] = name or false;
 				members[jid] = members[jid] or {};
 				members[jid][#members[jid]+1] = curr_group;
diff --git a/plugins/mod_http.lua b/plugins/mod_http.lua
index a1d409bd..84e2d18f 100644
--- a/plugins/mod_http.lua
+++ b/plugins/mod_http.lua
@@ -7,13 +7,21 @@
 --
 
 module:set_global();
-module:depends("http_errors");
+pcall(function ()
+	module:depends("http_errors");
+end);
 
 local portmanager = require "core.portmanager";
 local moduleapi = require "core.moduleapi";
 local url_parse = require "socket.url".parse;
 local url_build = require "socket.url".build;
 local normalize_path = require "util.http".normalize_path;
+local set = require "util.set";
+
+local ip_util = require "util.ip";
+local new_ip = ip_util.new_ip;
+local match_ip = ip_util.match;
+local parse_cidr = ip_util.parse_cidr;
 
 local server = require "net.http.server";
 
@@ -22,6 +30,12 @@ server.set_default_host(module:get_option_string("http_default_host"));
 server.set_option("body_size_limit", module:get_option_number("http_max_content_size"));
 server.set_option("buffer_size_limit", module:get_option_number("http_max_buffer_size"));
 
+-- CORS settigs
+local opt_methods = module:get_option_set("access_control_allow_methods", { "GET", "OPTIONS" });
+local opt_headers = module:get_option_set("access_control_allow_headers", { "Content-Type" });
+local opt_credentials = module:get_option_boolean("access_control_allow_credentials", false);
+local opt_max_age = module:get_option_number("access_control_max_age", 2 * 60 * 60);
+
 local function get_http_event(host, app_path, key)
 	local method, path = key:match("^(%S+)%s+(.+)$");
 	if not method then -- No path specified, default to "" (base path)
@@ -79,10 +93,22 @@ function moduleapi.http_url(module, app_name, default_path)
 			return url_build(url);
 		end
 	end
-	module:log("warn", "No http ports enabled, can't generate an external URL");
+	if prosody.process_type == "prosody" then
+		module:log("warn", "No http ports enabled, can't generate an external URL");
+	end
 	return "http://disabled.invalid/";
 end
 
+local function apply_cors_headers(response, methods, headers, max_age, allow_credentials, origin)
+	response.headers.access_control_allow_methods = tostring(methods);
+	response.headers.access_control_allow_headers = tostring(headers);
+	response.headers.access_control_max_age = tostring(max_age)
+	response.headers.access_control_allow_origin = origin or "*";
+	if allow_credentials then
+		response.headers.access_control_allow_credentials = "true";
+	end
+end
+
 function module.add_host(module)
 	local host = module.host;
 	if host ~= "*" then
@@ -101,9 +127,29 @@ function module.add_host(module)
 		end
 		apps[app_name] = apps[app_name] or {};
 		local app_handlers = apps[app_name];
+
+		local app_methods = opt_methods;
+
+		local function cors_handler(event_data)
+			local request, response = event_data.request, event_data.response;
+			apply_cors_headers(response, app_methods, opt_headers, opt_max_age, opt_credentials, request.headers.origin);
+		end
+
+		local function options_handler(event_data)
+			cors_handler(event_data);
+			return "";
+		end
+
+		local streaming = event.item.streaming_uploads;
+
 		for key, handler in pairs(event.item.route or {}) do
 			local event_name = get_http_event(host, app_path, key);
 			if event_name then
+				local method = event_name:match("^%S+");
+				if not app_methods:contains(method) then
+					app_methods = app_methods + set.new{ method };
+				end
+				local options_event_name = event_name:gsub("^%S+", "OPTIONS");
 				if type(handler) ~= "function" then
 					local data = handler;
 					handler = function () return data; end
@@ -118,9 +164,24 @@ function module.add_host(module)
 				elseif event_name:sub(-1, -1) == "/" then
 					module:hook_object_event(server, event_name:sub(1, -2), redir_handler, -1);
 				end
+				if not streaming then
+					-- COMPAT Modules not compatible with streaming uploads behave as before.
+					local _handler = handler;
+					function handler(event) -- luacheck: ignore 432/event
+						if event.request.body ~= false then
+							return _handler(event);
+						end
+					end
+				end
 				if not app_handlers[event_name] then
-					app_handlers[event_name] = handler;
+					app_handlers[event_name] = {
+						main = handler;
+						cors = cors_handler;
+						options = options_handler;
+					};
 					module:hook_object_event(server, event_name, handler);
+					module:hook_object_event(server, event_name, cors_handler, 1);
+					module:hook_object_event(server, options_event_name, options_handler, -1);
 				else
 					module:log("warn", "App %s added handler twice for '%s', ignoring", app_name, event_name);
 				end
@@ -130,8 +191,8 @@ function module.add_host(module)
 		end
 		local services = portmanager.get_active_services();
 		if services:get("https") or services:get("http") then
-			module:log("debug", "Serving '%s' at %s", app_name, module:http_url(app_name, app_path));
-		else
+			module:log("info", "Serving '%s' at %s", app_name, module:http_url(app_name, app_path));
+		elseif prosody.process_type == "prosody" then
 			module:log("warn", "Not listening on any ports, '%s' will be unreachable", app_name);
 		end
 	end
@@ -139,8 +200,11 @@ function module.add_host(module)
 	local function http_app_removed(event)
 		local app_handlers = apps[event.item.name];
 		apps[event.item.name] = nil;
-		for event_name, handler in pairs(app_handlers) do
-			module:unhook_object_event(server, event_name, handler);
+		for event_name, handlers in pairs(app_handlers) do
+			module:unhook_object_event(server, event_name, handlers.main);
+			module:unhook_object_event(server, event_name, handlers.cors);
+			local options_event_name = event_name:gsub("^%S+", "OPTIONS");
+			module:unhook_object_event(server, options_event_name, handlers.options);
 		end
 	end
 
@@ -158,13 +222,31 @@ module.add_host(module); -- set up handling on global context too
 
 local trusted_proxies = module:get_option_set("trusted_proxies", { "127.0.0.1", "::1" })._items;
 
+local function is_trusted_proxy(ip)
+	local parsed_ip = new_ip(ip)
+	for trusted_proxy in trusted_proxies do
+		if match_ip(parsed_ip, parse_cidr(trusted_proxy)) then
+			return true;
+		end
+	end
+	return false
+end
+
 local function get_ip_from_request(request)
 	local ip = request.conn:ip();
 	local forwarded_for = request.headers.x_forwarded_for;
 	if forwarded_for then
+		-- luacheck: ignore 631
+		-- This logic looks weird at first, but it makes sense.
+		-- The for loop will take the last non-trusted-proxy IP from `forwarded_for`.
+		-- We append the original request IP to the header. Then, since the last IP wins, there are two cases:
+		-- Case a) The original request IP is *not* in trusted proxies, in which case the X-Forwarded-For header will, effectively, be ineffective; the original request IP will win because it overrides any other IP in the header.
+		-- Case b) The original request IP is in trusted proxies. In that case, the if branch in the for loop will skip the last IP, causing it to be ignored. The second-to-last IP will be taken instead.
+		-- Case c) If the second-to-last IP is also a trusted proxy, it will also be ignored, iteratively, up to the last IP which isn’t in trusted proxies.
+		-- Case d) If all IPs are in trusted proxies, something went obviously wrong and the logic never overwrites `ip`, leaving it at the original request IP.
 		forwarded_for = forwarded_for..", "..ip;
 		for forwarded_ip in forwarded_for:gmatch("[^%s,]+") do
-			if not trusted_proxies[forwarded_ip] then
+			if not is_trusted_proxy(forwarded_ip) then
 				ip = forwarded_ip;
 			end
 		end
@@ -195,10 +277,8 @@ module:provides("net", {
 	listener = server.listener;
 	default_port = 5281;
 	encryption = "ssl";
-	ssl_config = {
-		verify = "none";
-	};
 	multiplex = {
+		protocol = "http/1.1";
 		pattern = "^[A-Z]";
 	};
 });
diff --git a/plugins/mod_http_errors.lua b/plugins/mod_http_errors.lua
index 13473219..a2138d88 100644
--- a/plugins/mod_http_errors.lua
+++ b/plugins/mod_http_errors.lua
@@ -26,28 +26,38 @@ local html = [[
 <meta charset="utf-8">
 <title>{title}</title>
 <style>
-body{
-	margin-top:14%;
-	text-align:center;
-	background-color:#F8F8F8;
-	font-family:sans-serif;
+body {
+	margin-top : 14%;
+	text-align : center;
+	background-color : #F8F8F8;
+	font-family : sans-serif
 }
-h1{
-	font-size:xx-large;
+
+h1 {
+	font-size : xx-large
 }
-p{
-	font-size:x-large;
+
+p {
+	font-size : x-large
 }
-p+p {
-	font-size:large;
-	font-family:courier;
+
+p.extra {
+	font-size : large;
+	font-family : courier
+}
+
+@media(prefers-color-scheme: dark) {
+	body {
+		background-color: #161616;
+		color: #eee
+	}
 }
 </style>
 </head>
 <body>
 <h1>{title}</h1>
 <p>{message}</p>
-<p>{extra?}</p>
+{extra&<p class="extra">{extra?}</p>}
 </body>
 </html>
 ]];
@@ -70,5 +80,17 @@ module:hook_object_event(server, "http-error", function (event)
 	if event.response then
 		event.response.headers.content_type = "text/html; charset=utf-8";
 	end
-	return get_page(event.code, (show_private and event.private_message) or event.message);
+	return get_page(event.code, (show_private and event.private_message) or event.message or (event.error and event.error.text));
 end);
+
+module:hook_object_event(server, "http-error", function (event)
+	local request, response = event.request, event.response;
+	if request and response and request.path == "/" and response.status_code == 404 then
+		response.headers.content_type = "text/html; charset=utf-8";
+		return render(html, {
+				title = "Prosody is running!";
+				message = "Welcome to the XMPP world!";
+			});
+	end
+end, 1);
+
diff --git a/plugins/mod_http_files.lua b/plugins/mod_http_files.lua
index a8398c01..4d0b14cd 100644
--- a/plugins/mod_http_files.lua
+++ b/plugins/mod_http_files.lua
@@ -7,14 +7,9 @@
 --
 
 module:depends("http");
-local server = require"net.http.server";
-local lfs = require "lfs";
 
-local os_date = os.date;
 local open = io.open;
-local stat = lfs.attributes;
-local build_path = require"socket.url".build_path;
-local path_sep = package.config:sub(1,1);
+local fileserver = require"net.http.files";
 
 local base_path = module:get_option_path("http_files_dir", module:get_option_path("http_path"));
 local cache_size = module:get_option_number("http_files_cache_size", 128);
@@ -38,7 +33,9 @@ if not mime_map then
 	module:shared("/*/http_files/mime").types = mime_map;
 
 	local mime_types, err = open(module:get_option_path("mime_types_file", "/etc/mime.types", "config"), "r");
-	if mime_types then
+	if not mime_types then
+		module:log("debug", "Could not open MIME database: %s", err);
+	else
 		local mime_data = mime_types:read("*a");
 		mime_types:close();
 		setmetatable(mime_map, {
@@ -51,148 +48,56 @@ if not mime_map then
 	end
 end
 
-local forbidden_chars_pattern = "[/%z]";
-if prosody.platform == "windows" then
-	forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]"
+local function get_calling_module()
+	local info = debug.getinfo(3, "S");
+	if not info then return "An unknown module"; end
+	return info.source:match"mod_[^/\\.]+" or info.short_src;
 end
 
-local urldecode = require "util.http".urldecode;
-function sanitize_path(path)
-	if not path then return end
-	local out = {};
-
-	local c = 0;
-	for component in path:gmatch("([^/]+)") do
-		component = urldecode(component);
-		if component:find(forbidden_chars_pattern) then
-			return nil;
-		elseif component == ".." then
-			if c <= 0 then
-				return nil;
-			end
-			out[c] = nil;
-			c = c - 1;
-		elseif component ~= "." then
-			c = c + 1;
-			out[c] = component;
-		end
-	end
-	if path:sub(-1,-1) == "/" then
-		out[c+1] = "";
-	end
-	return "/"..table.concat(out, "/");
-end
-
-local cache = require "util.cache".new(cache_size);
-
+-- COMPAT -- TODO deprecate
 function serve(opts)
 	if type(opts) ~= "table" then -- assume path string
 		opts = { path = opts };
 	end
-	-- luacheck: ignore 431
-	local base_path = opts.path;
-	local dir_indices = opts.index_files or dir_indices;
-	local directory_index = opts.directory_index;
-	local function serve_file(event, path)
-		local request, response = event.request, event.response;
-		local sanitized_path = sanitize_path(path);
-		if path and not sanitized_path then
-			return 400;
-		end
-		path = sanitized_path;
-		local orig_path = sanitize_path(request.path);
-		local full_path = base_path .. (path or ""):gsub("/", path_sep);
-		local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows
-		if not attr then
-			return 404;
-		end
-
-		local request_headers, response_headers = request.headers, response.headers;
-
-		local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification);
-		response_headers.last_modified = last_modified;
-
-		local etag = ('"%x-%x-%x"'):format(attr.change or 0, attr.size or 0, attr.modification or 0);
-		response_headers.etag = etag;
-
-		local if_none_match = request_headers.if_none_match
-		local if_modified_since = request_headers.if_modified_since;
-		if etag == if_none_match
-		or (not if_none_match and last_modified == if_modified_since) then
-			return 304;
-		end
-
-		local data = cache:get(orig_path);
-		if data and data.etag == etag then
-			response_headers.content_type = data.content_type;
-			data = data.data;
-		elseif attr.mode == "directory" and path then
-			if full_path:sub(-1) ~= "/" then
-				local dir_path = { is_absolute = true, is_directory = true };
-				for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end
-				response_headers.location = build_path(dir_path);
-				return 301;
-			end
-			for i=1,#dir_indices do
-				if stat(full_path..dir_indices[i], "mode") == "file" then
-					return serve_file(event, path..dir_indices[i]);
-				end
-			end
-
-			if directory_index then
-				data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path });
-			end
-			if not data then
-				return 403;
-			end
-			cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; });
-			response_headers.content_type = mime_map.html;
-
-		else
-			local f, err = open(full_path, "rb");
-			if not f then
-				module:log("debug", "Could not open %s. Error was %s", full_path, err);
-				return 403;
-			end
-			local ext = full_path:match("%.([^./]+)$");
-			local content_type = ext and mime_map[ext];
-			response_headers.content_type = content_type;
-			if attr.size > cache_max_file_size then
-				response_headers.content_length = attr.size;
-				module:log("debug", "%d > cache_max_file_size", attr.size);
-				return response:send_file(f);
-			else
-				data = f:read("*a");
-				f:close();
-			end
-			cache:set(orig_path, { data = data; content_type = content_type; etag = etag });
-		end
-
-		return response:send(data);
+	if opts.directory_index == nil then
+		opts.directory_index = directory_index;
 	end
-
-	return serve_file;
+	if opts.mime_map == nil then
+		opts.mime_map = mime_map;
+	end
+	if opts.cache_size == nil then
+		opts.cache_size = cache_size;
+	end
+	if opts.cache_max_file_size == nil then
+		opts.cache_max_file_size = cache_max_file_size;
+	end
+	if opts.index_files == nil then
+		opts.index_files = dir_indices;
+	end
+	-- TODO Crank up to warning
+	module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module());
+	return fileserver.serve(opts);
 end
 
 function wrap_route(routes)
+	module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module());
 	for route,handler in pairs(routes) do
 		if type(handler) ~= "function" then
-			routes[route] = serve(handler);
+			routes[route] = fileserver.serve(handler);
 		end
 	end
 	return routes;
 end
 
-if base_path then
-	module:provides("http", {
-		route = {
-			["GET /*"] = serve {
-				path = base_path;
-				directory_index = directory_index;
-			}
-		};
-	});
-else
-	module:log("debug", "http_files_dir not set, assuming use by some other module");
-end
-
+module:provides("http", {
+	route = {
+		["GET /*"] = fileserver.serve({
+			path = base_path;
+			directory_index = directory_index;
+			mime_map = mime_map;
+			cache_size = cache_size;
+			cache_max_file_size = cache_max_file_size;
+			index_files = dir_indices;
+		});
+	};
+});
diff --git a/plugins/mod_lastactivity.lua b/plugins/mod_lastactivity.lua
index 575e66be..91d11bd2 100644
--- a/plugins/mod_lastactivity.lua
+++ b/plugins/mod_lastactivity.lua
@@ -30,7 +30,7 @@ module:hook("iq-get/bare/jabber:iq:last:query", function(event)
 	if not stanza.attr.to or is_contact_subscribed(username, module.host, jid_bare(stanza.attr.from)) then
 		local seconds, text = "0", "";
 		if map[username] then
-			seconds = tostring(os.difftime(os.time(), map[username].t));
+			seconds = string.format("%d", os.difftime(os.time(), map[username].t));
 			text = map[username].s;
 		end
 		origin.send(st.reply(stanza):tag('query', {xmlns='jabber:iq:last', seconds=seconds}):text(text));
diff --git a/plugins/mod_legacyauth.lua b/plugins/mod_legacyauth.lua
index 0f41d3e7..941806d3 100644
--- a/plugins/mod_legacyauth.lua
+++ b/plugins/mod_legacyauth.lua
@@ -78,8 +78,10 @@ module:hook("stanza/iq/jabber:iq:auth:query", function(event)
 					session:close(); -- FIXME undo resource bind and auth instead of closing the session?
 					return true;
 				end
+				session.send(st.reply(stanza));
+			else
+				session.send(st.error_reply(stanza, "auth", "not-authorized", err));
 			end
-			session.send(st.reply(stanza));
 		else
 			session.send(st.error_reply(stanza, "auth", "not-authorized"));
 		end
diff --git a/plugins/mod_limits.lua b/plugins/mod_limits.lua
index 914d5c44..024ab686 100644
--- a/plugins/mod_limits.lua
+++ b/plugins/mod_limits.lua
@@ -32,7 +32,7 @@ local function parse_burst(burst, sess_type)
 	end
 	local n_burst = tonumber(burst);
 	if not n_burst then
-		module:log("error", "Unable to parse burst for %s: %q, using default burst interval (%ds)", sess_type, tostring(burst), default_burst);
+		module:log("error", "Unable to parse burst for %s: %q, using default burst interval (%ds)", sess_type, burst, default_burst);
 	end
 	return n_burst or default_burst;
 end
@@ -51,18 +51,18 @@ end
 local default_filter_set = {};
 
 function default_filter_set.bytes_in(bytes, session)
-  local sess_throttle = session.throttle;
-  if sess_throttle then
-    local ok, balance, outstanding = sess_throttle:poll(#bytes, true);
+	local sess_throttle = session.throttle;
+	if sess_throttle then
+		local ok, _, outstanding = sess_throttle:poll(#bytes, true);
 		if not ok then
-      session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding);
+			session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding);
 			outstanding = ceil(outstanding);
 			session.conn:pause(); -- Read no more data from the connection until there is no outstanding data
 			local outstanding_data = bytes:sub(-outstanding);
 			bytes = bytes:sub(1, #bytes-outstanding);
 			timer.add_task(limits_resolution, function ()
 				if not session.conn then return; end
-        if sess_throttle:peek(#outstanding_data) then
+				if sess_throttle:peek(#outstanding_data) then
 					session.log("debug", "Resuming paused session");
 					session.conn:resume();
 				end
@@ -84,8 +84,13 @@ local function filter_hook(session)
 	local session_type = session.type:match("^[^_]+");
 	local filter_set, opts = type_filters[session_type], limits[session_type];
 	if opts then
-		session.throttle = throttle.create(opts.bytes_per_second * opts.burst_seconds, opts.burst_seconds);
-		filters.add_filter(session, "bytes/in", filter_set.bytes_in, 1000);
+		if session.conn and session.conn.setlimit then
+			session.conn:setlimit(opts.bytes_per_second);
+			-- Currently no burst support
+		else
+			session.throttle = throttle.create(opts.bytes_per_second * opts.burst_seconds, opts.burst_seconds);
+			filters.add_filter(session, "bytes/in", filter_set.bytes_in, 1000);
+		end
 	end
 end
 
@@ -96,3 +101,25 @@ end
 function module.unload()
 	filters.remove_filter_hook(filter_hook);
 end
+
+function module.add_host(module)
+	local unlimited_jids = module:get_option_inherited_set("unlimited_jids", {});
+
+	if not unlimited_jids:empty() then
+		module:hook("authentication-success", function (event)
+			local session = event.session;
+			local session_type = session.type:match("^[^_]+");
+			local jid = session.username .. "@" .. session.host;
+			if unlimited_jids:contains(jid) then
+				if session.conn and session.conn.setlimit then
+					session.conn:setlimit(0);
+					-- Currently no burst support
+				else
+					local filter_set = type_filters[session_type];
+					filters.remove_filter(session, "bytes/in", filter_set.bytes_in);
+					session.throttle = nil;
+				end
+			end
+		end);
+	end
+end
diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index e7d89a95..4fe714eb 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -25,6 +25,7 @@ local jid_bare = require "util.jid".bare;
 local jid_split = require "util.jid".split;
 local jid_prepped_split = require "util.jid".prepped_split;
 local dataform = require "util.dataforms".new;
+local get_form_type = require "util.dataforms".get_type;
 local host = module.host;
 
 local rm_load_roster = require "core.rostermanager".load_roster;
@@ -40,6 +41,11 @@ local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://ja
 local archive_store = module:get_option_string("archive_store", "archive");
 local archive = module:open_store(archive_store, "archive");
 
+local cleanup_after = module:get_option_string("archive_expires_after", "1w");
+local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
+local archive_truncate = math.floor(archive_item_limit * 0.99);
+
 if not archive.find then
 	error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n"
 		.."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information");
@@ -98,7 +104,14 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local qwith, qstart, qend;
 	local form = query:get_child("x", "jabber:x:data");
 	if form then
-		local err;
+		local form_type, err = get_form_type(form);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid dataform: "..err));
+			return true;
+		elseif form_type ~= xmlns_mam then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'"));
+			return true;
+		end
 		form, err = query_form:data(form);
 		if err then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
@@ -117,10 +130,12 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query, id %s with %s from %s until %s",
-		tostring(qid), qwith or "anyone",
-		qstart and timestamp(qstart) or "the dawn of time",
-		qend and timestamp(qend) or "now");
+	module:log("debug", "Archive query by %s id=%s with=%s when=%s...%s",
+		origin.username,
+		qid or stanza.attr.id,
+		qwith or "*",
+		qstart and timestamp(qstart) or "",
+		qend and timestamp(qend) or "");
 
 	-- RSM stuff
 	local qset = rsm.get(query);
@@ -128,6 +143,9 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local reverse = qset and qset.before or false;
 	local before, after = qset and qset.before, qset and qset.after;
 	if type(before) ~= "string" then before = nil; end
+	if qset then
+		module:log("debug", "Archive query id=%s rsm=%q", qid or stanza.attr.id, qset);
+	end
 
 	-- Load all the data!
 	local data, err = archive:find(origin.username, {
@@ -140,7 +158,12 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
-		origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);
+		if err == "item-not-found" then
+			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
+		else
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		end
 		return true;
 	end
 	local total = tonumber(err);
@@ -189,13 +212,13 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		first, last = last, first;
 	end
 
-	-- That's all folks!
-	module:log("debug", "Archive query %s completed", tostring(qid));
-
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 			:add_child(rsm.generate {
 				first = first, last = last, count = total }));
+
+	-- That's all folks!
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);
 	return true;
 end);
 
@@ -213,13 +236,13 @@ local function shall_store(user, who)
 	end
 	local prefs = get_prefs(user);
 	local rule = prefs[who];
-	module:log("debug", "%s's rule for %s is %s", user, who, tostring(rule));
+	module:log("debug", "%s's rule for %s is %s", user, who, rule);
 	if rule ~= nil then
 		return rule;
 	end
 	-- Below could be done by a metatable
 	local default = prefs[false];
-	module:log("debug", "%s's default rule is %s", user, tostring(default));
+	module:log("debug", "%s's default rule is %s", user, default);
 	if default == "roster" then
 		return has_in_roster(user, who);
 	end
@@ -242,11 +265,70 @@ local function strip_stanza_id(stanza, user)
 	return stanza;
 end
 
+local function should_store(stanza, c2s) --> boolean, reason: string
+	local st_type = stanza.attr.type or "normal";
+	-- FIXME pass direction of stanza and use that along with bare/full JID addressing
+	-- for more accurate MUC / type=groupchat check
+
+	if st_type == "headline" then
+		-- Headline messages are ephemeral by definition
+		return false, "headline";
+	end
+	if st_type == "error" and not c2s then
+		-- Store delivery failure notifications so you know if your own messages were not delivered
+		return true, "bounce";
+	end
+	if st_type == "groupchat" then
+		-- MUC messages always go to the full JID, usually archived by the MUC
+		return false, "groupchat";
+	end
+	if stanza:get_child("no-store", "urn:xmpp:hints")
+	or stanza:get_child("no-permanent-store", "urn:xmpp:hints") then
+		-- XXX Experimental XEP
+		return false, "hint";
+	end
+	if stanza:get_child("store", "urn:xmpp:hints") then
+		return true, "hint";
+	end
+	if stanza:get_child("body") then
+		return true, "body";
+	end
+	if stanza:get_child("subject") then
+		-- XXX Who would send a message with a subject but without a body?
+		return true, "subject";
+	end
+	if stanza:get_child("encryption", "urn:xmpp:eme:0") then
+		-- Since we can't know what an encrypted message contains, we assume it's important
+		-- XXX Experimental XEP
+		return true, "encrypted";
+	end
+	if stanza:get_child(nil, "urn:xmpp:receipts") then
+		-- If it's important enough to ask for a receipt then it's important enough to archive
+		-- and the same applies to the receipt
+		return true, "receipt";
+	end
+	if stanza:get_child(nil, "urn:xmpp:chat-markers:0") then
+		-- XXX Experimental XEP
+		return true, "marker";
+	end
+	if stanza:get_child("x", "jabber:x:conference")
+	or stanza:find("{http://jabber.org/protocol/muc#user}x/invite") then
+		return true, "invite";
+	end
+	if stanza:get_child(nil, "urn:xmpp:jingle-message:0") then
+		-- XXX Experimental XEP stuck in Proposed for almost a year at the time of this comment
+		return true, "jingle call";
+	end
+
+	 -- The IM-NG thing to do here would be to return `not st_to_full`
+	 -- One day ...
+	return false, "default";
+end
+
 -- Handle messages
 local function message_handler(event, c2s)
 	local origin, stanza = event.origin, event.stanza;
 	local log = c2s and origin.log or module._log;
-	local orig_type = stanza.attr.type or "normal";
 	local orig_from = stanza.attr.from;
 	local orig_to = stanza.attr.to or orig_from;
 	-- Stanza without 'to' are treated as if it was to their own bare jid
@@ -259,21 +341,12 @@ local function message_handler(event, c2s)
 	-- Filter out <stanza-id> that claim to be from us
 	event.stanza = strip_stanza_id(stanza, store_user);
 
-	-- We store chat messages or normal messages that have a body
-	if not(orig_type == "chat" or (orig_type == "normal" and stanza:get_child("body")) ) then
-		log("debug", "Not archiving stanza: %s (type)", stanza:top_tag());
+	local should, why = should_store(stanza, c2s);
+	if not should then
+		log("debug", "Not archiving stanza: %s (%s)", stanza:top_tag(), why);
 		return;
 	end
 
-	-- or if hints suggest we shouldn't
-	if not stanza:get_child("store", "urn:xmpp:hints") then -- No hint telling us we should store
-		if stanza:get_child("no-permanent-store", "urn:xmpp:hints")
-			or stanza:get_child("no-store", "urn:xmpp:hints") then -- Hint telling us we should NOT store
-			log("debug", "Not archiving stanza: %s (hint)", stanza:top_tag());
-			return;
-		end
-	end
-
 	local clone_for_storage;
 	if not strip_tags:empty() then
 		clone_for_storage = st.clone(stanza);
@@ -294,10 +367,31 @@ local function message_handler(event, c2s)
 
 	-- Check with the users preferences
 	if shall_store(store_user, with) then
-		log("debug", "Archiving stanza: %s", stanza:top_tag());
+		log("debug", "Archiving stanza: %s (%s)", stanza:top_tag(), why);
 
 		-- And stash it
-		local ok, err = archive:append(store_user, nil, clone_for_storage, time_now(), with);
+		local time = time_now();
+		local ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+		if not ok and err == "quota-limit" then
+			if type(cleanup_after) == "number" then
+				module:log("debug", "User '%s' over quota, cleaning archive", store_user);
+				local cleaned = archive:delete(store_user, {
+					["end"] = (os.time() - cleanup_after);
+				});
+				if cleaned then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+			if not ok and (archive.caps and archive.caps.truncate) then
+				module:log("debug", "User '%s' over quota, truncating archive", store_user);
+				local truncated = archive:delete(store_user, {
+					truncate = archive_truncate;
+				});
+				if truncated then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+		end
 		if ok then
 			local clone_for_other_handlers = st.clone(stanza);
 			local id = ok;
@@ -325,8 +419,6 @@ end
 module:hook("pre-message/bare", strip_stanza_id_after_other_events, -1);
 module:hook("pre-message/full", strip_stanza_id_after_other_events, -1);
 
-local cleanup_after = module:get_option_string("archive_expires_after", "1w");
-local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
 if cleanup_after ~= "never" then
 	local cleanup_storage = module:open_store("archive_cleanup");
 	local cleanup_map = module:open_store("archive_cleanup", "map");
@@ -361,9 +453,11 @@ if cleanup_after ~= "never" then
 			last_date:set(username, date);
 		end
 	end
+	local cleanup_time = module:measure("cleanup", "times");
 
 	local async = require "util.async";
 	cleanup_runner = async.runner(function ()
+		local cleanup_done = cleanup_time();
 		local users = {};
 		local cut_off = datestamp(os.time() - cleanup_after);
 		for date in cleanup_storage:users() do
@@ -397,6 +491,7 @@ if cleanup_after ~= "never" then
 			wait();
 		end
 		module:log("info", "Deleted %d expired messages for %d users", sum, num_users);
+		cleanup_done();
 	end);
 
 	cleanup_task = module:add_timer(1, function ()
diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua
new file mode 100644
index 00000000..b586a70c
--- /dev/null
+++ b/plugins/mod_mimicking.lua
@@ -0,0 +1,85 @@
+-- Prosody IM
+-- Copyright (C) 2012 Florian Zeitz
+-- Copyright (C) 2019 Kim Alvefur
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local encodings = require "util.encodings";
+assert(encodings.confusable, "This module requires that Prosody be built with ICU");
+local skeleton = encodings.confusable.skeleton;
+
+local usage = require "util.prosodyctl".show_usage;
+local usermanager = require "core.usermanager";
+local storagemanager = require "core.storagemanager";
+
+local skeletons
+function module.load()
+	if module.host ~= "*" then
+		skeletons = module:open_store("skeletons");
+	end
+end
+
+module:hook("user-registered", function(user)
+	local skel = skeleton(user.username);
+	local ok, err = skeletons:set(skel, { username = user.username });
+	if not ok then
+		module:log("error", "Unable to store mimicry data (%q => %q): %s", user.username, skel, err);
+	end
+end);
+
+module:hook("user-deleted", function(user)
+	local skel = skeleton(user.username);
+	local ok, err = skeletons:set(skel, nil);
+	if not ok and err then
+		module:log("error", "Unable to clear mimicry data (%q): %s", skel, err);
+	end
+end);
+
+module:hook("user-registering", function(user)
+	local existing, err = skeletons:get(skeleton(user.username));
+	if existing then
+		module:log("debug", "Attempt to register username '%s' which could be confused with '%s'", user.username, existing.username);
+		user.allowed = false;
+	elseif err then
+		module:log("error", "Unable to check if new username '%s' can be confused with any existing user: %s", err);
+	end
+end);
+
+function module.command(arg)
+	if (arg[1] ~= "bootstrap" or not arg[2]) then
+		usage("mod_mimicking bootstrap <host>", "Initialize username mimicry database");
+		return;
+	end
+
+	local host = arg[2];
+
+	local host_session = prosody.hosts[host];
+	if not host_session then
+		return "No such host";
+	end
+
+	storagemanager.initialize_host(host);
+	usermanager.initialize_host(host);
+
+	skeletons = storagemanager.open(host, "skeletons");
+
+	local count = 0;
+	for user in usermanager.users(host) do
+		local skel = skeleton(user);
+		local existing, err = skeletons:get(skel);
+		if existing and existing.username ~= user then
+			module:log("warn", "Existing usernames '%s' and '%s' are confusable", existing.username, user);
+		elseif err then
+			module:log("error", "Error checking for existing mimicry data (%q = %q): %s", user, skel, err);
+		end
+		local ok, err = skeletons:set(skel, { username = user });
+		if ok then
+			count = count + 1;
+		elseif err then
+			module:log("error", "Unable to store mimicry data (%q => %q): %s", user, skel, err);
+		end
+	end
+	module:log("info", "%d usernames indexed", count);
+end
diff --git a/plugins/mod_muc_mam.lua b/plugins/mod_muc_mam.lua
index f0417889..65794cf4 100644
--- a/plugins/mod_muc_mam.lua
+++ b/plugins/mod_muc_mam.lua
@@ -4,7 +4,7 @@
 -- This file is MIT/X11 licensed.
 
 if module:get_host_type() ~= "component" then
-	module:log("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name);
+	module:log_status("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name);
 	return;
 end
 
@@ -21,6 +21,7 @@ local jid_bare = require "util.jid".bare;
 local jid_split = require "util.jid".split;
 local jid_prep = require "util.jid".prep;
 local dataform = require "util.dataforms".new;
+local get_form_type = require "util.dataforms".get_type;
 
 local mod_muc = module:depends"muc";
 local get_room_from_jid = mod_muc.get_room_from_jid;
@@ -32,6 +33,9 @@ local m_min = math.min;
 local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date");
 local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50);
 
+local cleanup_after = module:get_option_string("muc_log_expires_after", "1w");
+local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60);
+
 local default_history_length = 20;
 local max_history_length = module:get_option_number("max_history_messages", math.huge);
 
@@ -49,6 +53,9 @@ local log_by_default = module:get_option_boolean("muc_log_by_default", true);
 local archive_store = "muc_log";
 local archive = module:open_store(archive_store, "archive");
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
+local archive_truncate = math.floor(archive_item_limit * 0.99);
+
 if archive.name == "null" or not archive.find then
 	if not archive.find then
 		module:log("error", "Attempt to open archive storage returned a driver without archive API support");
@@ -63,12 +70,15 @@ end
 
 local function archiving_enabled(room)
 	if log_all_rooms then
+		module:log("debug", "Archiving all rooms");
 		return true;
 	end
 	local enabled = room._data.archiving;
 	if enabled == nil then
+		module:log("debug", "Default is %s (for %s)", log_by_default, room.jid);
 		return log_by_default;
 	end
+	module:log("debug", "Logging in room %s is %s", room.jid, enabled);
 	return enabled;
 end
 
@@ -135,7 +145,14 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 	local qstart, qend;
 	local form = query:get_child("x", "jabber:x:data");
 	if form then
-		local err;
+		local form_type, err = get_form_type(form);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid dataform: "..err));
+			return true;
+		elseif form_type ~= xmlns_mam then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'"));
+			return true;
+		end
 		form, err = query_form:data(form);
 		if err then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
@@ -153,10 +170,11 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query id %s from %s until %s)",
-		tostring(qid),
-		qstart and timestamp(qstart) or "the dawn of time",
-		qend and timestamp(qend) or "now");
+	module:log("debug", "Archive query by %s id=%s when=%s...%s",
+		origin.username,
+		qid or stanza.attr.id,
+		qstart and timestamp(qstart) or "",
+		qend and timestamp(qend) or "");
 
 	-- RSM stuff
 	local qset = rsm.get(query);
@@ -165,6 +183,9 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 
 	local before, after = qset and qset.before, qset and qset.after;
 	if type(before) ~= "string" then before = nil; end
+	if qset then
+		module:log("debug", "Archive query id=%s rsm=%q", qid or stanza.attr.id, qset);
+	end
 
 	-- Load all the data!
 	local data, err = archive:find(room_node, {
@@ -176,7 +197,12 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
-		origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);
+		if err == "item-not-found" then
+			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
+		else
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		end
 		return true;
 	end
 	local total = tonumber(err);
@@ -233,13 +259,14 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 		first, last = last, first;
 	end
 
-	-- That's all folks!
-	module:log("debug", "Archive query %s completed", tostring(qid));
 
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 			:add_child(rsm.generate {
 				first = first, last = last, count = total }));
+
+	-- That's all folks!
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);
 	return true;
 end);
 
@@ -274,7 +301,7 @@ module:hook("muc-get-history", function (event)
 	local data, err = archive:find(jid_split(room_jid), query);
 
 	if not data then
-		module:log("error", "Could not fetch history: %s", tostring(err));
+		module:log("error", "Could not fetch history: %s", err);
 		return
 	end
 
@@ -300,7 +327,7 @@ module:hook("muc-get-history", function (event)
 			maxchars = maxchars - chars;
 		end
 		history[i], i = item, i+1;
-		-- module:log("debug", tostring(item));
+		-- module:log("debug", item);
 	end
 	function event.next_stanza()
 		i = i - 1;
@@ -325,7 +352,7 @@ end, 1);
 
 -- Handle messages
 local function save_to_history(self, stanza)
-	local room_node, room_host = jid_split(self.jid);
+	local room_node = jid_split(self.jid);
 
 	local stored_stanza = stanza;
 
@@ -352,7 +379,29 @@ local function save_to_history(self, stanza)
 	end
 
 	-- And stash it
-	local id, err = archive:append(room_node, nil, stored_stanza, time_now(), with);
+	local time = time_now();
+	local id, err = archive:append(room_node, nil, stored_stanza, time, with);
+
+	if not id and err == "quota-limit" then
+		if type(cleanup_after) == "number" then
+			module:log("debug", "Room '%s' over quota, cleaning archive", room_node);
+			local cleaned = archive:delete(room_node, {
+				["end"] = (os.time() - cleanup_after);
+			});
+			if cleaned then
+				id, err = archive:append(room_node, nil, stored_stanza, time, with);
+			end
+		end
+		if not id and (archive.caps and archive.caps.truncate) then
+			module:log("debug", "User '%s' over quota, truncating archive", room_node);
+			local truncated = archive:delete(room_node, {
+				truncate = archive_truncate;
+			});
+			if truncated then
+				id, err = archive:append(room_node, nil, stored_stanza, time, with);
+			end
+		end
+	end
 
 	if id then
 		schedule_cleanup(room_node);
@@ -391,15 +440,14 @@ end
 module:add_feature(xmlns_mam);
 
 module:hook("muc-disco#info", function(event)
-	event.reply:tag("feature", {var=xmlns_mam}):up();
+	if archiving_enabled(event.room) then
+		event.reply:tag("feature", {var=xmlns_mam}):up();
+	end
 	event.reply:tag("feature", {var=xmlns_st_id}):up();
 end);
 
 -- Cleanup
 
-local cleanup_after = module:get_option_string("muc_log_expires_after", "1w");
-local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60);
-
 if cleanup_after ~= "never" then
 	local cleanup_storage = module:open_store("muc_log_cleanup");
 	local cleanup_map = module:open_store("muc_log_cleanup", "map");
@@ -435,8 +483,11 @@ if cleanup_after ~= "never" then
 		end
 	end
 
+	local cleanup_time = module:measure("cleanup", "times");
+
 	local async = require "util.async";
 	cleanup_runner = async.runner(function ()
+		local cleanup_done = cleanup_time();
 		local rooms = {};
 		local cut_off = datestamp(os.time() - cleanup_after);
 		for date in cleanup_storage:users() do
@@ -470,6 +521,7 @@ if cleanup_after ~= "never" then
 			wait();
 		end
 		module:log("info", "Deleted %d expired messages for %d rooms", sum, num_rooms);
+		cleanup_done();
 	end);
 
 	cleanup_task = module:add_timer(1, function ()
diff --git a/plugins/mod_net_multiplex.lua b/plugins/mod_net_multiplex.lua
index 8ef77883..ddd58463 100644
--- a/plugins/mod_net_multiplex.lua
+++ b/plugins/mod_net_multiplex.lua
@@ -1,22 +1,38 @@
 module:set_global();
 
+local array = require "util.array";
 local max_buffer_len = module:get_option_number("multiplex_buffer_size", 1024);
+local default_mode = module:get_option_number("network_default_read_size", 4096);
 
 local portmanager = require "core.portmanager";
 
 local available_services = {};
+local service_by_protocol = {};
+local available_protocols = array();
 
 local function add_service(service)
 	local multiplex_pattern = service.multiplex and service.multiplex.pattern;
+	local protocol_name = service.multiplex and service.multiplex.protocol;
+	if protocol_name then
+		module:log("debug", "Adding multiplex service %q with protocol %q", service.name, protocol_name);
+		service_by_protocol[protocol_name] = service;
+		available_protocols:push(protocol_name);
+	end
 	if multiplex_pattern then
 		module:log("debug", "Adding multiplex service %q with pattern %q", service.name, multiplex_pattern);
 		available_services[service] = multiplex_pattern;
-	else
+	elseif not protocol_name then
 		module:log("debug", "Service %q is not multiplex-capable", service.name);
 	end
 end
 module:hook("service-added", function (event) add_service(event.service); end);
-module:hook("service-removed", function (event)	available_services[event.service] = nil; end);
+module:hook("service-removed", function (event)
+	available_services[event.service] = nil;
+	if event.service.multiplex and event.service.multiplex.protocol then
+		available_protocols:filter(function (p) return p ~= event.service.multiplex.protocol end);
+		service_by_protocol[event.service.multiplex.protocol] = nil;
+	end
+end);
 
 for _, services in pairs(portmanager.get_registered_services()) do
 	for _, service in ipairs(services) do
@@ -26,9 +42,22 @@ end
 
 local buffers = {};
 
-local listener = { default_mode = "*a" };
+local listener = { default_mode = max_buffer_len };
 
-function listener.onconnect()
+function listener.onconnect(conn)
+	local sock = conn:socket();
+	if sock.getalpn then
+		local selected_proto = sock:getalpn();
+		local service = service_by_protocol[selected_proto];
+		if service then
+			module:log("debug", "Routing incoming connection to %s based on ALPN %q", service.name, selected_proto);
+			local next_listener = service.listener;
+			conn:setlistener(next_listener);
+			conn:set_mode(next_listener.default_mode or default_mode);
+			local onconnect = next_listener.onconnect;
+			if onconnect then return onconnect(conn) end
+		end
+	end
 end
 
 function listener.onincoming(conn, data)
@@ -40,6 +69,7 @@ function listener.onincoming(conn, data)
 			module:log("debug", "Routing incoming connection to %s", service.name);
 			local next_listener = service.listener;
 			conn:setlistener(next_listener);
+			conn:set_mode(next_listener.default_mode or default_mode);
 			local onconnect = next_listener.onconnect;
 			if onconnect then onconnect(conn) end
 			return next_listener.onincoming(conn, buf);
@@ -68,5 +98,10 @@ module:provides("net", {
 	name = "multiplex_ssl";
 	config_prefix = "ssl";
 	encryption = "ssl";
+	ssl_config = {
+		alpn = function ()
+			return available_protocols;
+		end;
+	};
 	listener = listener;
 });
diff --git a/plugins/mod_offline.lua b/plugins/mod_offline.lua
index 487098d1..dffe8357 100644
--- a/plugins/mod_offline.lua
+++ b/plugins/mod_offline.lua
@@ -24,11 +24,16 @@ module:hook("message/offline/handle", function(event)
 		node = origin.username;
 	end
 
-	return offline_messages:append(node, nil, stanza, os.time(), "");
+	local ok = offline_messages:append(node, nil, stanza, os.time(), "");
+	if ok then
+		module:log("debug", "Saved to offline storage: %s", stanza:top_tag());
+	end
+	return ok;
 end, -1);
 
 module:hook("message/offline/broadcast", function(event)
 	local origin = event.origin;
+	origin.log("debug", "Broadcasting offline messages");
 
 	local node, host = origin.username, origin.host;
 
@@ -38,6 +43,9 @@ module:hook("message/offline/broadcast", function(event)
 		stanza:tag("delay", {xmlns = "urn:xmpp:delay", from = host, stamp = datetime.datetime(when)}):up(); -- XEP-0203
 		origin.send(stanza);
 	end
-	offline_messages:delete(node);
+	local ok = offline_messages:delete(node);
+	if type(ok) == "number" and ok > 0 then
+		origin.log("debug", "%d offline messages consumed");
+	end
 	return true;
 end, -1);
diff --git a/plugins/mod_pep.lua b/plugins/mod_pep.lua
index f8047a92..db4fdc87 100644
--- a/plugins/mod_pep.lua
+++ b/plugins/mod_pep.lua
@@ -8,6 +8,7 @@ local calculate_hash = require "util.caps".calculate_hash;
 local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed;
 local cache = require "util.cache";
 local set = require "util.set";
+local new_id = require "util.id".medium;
 local storagemanager = require "core.storagemanager";
 
 local xmlns_pubsub = "http://jabber.org/protocol/pubsub";
@@ -123,9 +124,6 @@ local function get_broadcaster(username)
 		if kind == "retract" then
 			kind = "items"; -- XEP-0060 signals retraction in an <items> container
 		end
-		local message = st.message({ from = user_bare, type = "headline" })
-			:tag("event", { xmlns = xmlns_pubsub_event })
-				:tag(kind, { node = node });
 		if item then
 			item = st.clone(item);
 			item.attr.xmlns = nil; -- Clear the pubsub namespace
@@ -134,10 +132,19 @@ local function get_broadcaster(username)
 					item:maptags(function () return nil; end);
 				end
 			end
+		end
+
+		local id = new_id();
+		local message = st.message({ from = user_bare, type = "headline", id = id })
+			:tag("event", { xmlns = xmlns_pubsub_event })
+				:tag(kind, { node = node });
+
+		if item then
 			message:add_child(item);
 		end
+
 		for jid in pairs(jids) do
-			module:log("debug", "Sending notification to %s from %s: %s", jid, user_bare, tostring(item));
+			module:log("debug", "Sending notification to %s from %s for node %s", jid, user_bare, node);
 			message.attr.to = jid;
 			module:send(message);
 		end
@@ -166,12 +173,12 @@ local function get_subscriber_filter(username)
 end
 
 function get_pep_service(username)
-	module:log("debug", "get_pep_service(%q)", username);
 	local user_bare = jid_join(username, host);
 	local service = services[username];
 	if service then
 		return service;
 	end
+	module:log("debug", "Creating pubsub service for user %q", username);
 	service = pubsub.new({
 		pep_username = username;
 		node_defaults = {
@@ -238,8 +245,6 @@ end
 module:hook("iq/bare/"..xmlns_pubsub..":pubsub", handle_pubsub_iq);
 module:hook("iq/bare/"..xmlns_pubsub_owner..":pubsub", handle_pubsub_iq);
 
-module:add_identity("pubsub", "pep", module:get_option_string("name", "Prosody"));
-module:add_feature("http://jabber.org/protocol/pubsub#publish");
 
 local function get_caps_hash_from_presence(stanza, current)
 	local t = stanza.attr.type;
diff --git a/plugins/mod_pep_simple.lua b/plugins/mod_pep_simple.lua
index f0b5d7ef..e686b99b 100644
--- a/plugins/mod_pep_simple.lua
+++ b/plugins/mod_pep_simple.lua
@@ -14,6 +14,7 @@ local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed
 local pairs = pairs;
 local next = next;
 local type = type;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local calculate_hash = require "util.caps".calculate_hash;
 local core_post_stanza = prosody.core_post_stanza;
 local bare_sessions = prosody.bare_sessions;
@@ -84,6 +85,7 @@ local function publish_all(user, recipient, session)
 	if d and notify then
 		for node in pairs(notify) do
 			if d[node] then
+				-- luacheck: ignore id
 				local id, item = unpack(d[node]);
 				session.send(st.message({from=user, to=recipient, type='headline'})
 					:tag('event', {xmlns='http://jabber.org/protocol/pubsub#event'})
@@ -229,13 +231,13 @@ module:hook("iq/bare/http://jabber.org/protocol/pubsub:pubsub", function(event)
 				return true;
 			else --invalid request
 				session.send(st.error_reply(stanza, 'modify', 'bad-request'));
-				module:log("debug", "Invalid request: %s", tostring(payload));
+				module:log("debug", "Invalid request: %s", payload);
 				return true;
 			end
 		else --no presence subscription
 			session.send(st.error_reply(stanza, 'auth', 'not-authorized')
 				:tag('presence-subscription-required', {xmlns='http://jabber.org/protocol/pubsub#errors'}));
-			module:log("debug", "Unauthorized request: %s", tostring(payload));
+			module:log("debug", "Unauthorized request: %s", payload);
 			return true;
 		end
 	end
diff --git a/plugins/mod_ping.lua b/plugins/mod_ping.lua
index 5fff58d1..b6ccc928 100644
--- a/plugins/mod_ping.lua
+++ b/plugins/mod_ping.lua
@@ -11,23 +11,9 @@ local st = require "util.stanza";
 module:add_feature("urn:xmpp:ping");
 
 local function ping_handler(event)
-	return event.origin.send(st.reply(event.stanza));
+	event.origin.send(st.reply(event.stanza));
+	return true;
 end
 
 module:hook("iq-get/bare/urn:xmpp:ping:ping", ping_handler);
 module:hook("iq-get/host/urn:xmpp:ping:ping", ping_handler);
-
--- Ad-hoc command
-
-local datetime = require "util.datetime".datetime;
-
-function ping_command_handler (self, data, state) -- luacheck: ignore 212
-	local now = datetime();
-	return { info = "Pong\n"..now, status = "completed" };
-end
-
-module:depends "adhoc";
-local adhoc_new = module:require "adhoc".new;
-local descriptor = adhoc_new("Ping", "ping", ping_command_handler);
-module:provides("adhoc", descriptor);
-
diff --git a/plugins/mod_posix.lua b/plugins/mod_posix.lua
index fe826c22..03177530 100644
--- a/plugins/mod_posix.lua
+++ b/plugins/mod_posix.lua
@@ -20,7 +20,6 @@ if not have_signal then
 	module:log("warn", "Couldn't load signal library, won't respond to SIGTERM");
 end
 
-local format = require "util.format".format;
 local lfs = require "lfs";
 local stat = lfs.attributes;
 
@@ -31,39 +30,12 @@ module:set_global(); -- we're a global module
 local umask = module:get_option_string("umask", "027");
 pposix.umask(umask);
 
--- Allow switching away from root, some people like strange ports.
-module:hook("server-started", function ()
-	local uid = module:get_option("setuid");
-	local gid = module:get_option("setgid");
-	if gid then
-		local success, msg = pposix.setgid(gid);
-		if success then
-			module:log("debug", "Changed group to %s successfully.", gid);
-		else
-			module:log("error", "Failed to change group to %s. Error: %s", gid, msg);
-			prosody.shutdown("Failed to change group to %s", gid);
-		end
-	end
-	if uid then
-		local success, msg = pposix.setuid(uid);
-		if success then
-			module:log("debug", "Changed user to %s successfully.", uid);
-		else
-			module:log("error", "Failed to change user to %s. Error: %s", uid, msg);
-			prosody.shutdown("Failed to change user to %s", uid);
-		end
-	end
-end);
-
 -- Don't even think about it!
 if not prosody.start_time then -- server-starting
-	local suid = module:get_option("setuid");
-	if not suid or suid == 0 or suid == "root" then
-		if pposix.getuid() == 0 and not module:get_option_boolean("run_as_root") then
-			module:log("error", "Danger, Will Robinson! Prosody doesn't need to be run as root, so don't do it!");
-			module:log("error", "For more information on running Prosody as root, see https://prosody.im/doc/root");
-			prosody.shutdown("Refusing to run as root");
-		end
+	if pposix.getuid() == 0 and not module:get_option_boolean("run_as_root") then
+		module:log("error", "Danger, Will Robinson! Prosody doesn't need to be run as root, so don't do it!");
+		module:log("error", "For more information on running Prosody as root, see https://prosody.im/doc/root");
+		prosody.shutdown("Refusing to run as root");
 	end
 end
 
@@ -113,24 +85,15 @@ local function write_pidfile()
 	end
 end
 
-local syslog_opened;
-function syslog_sink_maker(config) -- luacheck: ignore 212/config
-	if not syslog_opened then
-		pposix.syslog_open("prosody", module:get_option_string("syslog_facility"));
-		syslog_opened = true;
-	end
-	local syslog = pposix.syslog_log;
-	return function (name, level, message, ...)
-		syslog(level, name, format(message, ...));
-	end;
-end
-require "core.loggingmanager".register_sink_type("syslog", syslog_sink_maker);
-
 local daemonize = prosody.opts.daemonize;
 
 if daemonize == nil then
 	-- Fall back to config file if not specified on command-line
-	daemonize = module:get_option("daemonize", prosody.installed);
+	daemonize = module:get_option_boolean("daemonize", nil);
+	if daemonize ~= nil then
+		module:log("warn", "The 'daemonize' option has been deprecated, specify -D or -F on the command line instead.");
+		-- TODO: Write some docs and include a link in the warning.
+	end
 end
 
 local function remove_log_sinks()
@@ -154,9 +117,7 @@ if daemonize then
 			write_pidfile();
 		end
 	end
-	if not prosody.start_time then -- server-starting
-		daemonize_server();
-	end
+	module:hook("server-started", daemonize_server)
 else
 	-- Not going to daemonize, so write the pid of this process
 	write_pidfile();
@@ -186,5 +147,20 @@ if have_signal then
 			prosody.shutdown("Received SIGINT");
 			prosody.lock_globals();
 		end);
+
+		signal.signal("SIGUSR1", function ()
+			module:log("info", "Received SIGUSR1");
+			module:fire_event("signal/SIGUSR1");
+		end);
+
+		signal.signal("SIGUSR2", function ()
+			module:log("info", "Received SIGUSR2");
+			module:fire_event("signal/SIGUSR2");
+		end);
 	end);
 end
+
+-- For other modules to reference
+features = {
+	signal_events = true;
+};
diff --git a/plugins/mod_presence.lua b/plugins/mod_presence.lua
index 268a2f0c..3f9a0c12 100644
--- a/plugins/mod_presence.lua
+++ b/plugins/mod_presence.lua
@@ -14,6 +14,7 @@ local s_find = string.find;
 local tonumber = tonumber;
 
 local core_post_stanza = prosody.core_post_stanza;
+local core_process_stanza = prosody.core_process_stanza;
 local st = require "util.stanza";
 local jid_split = require "util.jid".split;
 local jid_bare = require "util.jid".bare;
@@ -30,6 +31,14 @@ local recalc_resource_map = require "util.presence".recalc_resource_map;
 
 local ignore_presence_priority = module:get_option_boolean("ignore_presence_priority", false);
 
+local pre_approval_stream_feature = st.stanza("sub", {xmlns="urn:xmpp:features:pre-approval"});
+module:hook("stream-features", function(event)
+	local origin, features = event.origin, event.features;
+	if origin.username then
+		features:add_child(pre_approval_stream_feature);
+	end
+end);
+
 function handle_normal_presence(origin, stanza)
 	if ignore_presence_priority then
 		local priority = stanza:get_child("priority");
@@ -81,8 +90,14 @@ function handle_normal_presence(origin, stanza)
 				res.presence.attr.to = nil;
 			end
 		end
-		for jid in pairs(roster[false].pending) do -- resend incoming subscription requests
-			origin.send(st.presence({type="subscribe", from=jid})); -- TODO add to attribute? Use original?
+		for jid, pending_request in pairs(roster[false].pending) do -- resend incoming subscription requests
+			if type(pending_request) == "table" then
+				local subscribe = st.deserialize(pending_request);
+				subscribe.attr.type, subscribe.attr.from = "subscribe", jid;
+				origin.send(subscribe);
+			else
+				origin.send(st.presence({type="subscribe", from=jid}));
+			end
 		end
 		local request = st.presence({type="subscribe", from=origin.username.."@"..origin.host});
 		for jid, item in pairs(roster) do -- resend outgoing subscription requests
@@ -175,8 +190,10 @@ function handle_outbound_presence_subscriptions_and_probes(origin, stanza, from_
 		if rostermanager.subscribed(node, host, to_bare) then
 			rostermanager.roster_push(node, host, to_bare);
 		end
-		core_post_stanza(origin, stanza);
-		send_presence_of_available_resources(node, host, to_bare, origin);
+		if rostermanager.is_contact_subscribed(node, host, to_bare) then
+			core_post_stanza(origin, stanza);
+			send_presence_of_available_resources(node, host, to_bare, origin);
+		end
 		if rostermanager.is_user_subscribed(node, host, to_bare) then
 			core_post_stanza(origin, st.presence({ type = "probe", from = from_bare, to = to_bare }));
 		end
@@ -184,6 +201,8 @@ function handle_outbound_presence_subscriptions_and_probes(origin, stanza, from_
 		-- 1. send unavailable
 		-- 2. route stanza
 		-- 3. roster push (subscription = from or both)
+		-- luacheck: ignore 211/pending_in
+		-- Is pending_in meant to be used?
 		local success, pending_in, subscribed = rostermanager.unsubscribed(node, host, to_bare);
 		if success then
 			if subscribed then
@@ -223,10 +242,16 @@ function handle_inbound_presence_subscriptions_and_probes(origin, stanza, from_b
 			if 0 == send_presence_of_available_resources(node, host, from_bare, origin) then
 				core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="unavailable"}), true); -- TODO send last activity
 			end
+		elseif rostermanager.is_contact_preapproved(node, host, from_bare) then
+			if not rostermanager.is_contact_pending_in(node, host, from_bare) then
+				if rostermanager.set_contact_pending_in(node, host, from_bare, stanza) then
+					core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="subscribed"}), true);
+				end -- TODO else return error, unable to save
+			end
 		else
 			core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="unavailable"}), true); -- acknowledging receipt
 			if not rostermanager.is_contact_pending_in(node, host, from_bare) then
-				if rostermanager.set_contact_pending_in(node, host, from_bare) then
+				if rostermanager.set_contact_pending_in(node, host, from_bare, stanza) then
 					sessionmanager.send_to_available_resources(node, host, stanza);
 				end -- TODO else return error, unable to save
 			end
@@ -346,7 +371,7 @@ module:hook("resource-unbind", function(event)
 		if err then
 			pres:tag("status"):text("Disconnected: "..err):up();
 		end
-		session:dispatch_stanza(pres);
+		core_process_stanza(session, pres);
 	elseif session.directed then
 		local pres = st.presence{ type = "unavailable", from = session.full_jid };
 		if err then
diff --git a/plugins/mod_proxy65.lua b/plugins/mod_proxy65.lua
index 00833772..bac36b55 100644
--- a/plugins/mod_proxy65.lua
+++ b/plugins/mod_proxy65.lua
@@ -12,7 +12,6 @@ module:set_global();
 local jid_compare, jid_prep = require "util.jid".compare, require "util.jid".prep;
 local st = require "util.stanza";
 local sha1 = require "util.hashes".sha1;
-local b64 = require "util.encodings".base64.encode;
 local server = require "net.server";
 local portmanager = require "core.portmanager";
 
@@ -45,7 +44,7 @@ function listener.onincoming(conn, data)
 		end -- else error, unexpected input
 		conn:write("\5\255"); -- send (SOCKS version 5, no acceptable method)
 		conn:close();
-		module:log("debug", "Invalid SOCKS5 greeting received: '%s'", b64(data));
+		module:log("debug", "Invalid SOCKS5 greeting received: %q", data:sub(1, 300));
 	else -- connection request
 		--local head = string.char( 0x05, 0x01, 0x00, 0x03, 40 ); -- ( VER=5=SOCKS5, CMD=1=CONNECT, RSV=0=RESERVED, ATYP=3=DOMAIMNAME, SHA-1 size )
 		if #data == 47 and data:sub(1,5) == "\5\1\0\3\40" and data:sub(-2) == "\0\0" then
@@ -67,7 +66,7 @@ function listener.onincoming(conn, data)
 		else -- error, unexpected input
 			conn:write("\5\1\0\3\0\0\0"); -- VER, REP, RSV, ATYP, BND.ADDR (sha), BND.PORT (2 Byte)
 			conn:close();
-			module:log("debug", "Invalid SOCKS5 negotiation received: '%s'", b64(data));
+			module:log("debug", "Invalid SOCKS5 negotiation received: %q", data:sub(1, 300));
 		end
 	end
 end
@@ -117,7 +116,7 @@ function module.add_host(module)
 				if jid_compare(jid, acl) then allow = true; break; end
 			end
 			if allow then break; end
-			module:log("warn", "Denying use of proxy for %s", tostring(stanza.attr.from));
+			module:log("warn", "Denying use of proxy for %s", stanza.attr.from);
 			origin.send(st.error_reply(stanza, "auth", "forbidden"));
 			return true;
 		end
diff --git a/plugins/mod_pubsub/mod_pubsub.lua b/plugins/mod_pubsub/mod_pubsub.lua
index b6019c71..de1000b2 100644
--- a/plugins/mod_pubsub/mod_pubsub.lua
+++ b/plugins/mod_pubsub/mod_pubsub.lua
@@ -42,7 +42,7 @@ end
 
 local node_store = module:open_store(module.name.."_nodes");
 
-local function create_simple_itemstore(node_config, node_name)
+local function create_simple_itemstore(node_config, node_name) --> util.cache like object
 	local driver = storagemanager.get_driver(module.host, "pubsub_data");
 	local archive = driver:open("pubsub_"..node_name, "archive");
 	return lib_pubsub.archive_itemstore(archive, node_config, nil, node_name);
@@ -75,14 +75,13 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj)
 	local msg_type = node_obj and node_obj.config.message_type or "headline";
 	local message = st.message({ from = module.host, type = msg_type, id = id })
 		:tag("event", { xmlns = xmlns_pubsub_event })
-			:tag(kind, { node = node })
+			:tag(kind, { node = node });
 
 	if item then
 		message:add_child(item);
 	end
 
 	local summary;
-	-- Compose a sensible textual representation of at least Atom payloads
 	if item and item.tags[1] then
 		local payload = item.tags[1];
 		summary = module:fire_event("pubsub-summary/"..payload.attr.xmlns, {
@@ -101,11 +100,12 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj)
 end
 
 local max_max_items = module:get_option_number("pubsub_max_items", 256);
-function check_node_config(node, actor, new_config) -- luacheck: ignore 212/actor 212/node
+function check_node_config(node, actor, new_config) -- luacheck: ignore 212/node 212/actor
 	if (new_config["max_items"] or 1) > max_max_items then
 		return false;
 	end
-	if new_config["access_model"] ~= "whitelist" and new_config["access_model"] ~= "open" then
+	if new_config["access_model"] ~= "whitelist"
+	and new_config["access_model"] ~= "open" then
 		return false;
 	end
 	return true;
@@ -115,6 +115,7 @@ function is_item_stanza(item)
 	return st.is_stanza(item) and item.attr.xmlns == xmlns_pubsub and item.name == "item" and #item.tags == 1;
 end
 
+-- Compose a textual representation of Atom payloads
 module:hook("pubsub-summary/http://www.w3.org/2005/Atom", function (event)
 	local payload = event.payload;
 	local title = payload:get_child_text("title");
diff --git a/plugins/mod_pubsub/pubsub.lib.lua b/plugins/mod_pubsub/pubsub.lib.lua
index 50ef7ddf..c84208a2 100644
--- a/plugins/mod_pubsub/pubsub.lib.lua
+++ b/plugins/mod_pubsub/pubsub.lib.lua
@@ -7,6 +7,7 @@ local st = require "util.stanza";
 local it = require "util.iterators";
 local uuid_generate = require "util.uuid".generate;
 local dataform = require"util.dataforms".new;
+local errors = require "util.error";
 
 local xmlns_pubsub = "http://jabber.org/protocol/pubsub";
 local xmlns_pubsub_errors = "http://jabber.org/protocol/pubsub#errors";
@@ -34,6 +35,9 @@ local pubsub_errors = {
 };
 local function pubsub_error_reply(stanza, error)
 	local e = pubsub_errors[error];
+	if not e and errors.is_err(error) then
+		e = { error.type, error.condition, error.text, error.pubsub_condition };
+	end
 	local reply = st.error_reply(stanza, t_unpack(e, 1, 3));
 	if e[4] then
 		reply:tag(e[4], { xmlns = xmlns_pubsub_errors }):up();
@@ -185,6 +189,14 @@ local node_metadata_form = dataform {
 		type = "text-single";
 		name = "pubsub#type";
 	};
+	{
+		type = "text-single";
+		name = "pubsub#access_model";
+	};
+	{
+		type = "text-single";
+		name = "pubsub#publish_model";
+	};
 };
 
 local service_method_feature_map = {
@@ -258,6 +270,8 @@ function _M.handle_disco_info_node(event, service)
 			["pubsub#title"] = node_obj.config.title;
 			["pubsub#description"] = node_obj.config.description;
 			["pubsub#type"] = node_obj.config.payload_type;
+			["pubsub#access_model"] = node_obj.config.access_model;
+			["pubsub#publish_model"] = node_obj.config.publish_model;
 		}, "result"));
 	end
 end
@@ -318,14 +332,9 @@ function handlers.get_items(origin, stanza, items, service)
 	for _, id in ipairs(results) do
 		data:add_child(results[id]);
 	end
-	local reply;
-	if data then
-		reply = st.reply(stanza)
-			:tag("pubsub", { xmlns = xmlns_pubsub })
-				:add_child(data);
-	else
-		reply = pubsub_error_reply(stanza, "item-not-found");
-	end
+	local reply = st.reply(stanza)
+		:tag("pubsub", { xmlns = xmlns_pubsub })
+			:add_child(data);
 	origin.send(reply);
 	return true;
 end
@@ -633,14 +642,13 @@ function handlers.set_retract(origin, stanza, retract, service)
 end
 
 function handlers.owner_set_purge(origin, stanza, purge, service)
-	local node, notify = purge.attr.node, purge.attr.notify;
-	notify = (notify == "1") or (notify == "true");
+	local node = purge.attr.node;
 	local reply;
 	if not node then
 		origin.send(pubsub_error_reply(stanza, "nodeid-required"));
 		return true;
 	end
-	local ok, ret = service:purge(node, stanza.attr.from, notify);
+	local ok, ret = service:purge(node, stanza.attr.from, true);
 	if ok then
 		reply = st.reply(stanza);
 	else
@@ -788,7 +796,7 @@ local function create_encapsulating_item(id, payload)
 end
 
 local function archive_itemstore(archive, config, user, node)
-	module:log("debug", "Creation of itemstore for node %s with config %s", node, config);
+	module:log("debug", "Creation of archive itemstore for node %s with config %q", node, config);
 	local get_set = {};
 	local max_items = config["max_items"];
 	function get_set:items() -- luacheck: ignore 212/self
@@ -802,6 +810,7 @@ local function archive_itemstore(archive, config, user, node)
 		end
 		module:log("debug", "Listed items %s", data);
 		return it.reverse(function()
+			-- luacheck: ignore 211/when
 			local id, payload, when, publisher = data();
 			if id == nil then
 				return;
@@ -863,7 +872,7 @@ local function archive_itemstore(archive, config, user, node)
 			return item.attr.id, item;
 		end
 	end
-	return setmetatable(get_set, archive);
+	return get_set;
 end
 _M.archive_itemstore = archive_itemstore;
 
diff --git a/plugins/mod_register.lua b/plugins/mod_register.lua
index 763e2fd9..bec3fb5a 100644
--- a/plugins/mod_register.lua
+++ b/plugins/mod_register.lua
@@ -11,6 +11,7 @@ local allow_registration = module:get_option_boolean("allow_registration", false
 
 if allow_registration then
 	module:depends("register_ibr");
+	module:depends("watchregistrations");
 end
 
 module:depends("user_account_management");
diff --git a/plugins/mod_register_ibr.lua b/plugins/mod_register_ibr.lua
index e04e6ecd..83d284c8 100644
--- a/plugins/mod_register_ibr.lua
+++ b/plugins/mod_register_ibr.lua
@@ -9,10 +9,12 @@
 
 local st = require "util.stanza";
 local dataform_new = require "util.dataforms".new;
-local usermanager_user_exists = require "core.usermanager".user_exists;
-local usermanager_create_user = require "core.usermanager".create_user;
-local usermanager_delete_user = require "core.usermanager".delete_user;
+local usermanager_user_exists  = require "core.usermanager".user_exists;
+local usermanager_create_user  = require "core.usermanager".create_user;
+local usermanager_set_password = require "core.usermanager".create_user;
+local usermanager_delete_user  = require "core.usermanager".delete_user;
 local nodeprep = require "util.encodings".stringprep.nodeprep;
+local util_error = require "util.error";
 
 local additional_fields = module:get_option("additional_registration_fields", {});
 local require_encryption = module:get_option_boolean("c2s_require_encryption",
@@ -155,7 +157,7 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 		return true;
 	end
 
-	local username, password = nodeprep(data.username), data.password;
+	local username, password = nodeprep(data.username, true), data.password;
 	data.username, data.password = nil, nil;
 	local host = module.host;
 	if not username or username == "" then
@@ -167,25 +169,44 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 	local user = { username = username, password = password, host = host, additional = data, ip = session.ip, session = session, allowed = true }
 	module:fire_event("user-registering", user);
 	if not user.allowed then
-		log("debug", "Registration disallowed by module: %s", user.reason or "no reason given");
-		session.send(st.error_reply(stanza, "modify", "not-acceptable", user.reason));
+		local error_type, error_condition, reason;
+		local err = user.error;
+		if err then
+			error_type, error_condition, reason = err.type, err.condition, err.text;
+		else
+			-- COMPAT pre-util.error
+			error_type, error_condition, reason = user.error_type, user.error_condition, user.reason;
+		end
+		log("debug", "Registration disallowed by module: %s", reason or "no reason given");
+		session.send(st.error_reply(stanza, error_type or "modify", error_condition or "not-acceptable", reason));
 		return true;
 	end
 
 	if usermanager_user_exists(username, host) then
-		log("debug", "Attempt to register with existing username");
-		session.send(st.error_reply(stanza, "cancel", "conflict", "The requested username already exists."));
-		return true;
+		if user.allow_reset == username then
+			local ok, err = util_error.coerce(usermanager_set_password(username, password, host));
+			if ok then
+				module:fire_event("user-password-reset", user);
+				session.send(st.reply(stanza)); -- reset ok!
+			else
+				session.log("error", "Unable to reset password for %s@%s: %s", username, host, err);
+				session.send(st.error_reply(stanza, err.type, err.condition, err.text));
+			end
+			return true;
+		else
+			log("debug", "Attempt to register with existing username");
+			session.send(st.error_reply(stanza, "cancel", "conflict", "The requested username already exists."));
+			return true;
+		end
 	end
 
-	-- TODO unable to write file, file may be locked, etc, what's the correct error?
-	local error_reply = st.error_reply(stanza, "wait", "internal-server-error", "Failed to write data to disk.");
-	if usermanager_create_user(username, password, host) then
+	local created, err = usermanager_create_user(username, password, host);
+	if created then
 		data.registered = os.time();
 		if not account_details:set(username, data) then
 			log("debug", "Could not store extra details");
 			usermanager_delete_user(username, host);
-			session.send(error_reply);
+			session.send(st.error_reply(stanza, "wait", "internal-server-error", "Failed to write data to disk."));
 			return true;
 		end
 		session.send(st.reply(stanza)); -- user created!
@@ -194,8 +215,8 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 			username = username, host = host, source = "mod_register",
 			session = session });
 	else
-		log("debug", "Could not create user");
-		session.send(error_reply);
+		log("debug", "Could not create user", err);
+		session.send(st.error_reply(stanza, "cancel", "feature-not-implemented", err));
 	end
 	return true;
 end);
diff --git a/plugins/mod_register_limits.lua b/plugins/mod_register_limits.lua
index 736282a5..7c80f18b 100644
--- a/plugins/mod_register_limits.lua
+++ b/plugins/mod_register_limits.lua
@@ -13,6 +13,7 @@ local ip_util = require "util.ip";
 local new_ip = ip_util.new_ip;
 local match_ip = ip_util.match;
 local parse_cidr = ip_util.parse_cidr;
+local errors = require "util.error";
 
 local min_seconds_between_registrations = module:get_option_number("min_seconds_between_registrations");
 local whitelist_only = module:get_option_boolean("whitelist_registration_only");
@@ -54,6 +55,24 @@ local function ip_in_set(set, ip)
 	return false;
 end
 
+local err_registry = {
+	blacklisted = {
+		text = "Your IP address is blacklisted";
+		type = "auth";
+		condition = "forbidden";
+	};
+	not_whitelisted = {
+		text = "Your IP address is not whitelisted";
+		type = "auth";
+		condition = "forbidden";
+	};
+	throttled = {
+		text = "Too many registrations from this IP address recently";
+		type = "wait";
+		condition = "policy-violation";
+	};
+}
+
 module:hook("user-registering", function (event)
 	local session = event.session;
 	local ip = event.ip or session and session.ip;
@@ -63,16 +82,22 @@ module:hook("user-registering", function (event)
 	elseif ip_in_set(blacklisted_ips, ip) then
 		log("debug", "Registration disallowed by blacklist");
 		event.allowed = false;
-		event.reason = "Your IP address is blacklisted";
+		event.error = errors.new("blacklisted", event, err_registry);
 	elseif (whitelist_only and not ip_in_set(whitelisted_ips, ip)) then
 		log("debug", "Registration disallowed by whitelist");
 		event.allowed = false;
-		event.reason = "Your IP address is not whitelisted";
+		event.error = errors.new("not_whitelisted", event, err_registry);
 	elseif throttle_max and not ip_in_set(whitelisted_ips, ip) then
 		if not check_throttle(ip) then
 			log("debug", "Registrations over limit for ip %s", ip or "?");
 			event.allowed = false;
-			event.reason = "Too many registrations from this IP address recently";
+			event.error = errors.new("throttled", event, err_registry);
 		end
 	end
+	if event.error then
+		-- COMPAT pre-util.error
+		event.reason = event.error.text;
+		event.error_type = event.error.type;
+		event.error_condition = event.error.condition;
+	end
 end);
diff --git a/plugins/mod_s2s/mod_s2s.lua b/plugins/mod_s2s.lua
index 5439aa96..0674f981 100644
--- a/plugins/mod_s2s/mod_s2s.lua
+++ b/plugins/mod_s2s.lua
@@ -27,8 +27,10 @@ local s2s_destroy_session = require "core.s2smanager".destroy_session;
 local uuid_gen = require "util.uuid".generate;
 local fire_global_event = prosody.events.fire_event;
 local runner = require "util.async".runner;
-
-local s2sout = module:require("s2sout");
+local connect = require "net.connect".connect;
+local service = require "net.resolvers.service";
+local errors = require "util.error";
+local set = require "util.set";
 
 local connect_timeout = module:get_option_number("s2s_timeout", 90);
 local stream_close_timeout = module:get_option_number("s2s_close_timeout", 5);
@@ -46,8 +48,16 @@ local sessions = module:shared("sessions");
 
 local runner_callbacks = {};
 
+local listener = {};
+
 local log = module._log;
 
+local s2s_service_options = {
+	default_port = 5269;
+	use_ipv4 = module:get_option_boolean("use_ipv4", true);
+	use_ipv6 = module:get_option_boolean("use_ipv6", true);
+};
+
 module:hook("stats-update", function ()
 	local count = 0;
 	local ipv6 = 0;
@@ -78,15 +88,28 @@ local function bounce_sendq(session, reason)
 			(session.log or log)("error", "Attempting to close the dummy origin of s2s error replies, please report this! Traceback: %s", traceback());
 		end;
 	};
+	-- FIXME Allow for more specific error conditions
+	-- TODO use util.error ?
+	local error_type = "cancel";
+	local condition = "remote-server-not-found";
+	local reason_text;
+	if session.had_stream then -- set when a stream is opened by the remote
+		error_type, condition = "wait", "remote-server-timeout";
+	end
+	if errors.is_err(reason) then
+		error_type, condition, reason_text = reason.type, reason.condition, reason.text;
+	elseif type(reason) == "string" then
+		reason_text = reason;
+	end
 	for i, data in ipairs(sendq) do
 		local reply = data[2];
 		if reply and not(reply.attr.xmlns) and bouncy_stanzas[reply.name] then
 			reply.attr.type = "error";
-			reply:tag("error", {type = "cancel", by = session.from_host})
-				:tag("remote-server-not-found", {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"}):up();
-			if reason then
+			reply:tag("error", {type = error_type, by = session.from_host})
+				:tag(condition, {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"}):up();
+			if reason_text then
 				reply:tag("text", {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"})
-					:text("Server-to-server connection failed: "..reason):up();
+					:text("Server-to-server connection failed: "..reason_text):up();
 			end
 			core_process_stanza(dummy, reply);
 		end
@@ -107,38 +130,33 @@ function route_to_existing_session(event)
 		return false;
 	end
 	local host = hosts[from_host].s2sout[to_host];
-	if host then
-		-- We have a connection to this host already
-		if host.type == "s2sout_unauthed" and (stanza.name ~= "db:verify" or not host.dialback_key) then
-			(host.log or log)("debug", "trying to send over unauthed s2sout to "..to_host);
-
-			-- Queue stanza until we are able to send it
-			local queued_item = {
-				tostring(stanza),
-				stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza);
-			};
-			if host.sendq then
-				t_insert(host.sendq, queued_item);
-			else
-				-- luacheck: ignore 122
-				host.sendq = { queued_item };
-			end
-			host.log("debug", "stanza [%s] queued ", stanza.name);
-			return true;
-		elseif host.type == "local" or host.type == "component" then
-			log("error", "Trying to send a stanza to ourselves??")
-			log("error", "Traceback: %s", traceback());
-			log("error", "Stanza: %s", tostring(stanza));
-			return false;
+	if not host then return end
+
+	-- We have a connection to this host already
+	if host.type == "s2sout_unauthed" and (stanza.name ~= "db:verify" or not host.dialback_key) then
+		(host.log or log)("debug", "trying to send over unauthed s2sout to "..to_host);
+
+		-- Queue stanza until we are able to send it
+		local queued_item = {
+			tostring(stanza),
+			stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza);
+		};
+		if host.sendq then
+			t_insert(host.sendq, queued_item);
 		else
-			-- FIXME
-			if host.from_host ~= from_host then
-				log("error", "WARNING! This might, possibly, be a bug, but it might not...");
-				log("error", "We are going to send from %s instead of %s", host.from_host, from_host);
-			end
-			if host.sends2s(stanza) then
-				return true;
-			end
+			-- luacheck: ignore 122
+			host.sendq = { queued_item };
+		end
+		host.log("debug", "stanza [%s] queued ", stanza.name);
+		return true;
+	elseif host.type == "local" or host.type == "component" then
+		log("error", "Trying to send a stanza to ourselves??")
+		log("error", "Traceback: %s", traceback());
+		log("error", "Stanza: %s", stanza);
+		return false;
+	else
+		if host.sends2s(stanza) then
+			return true;
 		end
 	end
 end
@@ -148,17 +166,13 @@ function route_to_new_session(event)
 	local from_host, to_host, stanza = event.from_host, event.to_host, event.stanza;
 	log("debug", "opening a new outgoing connection for this stanza");
 	local host_session = s2s_new_outgoing(from_host, to_host);
+	host_session.version = 1;
 
 	-- Store in buffer
 	host_session.bounce_sendq = bounce_sendq;
 	host_session.sendq = { {tostring(stanza), stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza)} };
-	log("debug", "stanza [%s] queued until connection complete", tostring(stanza.name));
-	s2sout.initiate_connection(host_session);
-	if (not host_session.connecting) and (not host_session.conn) then
-		log("warn", "Connection to %s failed already, destroying session...", to_host);
-		s2s_destroy_session(host_session, "Connection failed");
-		return false;
-	end
+	log("debug", "stanza [%s] queued until connection complete", stanza.name);
+	connect(service.new(to_host, "xmpp-server", "tcp", s2s_service_options), listener, nil, { session = host_session });
 	return true;
 end
 
@@ -183,10 +197,20 @@ function module.add_host(module)
 			-- so the stream is ready for stanzas.  RFC 6120 Section 4.3
 			mark_connected(session);
 			return true;
+		elseif require_encryption and not session.secure then
+			session.log("warn", "Encrypted server-to-server communication is required but was not offered by %s", session.to_host);
+			session:close({
+					condition = "policy-violation",
+					text = "Encrypted server-to-server communication is required but was not offered",
+				}, nil, "Could not establish encrypted connection to remote server");
+			return true;
 		elseif not session.dialback_verifying then
 			session.log("warn", "No SASL EXTERNAL offer and Dialback doesn't seem to be enabled, giving up");
-			session:close();
-			return false;
+			session:close({
+					condition = "unsupported-feature",
+					text = "No viable authentication method offered",
+				}, nil, "No viable authentication method offered by remote server");
+			return true;
 		end
 	end, -1);
 end
@@ -204,7 +228,18 @@ function mark_connected(session)
 	if session.type == "s2sout" then
 		fire_global_event("s2sout-established", event_data);
 		hosts[from].events.fire_event("s2sout-established", event_data);
+
+		if session.incoming then
+			session.send = function(stanza)
+				return hosts[from].events.fire_event("route/remote", { from_host = from, to_host = to, stanza = stanza });
+			end;
+		end
+
 	else
+		if session.outgoing and not hosts[to].s2sout[from] then
+			session.log("debug", "Setting up to handle route from %s to %s", to, from);
+			hosts[to].s2sout[from] = session; -- luacheck: ignore 122
+		end
 		local host_session = hosts[to];
 		session.send = function(stanza)
 			return host_session.events.fire_event("route/remote", { from_host = to, to_host = from, stanza = stanza });
@@ -224,13 +259,6 @@ function mark_connected(session)
 			end
 			session.sendq = nil;
 		end
-
-		if session.resolver then
-			session.resolver._resolver:closeall()
-		end
-		session.resolver = nil;
-		session.ip_hosts = nil;
-		session.srv_hosts = nil;
 	end
 end
 
@@ -242,7 +270,7 @@ function make_authenticated(event)
 				condition = "policy-violation",
 				text = "Encrypted server-to-server communication is required but was not "
 				       ..((session.direction == "outgoing" and "offered") or "used")
-			});
+			}, nil, "Could not establish encrypted connection to remote server");
 		end
 	end
 	if hosts[host] then
@@ -252,15 +280,13 @@ function make_authenticated(event)
 		session.type = "s2sout";
 	elseif session.type == "s2sin_unauthed" then
 		session.type = "s2sin";
-		if host then
-			if not session.hosts[host] then session.hosts[host] = {}; end
-			session.hosts[host].authed = true;
-		end
-	elseif session.type == "s2sin" and host then
+	elseif session.type ~= "s2sin" and session.type ~= "s2sout" then
+		return false;
+	end
+
+	if session.incoming and host then
 		if not session.hosts[host] then session.hosts[host] = {}; end
 		session.hosts[host].authed = true;
-	else
-		return false;
 	end
 	session.log("debug", "connection %s->%s is now authenticated for %s", session.from_host, session.to_host, host);
 
@@ -297,11 +323,12 @@ local xmlns_xmpp_streams = "urn:ietf:params:xml:ns:xmpp-streams";
 
 function stream_callbacks.streamopened(session, attr)
 	-- run _streamopened in async context
-	session.thread:run({ attr = attr });
+	session.thread:run({ stream = "opened", attr = attr });
 end
 
 function stream_callbacks._streamopened(session, attr)
 	session.version = tonumber(attr.version) or 0;
+	session.had_stream = true; -- Had a stream opened at least once
 
 	-- TODO: Rename session.secure to session.encrypted
 	if session.secure == false then
@@ -315,7 +342,6 @@ function stream_callbacks._streamopened(session, attr)
 			session.compressed = info.compression;
 		else
 			(session.log or log)("info", "Stream encrypted");
-			session.compressed = sock.compression and sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -323,7 +349,9 @@ function stream_callbacks._streamopened(session, attr)
 		-- Send a reply stream header
 
 		-- Validate to/from
-		local to, from = nameprep(attr.to), nameprep(attr.from);
+		local to, from = attr.to, attr.from;
+		if to then to = nameprep(attr.to); end
+		if from then from = nameprep(attr.from); end
 		if not to and attr.to then -- COMPAT: Some servers do not reliably set 'to' (especially on stream restarts)
 			session:close({ condition = "improper-addressing", text = "Invalid 'to' address" });
 			return;
@@ -417,20 +445,6 @@ function stream_callbacks._streamopened(session, attr)
 			end
 		end
 
-		-- Send unauthed buffer
-		-- (stanzas which are fine to send before dialback)
-		-- Note that this is *not* the stanza queue (which
-		-- we can only send if auth succeeds) :)
-		local send_buffer = session.send_buffer;
-		if send_buffer and #send_buffer > 0 then
-			log("debug", "Sending s2s send_buffer now...");
-			for i, data in ipairs(send_buffer) do
-				session.sends2s(tostring(data));
-				send_buffer[i] = nil;
-			end
-		end
-		session.send_buffer = nil;
-
 		-- If server is pre-1.0, don't wait for features, just do dialback
 		if session.version < 1.0 then
 			if not session.dialback_verifying then
@@ -442,11 +456,16 @@ function stream_callbacks._streamopened(session, attr)
 	end
 end
 
-function stream_callbacks.streamclosed(session)
+function stream_callbacks._streamclosed(session)
 	(session.log or log)("debug", "Received </stream:stream>");
 	session:close(false);
 end
 
+function stream_callbacks.streamclosed(session, attr)
+	-- run _streamclosed in async context
+	session.thread:run({ stream = "closed", attr = attr });
+end
+
 function stream_callbacks.error(session, error, data)
 	if error == "no-stream" then
 		session.log("debug", "Invalid opening stream header (%s)", (data:gsub("^([^\1]+)\1", "{%1}")));
@@ -472,11 +491,12 @@ function stream_callbacks.error(session, error, data)
 	end
 end
 
-local listener = {};
-
 --- Session methods
 local stream_xmlns_attr = {xmlns='urn:ietf:params:xml:ns:xmpp-streams'};
-local function session_close(session, reason, remote_reason)
+-- reason: stream error to send to the remote server
+-- remote_reason: stream error received from the remote server
+-- bounce_reason: stanza error to pass to bounce_sendq because stream- and stanza errors are different
+local function session_close(session, reason, remote_reason, bounce_reason)
 	local log = session.log or log;
 	if session.conn then
 		if session.notopen then
@@ -487,27 +507,23 @@ local function session_close(session, reason, remote_reason)
 			end
 		end
 		if reason then -- nil == no err, initiated by us, false == initiated by remote
+			local stream_error;
 			if type(reason) == "string" then -- assume stream error
-				log("debug", "Disconnecting %s[%s], <stream:error> is: %s", session.host or session.ip or "(unknown host)", session.type, reason);
-				session.sends2s(st.stanza("stream:error"):tag(reason, {xmlns = 'urn:ietf:params:xml:ns:xmpp-streams' }));
-			elseif type(reason) == "table" then
-				if reason.condition then
-					local stanza = st.stanza("stream:error"):tag(reason.condition, stream_xmlns_attr):up();
-					if reason.text then
-						stanza:tag("text", stream_xmlns_attr):text(reason.text):up();
-					end
-					if reason.extra then
-						stanza:add_child(reason.extra);
-					end
-					log("debug", "Disconnecting %s[%s], <stream:error> is: %s",
-					session.host or session.ip or "(unknown host)", session.type, stanza);
-					session.sends2s(stanza);
-				elseif reason.name then -- a stanza
-					log("debug", "Disconnecting %s->%s[%s], <stream:error> is: %s",
-						session.from_host or "(unknown host)", session.to_host or "(unknown host)",
-						session.type, reason);
-					session.sends2s(reason);
+				stream_error = st.stanza("stream:error"):tag(reason, {xmlns = 'urn:ietf:params:xml:ns:xmpp-streams' });
+			elseif type(reason) == "table" and not st.is_stanza(reason) then
+				stream_error = st.stanza("stream:error"):tag(reason.condition or "undefined-condition", stream_xmlns_attr):up();
+				if reason.text then
+					stream_error:tag("text", stream_xmlns_attr):text(reason.text):up();
 				end
+				if reason.extra then
+					stream_error:add_child(reason.extra);
+				end
+			end
+			if st.is_stanza(stream_error) then
+				-- to and from are never unknown on outgoing connections
+				log("debug", "Disconnecting %s->%s[%s], <stream:error> is: %s",
+					session.from_host or "(unknown host)" or session.ip, session.to_host or "(unknown host)", session.type, reason);
+				session.sends2s(stream_error);
 			end
 		end
 
@@ -522,16 +538,16 @@ local function session_close(session, reason, remote_reason)
 
 		-- Authenticated incoming stream may still be sending us stanzas, so wait for </stream:stream> from remote
 		local conn = session.conn;
-		if reason == nil and not session.notopen and session.type == "s2sin" then
+		if reason == nil and not session.notopen and session.incoming then
 			add_task(stream_close_timeout, function ()
 				if not session.destroyed then
 					session.log("warn", "Failed to receive a stream close response, closing connection anyway...");
-					s2s_destroy_session(session, reason);
+					s2s_destroy_session(session, reason, bounce_reason);
 					conn:close();
 				end
 			end);
 		else
-			s2s_destroy_session(session, reason);
+			s2s_destroy_session(session, reason, bounce_reason);
 			conn:close(); -- Close immediately, as this is an outgoing connection or is not authed
 		end
 	end
@@ -554,10 +570,12 @@ local function initialize_session(session)
 	local stream = new_xmpp_stream(session, stream_callbacks, stanza_size_limit);
 
 	session.thread = runner(function (stanza)
-		if stanza.name == nil then
-			stream_callbacks._streamopened(session, stanza.attr);
-		else
+		if st.is_stanza(stanza) then
 			core_process_stanza(session, stanza);
+		elseif stanza.stream == "opened" then
+			stream_callbacks._streamopened(session, stanza.attr);
+		elseif stanza.stream == "closed" then
+			stream_callbacks._streamclosed(session, stanza.attr);
 		end
 	end, runner_callbacks, session);
 
@@ -596,8 +614,12 @@ local function initialize_session(session)
 		if data then
 			local ok, err = stream:feed(data);
 			if ok then return; end
-			log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
-			session:close("not-well-formed");
+			log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
+			if err == "stanza-too-large" then
+				session:close({ condition = "policy-violation", text = "XML stanza is too big" }, nil, "Received invalid XML from remote server");
+			else
+				session:close("not-well-formed", nil, "Received invalid XML from remote server");
+			end
 		end
 	end
 
@@ -672,11 +694,20 @@ function listener.ondisconnect(conn, err)
 	local session = sessions[conn];
 	if session then
 		sessions[conn] = nil;
+		(session.log or log)("debug", "s2s disconnected: %s->%s (%s)", session.from_host, session.to_host, err or "connection closed");
+		if session.secure == false and err then
+			-- TODO util.error-ify this
+			err = "Error during negotiation of encrypted connection: "..err;
+		end
+		s2s_destroy_session(session, err);
+	end
+end
+
+function listener.onfail(data, err)
+	local session = data and data.session;
+	if session then
 		if err and session.direction == "outgoing" and session.notopen then
 			(session.log or log)("debug", "s2s connection attempt failed: %s", err);
-			if s2sout.attempt_connection(session, err) then
-				return; -- Session lives for now
-			end
 		end
 		(session.log or log)("debug", "s2s disconnected: %s->%s (%s)", session.from_host, session.to_host, err or "connection closed");
 		s2s_destroy_session(session, err);
@@ -700,6 +731,34 @@ function listener.ondetach(conn)
 	sessions[conn] = nil;
 end
 
+function listener.onattach(conn, data)
+	local session = data and data.session;
+	if session then
+		session.conn = conn;
+		sessions[conn] = session;
+		initialize_session(session);
+	end
+end
+
+-- Complete the sentence "Your certificate " with what's wrong
+local function friendly_cert_error(session) --> string
+	if session.cert_chain_status == "invalid" then
+		if session.cert_chain_errors then
+			local cert_errors = set.new(session.cert_chain_errors[1]);
+			if cert_errors:contains("certificate has expired") then
+				return "has expired";
+			elseif cert_errors:contains("self signed certificate") then
+				return "is self-signed";
+			end
+		end
+		return "is not trusted"; -- for some other reason
+	elseif session.cert_identity_status == "invalid" then
+		return "is not valid for this name";
+	end
+	-- this should normally be unreachable except if no s2s auth module was loaded
+	return "could not be validated";
+end
+
 function check_auth_policy(event)
 	local host, session = event.host, event.session;
 	local must_secure = secure_auth;
@@ -711,20 +770,21 @@ function check_auth_policy(event)
 	end
 
 	if must_secure and (session.cert_chain_status ~= "valid" or session.cert_identity_status ~= "valid") then
-		module:log("warn", "Forbidding insecure connection to/from %s", host or session.ip or "(unknown host)");
-		if session.direction == "incoming" then
-			session:close({ condition = "not-authorized", text = "Your server's certificate is invalid, expired, or not trusted by "..session.to_host });
-		else -- Close outgoing connections without warning
-			session:close(false);
-		end
+		local reason = friendly_cert_error(session);
+		session.log("warn", "Forbidding insecure connection to/from %s because its certificate %s", host or session.ip or "(unknown host)", reason);
+		-- XEP-0178 recommends closing outgoing connections without warning
+		-- but does not give a rationale for this.
+		-- In practice most cases are configuration mistakes or forgotten
+		-- certificate renewals. We think it's better to let the other party
+		-- know about the problem so that they can fix it.
+		session:close({ condition = "not-authorized", text = "Your server's certificate "..reason },
+			nil, "Remote server's certificate "..reason);
 		return false;
 	end
 end
 
 module:hook("s2s-check-certificate", check_auth_policy, -1);
 
-s2sout.set_listener(listener);
-
 module:hook("server-stopping", function(event)
 	local reason = event.reason;
 	for _, session in pairs(sessions) do
@@ -739,7 +799,11 @@ module:provides("net", {
 	listener = listener;
 	default_port = 5269;
 	encryption = "starttls";
+	ssl_config = { -- FIXME This is not used atm, see mod_tls
+		verify = { "peer", "client_once", };
+	};
 	multiplex = {
+		protocol = "xmpp-server";
 		pattern = "^<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:server%1.*>";
 	};
 });
diff --git a/plugins/mod_s2s/s2sout.lib.lua b/plugins/mod_s2s/s2sout.lib.lua
deleted file mode 100644
index 5f765da8..00000000
--- a/plugins/mod_s2s/s2sout.lib.lua
+++ /dev/null
@@ -1,349 +0,0 @@
--- Prosody IM
--- Copyright (C) 2008-2010 Matthew Wild
--- Copyright (C) 2008-2010 Waqas Hussain
---
--- This project is MIT/X11 licensed. Please see the
--- COPYING file in the source package for more information.
---
-
---- Module containing all the logic for connecting to a remote server
-
--- luacheck: ignore 432/err
-
-local portmanager = require "core.portmanager";
-local wrapclient = require "net.server".wrapclient;
-local initialize_filters = require "util.filters".initialize;
-local idna_to_ascii = require "util.encodings".idna.to_ascii;
-local new_ip = require "util.ip".new_ip;
-local rfc6724_dest = require "util.rfc6724".destination;
-local socket = require "socket";
-local adns = require "net.adns";
-local t_insert, t_sort, ipairs = table.insert, table.sort, ipairs;
-local local_addresses = require "util.net".local_addresses;
-
-local s2s_destroy_session = require "core.s2smanager".destroy_session;
-
-local default_mode = module:get_option("network_default_read_size", 4096);
-
-local log = module._log;
-
-local sources = {};
-local has_ipv4, has_ipv6;
-
-local dns_timeout = module:get_option_number("dns_timeout", 15);
-local resolvers = module:get_option_set("s2s_dns_resolvers")
-
-local s2sout = {};
-
-local s2s_listener;
-
-
-function s2sout.set_listener(listener)
-	s2s_listener = listener;
-end
-
-local function compare_srv_priorities(a,b)
-	return a.priority < b.priority or (a.priority == b.priority and a.weight > b.weight);
-end
-
-function s2sout.initiate_connection(host_session)
-	local log = host_session.log or log;
-
-	initialize_filters(host_session);
-	host_session.version = 1;
-
-	host_session.resolver = adns.resolver();
-	host_session.resolver._resolver:settimeout(dns_timeout);
-	if resolvers then
-		for resolver in resolvers do
-			host_session.resolver._resolver:addnameserver(resolver);
-		end
-	end
-
-	-- Kick the connection attempting machine into life
-	if not s2sout.attempt_connection(host_session) then
-		-- Intentionally not returning here, the
-		-- session is needed, connected or not
-		s2s_destroy_session(host_session);
-	end
-
-	if not host_session.sends2s then
-		-- A sends2s which buffers data (until the stream is opened)
-		-- note that data in this buffer will be sent before the stream is authed
-		-- and will not be ack'd in any way, successful or otherwise
-		local buffer;
-		function host_session.sends2s(data)
-			if not buffer then
-				buffer = {};
-				host_session.send_buffer = buffer;
-			end
-			log("debug", "Buffering data on unconnected s2sout to %s", host_session.to_host);
-			buffer[#buffer+1] = data;
-			log("debug", "Buffered item %d: %s", #buffer, data);
-		end
-	end
-end
-
-function s2sout.attempt_connection(host_session, err)
-	local to_host = host_session.to_host;
-	local connect_host, connect_port = to_host and idna_to_ascii(to_host), 5269;
-	local log = host_session.log or log;
-
-	if not connect_host then
-		return false;
-	end
-
-	if not err then -- This is our first attempt
-		log("debug", "First attempt to connect to %s, starting with SRV lookup...", to_host);
-		host_session.connecting = true;
-		host_session.resolver:lookup(function (answer)
-			local srv_hosts = { answer = answer };
-			host_session.srv_hosts = srv_hosts;
-			host_session.srv_choice = 0;
-			host_session.connecting = nil;
-			if answer and #answer > 0 then
-				log("debug", "%s has SRV records, handling...", to_host);
-				for _, record in ipairs(answer) do
-					t_insert(srv_hosts, record.srv);
-				end
-				if #srv_hosts == 1 and srv_hosts[1].target == "." then
-					log("debug", "%s does not provide a XMPP service", to_host);
-					s2s_destroy_session(host_session, err); -- Nothing to see here
-					return;
-				end
-				t_sort(srv_hosts, compare_srv_priorities);
-
-				local srv_choice = srv_hosts[1];
-				host_session.srv_choice = 1;
-				if srv_choice then
-					connect_host, connect_port = srv_choice.target or to_host, srv_choice.port or connect_port;
-					log("debug", "Best record found, will connect to %s:%d", connect_host, connect_port);
-				end
-			else
-				log("debug", "%s has no SRV records, falling back to A/AAAA", to_host);
-			end
-			-- Try with SRV, or just the plain hostname if no SRV
-			local ok, err = s2sout.try_connect(host_session, connect_host, connect_port);
-			if not ok then
-				if not s2sout.attempt_connection(host_session, err) then
-					-- No more attempts will be made
-					s2s_destroy_session(host_session, err);
-				end
-			end
-		end, "_xmpp-server._tcp."..connect_host..".", "SRV");
-
-		return true; -- Attempt in progress
-	elseif host_session.ip_hosts then
-		return s2sout.try_connect(host_session, connect_host, connect_port, err);
-	elseif host_session.srv_hosts and #host_session.srv_hosts > host_session.srv_choice then -- Not our first attempt, and we also have SRV
-		host_session.srv_choice = host_session.srv_choice + 1;
-		local srv_choice = host_session.srv_hosts[host_session.srv_choice];
-		connect_host, connect_port = srv_choice.target or to_host, srv_choice.port or connect_port;
-		host_session.log("info", "Connection failed (%s). Attempt #%d: This time to %s:%d", err, host_session.srv_choice, connect_host, connect_port);
-	else
-		host_session.log("info", "Failed in all attempts to connect to %s", host_session.to_host);
-		-- We're out of options
-		return false;
-	end
-
-	if not (connect_host and connect_port) then
-		-- Likely we couldn't resolve DNS
-		log("warn", "Hmm, we're without a host (%s) and port (%s) to connect to for %s, giving up :(", connect_host, connect_port, to_host);
-		return false;
-	end
-
-	return s2sout.try_connect(host_session, connect_host, connect_port);
-end
-
-function s2sout.try_next_ip(host_session)
-	host_session.connecting = nil;
-	host_session.ip_choice = host_session.ip_choice + 1;
-	local ip = host_session.ip_hosts[host_session.ip_choice];
-	local ok, err= s2sout.make_connect(host_session, ip.ip, ip.port);
-	if not ok then
-		if not s2sout.attempt_connection(host_session, err or "closed") then
-			err = err and (": "..err) or "";
-			s2s_destroy_session(host_session, "Connection failed"..err);
-		end
-	end
-end
-
-function s2sout.try_connect(host_session, connect_host, connect_port, err)
-	host_session.connecting = true;
-	local log = host_session.log or log;
-
-	if not err then
-		local IPs = {};
-		host_session.ip_hosts = IPs;
-		-- luacheck: ignore 231/handle4 231/handle6
-		local handle4, handle6;
-		local have_other_result = not(has_ipv4) or not(has_ipv6) or false;
-
-		if has_ipv4 then
-			handle4 = host_session.resolver:lookup(function (reply, err)
-				handle4 = nil;
-
-				if reply and reply[#reply] and reply[#reply].a then
-					for _, ip in ipairs(reply) do
-						log("debug", "DNS reply for %s gives us %s", connect_host, ip.a);
-						IPs[#IPs+1] = new_ip(ip.a, "IPv4");
-					end
-				elseif err then
-					log("debug", "Error in DNS lookup: %s", err);
-				end
-
-				if have_other_result then
-					if #IPs > 0 then
-						rfc6724_dest(host_session.ip_hosts, sources);
-						for i = 1, #IPs do
-							IPs[i] = {ip = IPs[i], port = connect_port};
-						end
-						host_session.ip_choice = 0;
-						s2sout.try_next_ip(host_session);
-					else
-						log("debug", "DNS lookup failed to get a response for %s", connect_host);
-						host_session.ip_hosts = nil;
-						if not s2sout.attempt_connection(host_session, "name resolution failed") then -- Retry if we can
-							log("debug", "No other records to try for %s - destroying", host_session.to_host);
-							err = err and (": "..err) or "";
-							s2s_destroy_session(host_session, "DNS resolution failed"..err); -- End of the line, we can't
-						end
-					end
-				else
-					have_other_result = true;
-				end
-			end, connect_host, "A", "IN");
-		else
-			have_other_result = true;
-		end
-
-		if has_ipv6 then
-			handle6 = host_session.resolver:lookup(function (reply, err)
-				handle6 = nil;
-
-				if reply and reply[#reply] and reply[#reply].aaaa then
-					for _, ip in ipairs(reply) do
-						log("debug", "DNS reply for %s gives us %s", connect_host, ip.aaaa);
-						IPs[#IPs+1] = new_ip(ip.aaaa, "IPv6");
-					end
-				elseif err then
-					log("debug", "Error in DNS lookup: %s", err);
-				end
-
-				if have_other_result then
-					if #IPs > 0 then
-						rfc6724_dest(host_session.ip_hosts, sources);
-						for i = 1, #IPs do
-							IPs[i] = {ip = IPs[i], port = connect_port};
-						end
-						host_session.ip_choice = 0;
-						s2sout.try_next_ip(host_session);
-					else
-						log("debug", "DNS lookup failed to get a response for %s", connect_host);
-						host_session.ip_hosts = nil;
-						if not s2sout.attempt_connection(host_session, "name resolution failed") then -- Retry if we can
-							log("debug", "No other records to try for %s - destroying", host_session.to_host);
-							err = err and (": "..err) or "";
-							s2s_destroy_session(host_session, "DNS resolution failed"..err); -- End of the line, we can't
-						end
-					end
-				else
-					have_other_result = true;
-				end
-			end, connect_host, "AAAA", "IN");
-		else
-			have_other_result = true;
-		end
-		return true;
-	elseif host_session.ip_hosts and #host_session.ip_hosts > host_session.ip_choice then -- Not our first attempt, and we also have IPs left to try
-		s2sout.try_next_ip(host_session);
-	else
-		log("debug", "Out of IP addresses, trying next SRV record (if any)");
-		host_session.ip_hosts = nil;
-		if not s2sout.attempt_connection(host_session, "out of IP addresses") then -- Retry if we can
-			log("debug", "No other records to try for %s - destroying", host_session.to_host);
-			err = err and (": "..err) or "";
-			s2s_destroy_session(host_session, "Connecting failed"..err); -- End of the line, we can't
-			return false;
-		end
-	end
-
-	return true;
-end
-
-function s2sout.make_connect(host_session, connect_host, connect_port)
-	local log = host_session.log or log;
-	log("debug", "Beginning new connection attempt to %s ([%s]:%d)", host_session.to_host, connect_host.addr, connect_port);
-
-	-- Reset secure flag in case this is another
-	-- connection attempt after a failed STARTTLS
-	host_session.secure = nil;
-	host_session.encrypted = nil;
-
-	local conn, handler;
-	local proto = connect_host.proto;
-	if proto == "IPv4" then
-		conn, handler = socket.tcp();
-	elseif proto == "IPv6" and socket.tcp6 then
-		conn, handler = socket.tcp6();
-	else
-		handler = "Unsupported protocol: "..tostring(proto);
-	end
-
-	if not conn then
-		log("warn", "Failed to create outgoing connection, system error: %s", handler);
-		return false, handler;
-	end
-
-	conn:settimeout(0);
-	local success, err = conn:connect(connect_host.addr, connect_port);
-	if not success and err ~= "timeout" then
-		log("warn", "s2s connect() to %s (%s:%d) failed: %s", host_session.to_host, connect_host.addr, connect_port, err);
-		return false, err;
-	end
-
-	conn = wrapclient(conn, connect_host.addr, connect_port, s2s_listener, default_mode);
-	host_session.conn = conn;
-
-	-- Register this outgoing connection so that xmppserver_listener knows about it
-	-- otherwise it will assume it is a new incoming connection
-	s2s_listener.register_outgoing(conn, host_session);
-
-	log("debug", "Connection attempt in progress...");
-	return true;
-end
-
-module:hook_global("service-added", function (event)
-	if event.name ~= "s2s" then return end
-
-	local s2s_sources = portmanager.get_active_services():get("s2s");
-	if not s2s_sources then
-		module:log("warn", "s2s not listening on any ports, outgoing connections may fail");
-		return;
-	end
-	for source, _ in pairs(s2s_sources) do
-		if source == "*" or source == "0.0.0.0" then
-			for _, addr in ipairs(local_addresses("ipv4", true)) do
-				sources[#sources + 1] = new_ip(addr, "IPv4");
-			end
-		elseif source == "::" then
-			for _, addr in ipairs(local_addresses("ipv6", true)) do
-				sources[#sources + 1] = new_ip(addr, "IPv6");
-			end
-		else
-			sources[#sources + 1] = new_ip(source, (source:find(":") and "IPv6") or "IPv4");
-		end
-	end
-	for i = 1,#sources do
-		if sources[i].proto == "IPv6" then
-			has_ipv6 = true;
-		elseif sources[i].proto == "IPv4" then
-			has_ipv4 = true;
-		end
-	end
-	if not (has_ipv4 or has_ipv6)  then
-		module:log("warn", "No local IPv4 or IPv6 addresses detected, outgoing connections may fail");
-	end
-end);
-
-return s2sout;
diff --git a/plugins/mod_s2s_auth_certs.lua b/plugins/mod_s2s_auth_certs.lua
index dd0eb3cb..37519aa1 100644
--- a/plugins/mod_s2s_auth_certs.lua
+++ b/plugins/mod_s2s_auth_certs.lua
@@ -17,9 +17,6 @@ module:hook("s2s-check-certificate", function(event)
 	local chain_valid, errors;
 	if conn.getpeerverification then
 		chain_valid, errors = conn:getpeerverification();
-	elseif conn.getpeerchainvalid then -- COMPAT mw/luasec-hg
-		chain_valid, errors = conn:getpeerchainvalid();
-		errors = (not chain_valid) and { { errors } } or nil;
 	else
 		chain_valid, errors = false, { { "Chain verification not supported by this version of LuaSec" } };
 	end
@@ -30,6 +27,7 @@ module:hook("s2s-check-certificate", function(event)
 			log("debug", "certificate error(s) at depth %d: %s", depth-1, table.concat(t, ", "))
 		end
 		session.cert_chain_status = "invalid";
+		session.cert_chain_errors = errors;
 	else
 		log("debug", "certificate chain validation result: valid");
 		session.cert_chain_status = "valid";
diff --git a/plugins/mod_s2s_bidi.lua b/plugins/mod_s2s_bidi.lua
new file mode 100644
index 00000000..28e047de
--- /dev/null
+++ b/plugins/mod_s2s_bidi.lua
@@ -0,0 +1,40 @@
+-- Prosody IM
+-- Copyright (C) 2019 Kim Alvefur
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local st = require "util.stanza";
+
+local xmlns_bidi_feature = "urn:xmpp:features:bidi"
+local xmlns_bidi = "urn:xmpp:bidi";
+
+local require_encryption = module:get_option_boolean("s2s_require_encryption", false);
+
+module:hook("s2s-stream-features", function(event)
+	local origin, features = event.origin, event.features;
+	if origin.type == "s2sin_unauthed" and (not require_encryption or origin.secure) then
+		features:tag("bidi", { xmlns = xmlns_bidi_feature }):up();
+	end
+end);
+
+module:hook_tag("http://etherx.jabber.org/streams", "features", function (session, stanza)
+	if session.type == "s2sout_unauthed" and (not require_encryption or session.secure) then
+		local bidi = stanza:get_child("bidi", xmlns_bidi_feature);
+		if bidi then
+			session.incoming = true;
+			session.log("debug", "Requesting bidirectional stream");
+			session.sends2s(st.stanza("bidi", { xmlns = xmlns_bidi }));
+		end
+	end
+end, 200);
+
+module:hook_tag("urn:xmpp:bidi", "bidi", function(session)
+	if session.type == "s2sin_unauthed" and (not require_encryption or session.secure) then
+		session.log("debug", "Requested bidirectional stream");
+		session.outgoing = true;
+		return true;
+	end
+end);
+
diff --git a/plugins/mod_saslauth.lua b/plugins/mod_saslauth.lua
index fba84ef8..2a5edcb2 100644
--- a/plugins/mod_saslauth.lua
+++ b/plugins/mod_saslauth.lua
@@ -12,9 +12,10 @@ local st = require "util.stanza";
 local sm_bind_resource = require "core.sessionmanager".bind_resource;
 local sm_make_authenticated = require "core.sessionmanager".make_authenticated;
 local base64 = require "util.encodings".base64;
+local set = require "util.set";
+local errors = require "util.error";
 
 local usermanager_get_sasl_handler = require "core.usermanager".get_sasl_handler;
-local tostring = tostring;
 
 local secure_auth_only = module:get_option_boolean("c2s_require_encryption", module:get_option_boolean("require_encryption", false));
 local allow_unencrypted_plain_auth = module:get_option_boolean("allow_unencrypted_plain_auth", false)
@@ -48,7 +49,7 @@ local function handle_status(session, status, ret, err_msg)
 		module:fire_event("authentication-failure", { session = session, condition = ret, text = err_msg });
 		session.sasl_handler = session.sasl_handler:clean_clone();
 	elseif status == "success" then
-		local ok, err = sm_make_authenticated(session, session.sasl_handler.username);
+		local ok, err = sm_make_authenticated(session, session.sasl_handler.username, session.sasl_handler.scope);
 		if ok then
 			module:fire_event("authentication-success", { session = session });
 			session.sasl_handler = nil;
@@ -67,7 +68,6 @@ local function sasl_process_cdata(session, stanza)
 	local text = stanza[1];
 	if text then
 		text = base64.decode(text);
-		--log("debug", "AUTH: %s", text:gsub("[%z\001-\008\011\012\014-\031]", " "));
 		if not text then
 			session.sasl_handler = nil;
 			session.send(build_reply("failure", "incorrect-encoding"));
@@ -77,7 +77,6 @@ local function sasl_process_cdata(session, stanza)
 	local status, ret, err_msg = session.sasl_handler:process(text);
 	status, ret, err_msg = handle_status(session, status, ret, err_msg);
 	local s = build_reply(status, ret, err_msg);
-	log("debug", "sasl reply: %s", tostring(s));
 	session.send(s);
 	return true;
 end
@@ -104,18 +103,27 @@ module:hook_tag(xmlns_sasl, "failure", function (session, stanza)
 			break;
 		end
 	end
-	if text and condition then
-		condition = condition .. ": " .. text;
-	end
-	module:log("info", "SASL EXTERNAL with %s failed: %s", session.to_host, condition);
+	local err = errors.new({
+			-- TODO type = what?
+			text = text,
+			condition = condition,
+		}, {
+			session = session,
+			stanza = stanza,
+		});
+
+	module:log("info", "SASL EXTERNAL with %s failed: %s", session.to_host, err);
 
 	session.external_auth = "failed"
-	session.external_auth_failure_reason = condition;
+	session.external_auth_failure_reason = err;
 end, 500)
 
 module:hook_tag(xmlns_sasl, "failure", function (session, stanza) -- luacheck: ignore 212/stanza
 	session.log("debug", "No fallback from SASL EXTERNAL failure, giving up");
-	session:close(nil, session.external_auth_failure_reason);
+	session:close(nil, session.external_auth_failure_reason, errors.new({
+				type = "wait", condition = "remote-server-timeout",
+				text = "Could not authenticate to remote server",
+		}, { session = session, sasl_failure = session.external_auth_failure_reason, }));
 	return true;
 end, 90)
 
@@ -248,37 +256,72 @@ module:hook("stream-features", function(event)
 		local sasl_handler = usermanager_get_sasl_handler(module.host, origin)
 		origin.sasl_handler = sasl_handler;
 		if origin.encrypted then
-			-- check wether LuaSec has the nifty binding to the function needed for tls-unique
+			-- check whether LuaSec has the nifty binding to the function needed for tls-unique
 			-- FIXME: would be nice to have this check only once and not for every socket
 			if sasl_handler.add_cb_handler then
 				local socket = origin.conn:socket();
 				if socket.getpeerfinished then
+					log("debug", "Channel binding 'tls-unique' supported");
 					sasl_handler:add_cb_handler("tls-unique", tls_unique);
+				else
+					log("debug", "Channel binding 'tls-unique' not supported (by LuaSec?)");
 				end
 				sasl_handler["userdata"] = {
 					["tls-unique"] = socket;
 				};
+			else
+				log("debug", "Channel binding not supported by SASL handler");
 			end
 		end
 		local mechanisms = st.stanza("mechanisms", mechanisms_attr);
 		local sasl_mechanisms = sasl_handler:mechanisms()
+		local available_mechanisms = set.new();
 		for mechanism in pairs(sasl_mechanisms) do
-			if disabled_mechanisms:contains(mechanism) then
-				log("debug", "Not offering disabled mechanism %s", mechanism);
-			elseif not origin.secure and insecure_mechanisms:contains(mechanism) then
-				log("debug", "Not offering mechanism %s on insecure connection", mechanism);
-			else
-				log("debug", "Offering mechanism %s", mechanism);
+			available_mechanisms:add(mechanism);
+		end
+		log("debug", "SASL mechanisms supported by handler: %s", available_mechanisms);
+
+		local usable_mechanisms = available_mechanisms - disabled_mechanisms;
+
+		local available_disabled = set.intersection(available_mechanisms, disabled_mechanisms);
+		if not available_disabled:empty() then
+			log("debug", "Not offering disabled mechanisms: %s", available_disabled);
+		end
+
+		local available_insecure = set.intersection(available_mechanisms, insecure_mechanisms);
+		if not origin.secure and not available_insecure:empty() then
+			log("debug", "Session is not secure, not offering insecure mechanisms: %s", available_insecure);
+			usable_mechanisms = usable_mechanisms - insecure_mechanisms;
+		end
+
+		if not usable_mechanisms:empty() then
+			log("debug", "Offering usable mechanisms: %s", usable_mechanisms);
+			for mechanism in usable_mechanisms do
 				mechanisms:tag("mechanism"):text(mechanism):up();
 			end
-		end
-		if mechanisms[1] then
 			features:add_child(mechanisms);
-		elseif not next(sasl_mechanisms) then
-			log("warn", "No available SASL mechanisms, verify that the configured authentication module is working");
-		else
-			log("warn", "All available authentication mechanisms are either disabled or not suitable for an insecure connection");
+			return;
+		end
+
+		local authmod = module:get_option_string("authentication", "internal_plain");
+		if available_mechanisms:empty() then
+			log("warn", "No available SASL mechanisms, verify that the configured authentication module '%s' is loaded and configured correctly", authmod);
+			return;
 		end
+
+		if not origin.secure and not available_insecure:empty() then
+			if not available_disabled:empty() then
+				log("warn", "All SASL mechanisms provided by authentication module '%s' are forbidden on insecure connections (%s) or disabled (%s)",
+					authmod, available_insecure, available_disabled);
+			else
+				log("warn", "All SASL mechanisms provided by authentication module '%s' are forbidden on insecure connections (%s)",
+					authmod, available_insecure);
+			end
+		elseif not available_disabled:empty() then
+			log("warn", "All SASL mechanisms provided by authentication module '%s' are disabled (%s)",
+				authmod, available_disabled);
+		end
+
 	else
 		features:tag("bind", bind_attr):tag("required"):up():up();
 		features:tag("session", xmpp_session_attr):tag("optional"):up():up();
diff --git a/plugins/mod_scansion_record.lua b/plugins/mod_scansion_record.lua
index 8d772b4e..0e109a0d 100644
--- a/plugins/mod_scansion_record.lua
+++ b/plugins/mod_scansion_record.lua
@@ -37,8 +37,7 @@ local function record_event(session, event)
 end
 
 local function record_stanza(stanza, session, verb)
-	local flattened = tostring(stanza):gsub("><", ">\n\t<");
-	-- TODO Proper prettyprinting with indentation
+	local flattened = tostring(stanza:indent(2, "\t"));
 	record(session.scansion_id.." "..verb..":\n\t"..flattened.."\n\n");
 end
 
diff --git a/plugins/mod_server_contact_info.lua b/plugins/mod_server_contact_info.lua
index 0110ceaf..9c916ebc 100644
--- a/plugins/mod_server_contact_info.lua
+++ b/plugins/mod_server_contact_info.lua
@@ -16,6 +16,7 @@ local form_layout = require "util.dataforms".new({
 	{ name = "feedback", var = "feedback-addresses", type = "list-multi" },
 	{ name = "sales", var = "sales-addresses", type = "list-multi" },
 	{ name = "security", var = "security-addresses", type = "list-multi" },
+	{ name = "status", var = "status-addresses", type = "list-multi" },
 	{ name = "support", var = "support-addresses", type = "list-multi" },
 });
 
diff --git a/plugins/mod_stanza_debug.lua b/plugins/mod_stanza_debug.lua
index 6dedb6f7..af98670c 100644
--- a/plugins/mod_stanza_debug.lua
+++ b/plugins/mod_stanza_debug.lua
@@ -1,18 +1,17 @@
 module:set_global();
 
-local tostring = tostring;
 local filters = require "util.filters";
 
 local function log_send(t, session)
 	if t and t ~= "" and t ~= " " then
-		session.log("debug", "SEND: %s", tostring(t));
+		session.log("debug", "SEND: %s", t);
 	end
 	return t;
 end
 
 local function log_recv(t, session)
 	if t and t ~= "" and t ~= " " then
-		session.log("debug", "RECV: %s", tostring(t));
+		session.log("debug", "RECV: %s", t);
 	end
 	return t;
 end
diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua
index 0becfc8f..28dc8921 100644
--- a/plugins/mod_storage_internal.lua
+++ b/plugins/mod_storage_internal.lua
@@ -1,12 +1,17 @@
+local cache = require "util.cache";
 local datamanager = require "core.storagemanager".olddm;
 local array = require "util.array";
 local datetime = require "util.datetime";
 local st = require "util.stanza";
 local now = require "util.time".now;
 local id = require "util.id".medium;
+local jid_join = require "util.jid".join;
 
 local host = module.host;
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000);
+local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
+
 local driver = {};
 
 function driver:open(store, typ)
@@ -43,6 +48,12 @@ end
 local archive = {};
 driver.archive = { __index = archive };
 
+archive.caps = {
+	total = true;
+	quota = archive_item_limit;
+	truncate = true;
+};
+
 function archive:append(username, key, value, when, with)
 	when = when or now();
 	if not st.is_stanza(value) then
@@ -54,28 +65,57 @@ function archive:append(username, key, value, when, with)
 	value.attr.stamp = datetime.datetime(when);
 	value.attr.stamp_legacy = datetime.legacy(when);
 
+	local cache_key = jid_join(username, host, self.store);
+	local item_count = archive_item_count_cache:get(cache_key);
+
 	if key then
 		local items, err = datamanager.list_load(username, host, self.store);
 		if not items and err then return items, err; end
+
+		-- Check the quota
+		item_count = items and #items or 0;
+		archive_item_count_cache:set(cache_key, item_count);
+		if item_count >= archive_item_limit then
+			module:log("debug", "%s reached or over quota, not adding to store", username);
+			return nil, "quota-limit";
+		end
+
 		if items then
+			-- Filter out any item with the same key as the one being added
 			items = array(items);
 			items:filter(function (item)
 				return item.key ~= key;
 			end);
+
 			value.key = key;
 			items:push(value);
 			local ok, err = datamanager.list_store(username, host, self.store, items);
 			if not ok then return ok, err; end
+			archive_item_count_cache:set(cache_key, #items);
 			return key;
 		end
 	else
+		if not item_count then -- Item count not cached?
+			-- We need to load the list to get the number of items currently stored
+			local items, err = datamanager.list_load(username, host, self.store);
+			if not items and err then return items, err; end
+			item_count = items and #items or 0;
+			archive_item_count_cache:set(cache_key, item_count);
+		end
+		if item_count >= archive_item_limit then
+			module:log("debug", "%s reached or over quota, not adding to store", username);
+			return nil, "quota-limit";
+		end
 		key = id();
 	end
 
+	module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store);
+
 	value.key = key;
 
 	local ok, err = datamanager.list_append(username, host, self.store, value);
 	if not ok then return ok, err; end
+	archive_item_count_cache:set(cache_key, item_count+1);
 	return key;
 end
 
@@ -84,12 +124,18 @@ function archive:find(username, query)
 	if not items then
 		if err then
 			return items, err;
-		else
-			return function () end, 0;
+		elseif query then
+			if query.before or query.after then
+				return nil, "item-not-found";
+			end
+			if query.total then
+				return function () end, 0;
+			end
 		end
+		return function () end;
 	end
-	local count = #items;
-	local i = 0;
+	local count = nil;
+	local i, last_key = 0;
 	if query then
 		items = array(items);
 		if query.key then
@@ -114,24 +160,38 @@ function archive:find(username, query)
 				return when <= query["end"];
 			end);
 		end
-		count = #items;
+		if query.total then
+			count = #items;
+		end
 		if query.reverse then
 			items:reverse();
 			if query.before then
-				for j = 1, count do
+				local found = false;
+				for j = 1, #items do
 					if (items[j].key or tostring(j)) == query.before then
+						found = true;
 						i = j;
 						break;
 					end
 				end
+				if not found then
+					return nil, "item-not-found";
+				end
 			end
+		elseif query.before then
+			last_key = query.before;
 		elseif query.after then
-			for j = 1, count do
+			local found = false;
+			for j = 1, #items do
 				if (items[j].key or tostring(j)) == query.after then
+					found = true;
 					i = j;
 					break;
 				end
 			end
+			if not found then
+				return nil, "item-not-found";
+			end
 		end
 		if query.limit and #items - i > query.limit then
 			items[i+query.limit+1] = nil;
@@ -140,7 +200,9 @@ function archive:find(username, query)
 	return function ()
 		i = i + 1;
 		local item = items[i];
-		if not item then return; end
+		if not item or (last_key and item.key == last_key) then
+			return;
+		end
 		local key = item.key or tostring(i);
 		local when = item.when or datetime.parse(item.attr.stamp);
 		local with = item.with;
@@ -152,14 +214,83 @@ function archive:find(username, query)
 	end, count;
 end
 
+function archive:get(username, wanted_key)
+	local iter, err = self:find(username, { key = wanted_key })
+	if not iter then return iter, err; end
+	for key, stanza, when, with in iter do
+		if key == wanted_key then
+			return stanza, when, with;
+		end
+	end
+	return nil, "item-not-found";
+end
+
+function archive:set(username, key, new_value, new_when, new_with)
+	local items, err = datamanager.list_load(username, host, self.store);
+	if not items then
+		if err then
+			return items, err;
+		else
+			return nil, "item-not-found";
+		end
+	end
+
+	for i = 1, #items do
+		local old_item = items[i];
+		if old_item.key == key then
+			local item = st.preserialize(st.clone(new_value));
+
+			local when = new_when or old_item.when or datetime.parse(old_item.attr.stamp);
+			item.key = key;
+			item.when = when;
+			item.with = new_with or old_item.with;
+			item.attr.stamp = datetime.datetime(when);
+			item.attr.stamp_legacy = datetime.legacy(when);
+			items[i] = item;
+			return datamanager.list_store(username, host, self.store, items);
+		end
+	end
+
+	return nil, "item-not-found";
+end
+
 function archive:dates(username)
 	local items, err = datamanager.list_load(username, host, self.store);
 	if not items then return items, err; end
 	return array(items):pluck("when"):map(datetime.date):unique();
 end
 
+function archive:summary(username, query)
+	local iter, err = self:find(username, query)
+	if not iter then return iter, err; end
+	local counts = {};
+	local earliest = {};
+	local latest = {};
+	local body = {};
+	for _, stanza, when, with in iter do
+		counts[with] = (counts[with] or 0) + 1;
+		if earliest[with] == nil then
+			earliest[with] = when;
+		end
+		latest[with] = when;
+		body[with] = stanza:get_child_text("body") or body[with];
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+		body = body;
+	};
+end
+
+function archive:users()
+	return datamanager.users(host, self.store, "list");
+end
+
 function archive:delete(username, query)
+	local cache_key = jid_join(username, host, self.store);
 	if not query or next(query) == nil then
+		archive_item_count_cache:set(cache_key, nil);
 		return datamanager.list_store(username, host, self.store, nil);
 	end
 	local items, err = datamanager.list_load(username, host, self.store);
@@ -167,6 +298,7 @@ function archive:delete(username, query)
 		if err then
 			return items, err;
 		end
+		archive_item_count_cache:set(cache_key, 0);
 		-- Store is empty
 		return 0;
 	end
@@ -216,6 +348,7 @@ function archive:delete(username, query)
 	end
 	local ok, err = datamanager.list_store(username, host, self.store, items);
 	if not ok then return ok, err; end
+	archive_item_count_cache:set(cache_key, #items);
 	return count;
 end
 
diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua
index 745e394b..d71dc0f0 100644
--- a/plugins/mod_storage_memory.lua
+++ b/plugins/mod_storage_memory.lua
@@ -8,6 +8,8 @@ local new_id = require "util.id".medium;
 local auto_purge_enabled = module:get_option_boolean("storage_memory_temporary", false);
 local auto_purge_stores = module:get_option_set("storage_memory_temporary_stores", {});
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000);
+
 local memory = setmetatable({}, {
 	__index = function(t, k)
 		local store = module:shared(k)
@@ -51,6 +53,12 @@ archive_store.__index = archive_store;
 
 archive_store.users = _users;
 
+archive_store.caps = {
+	total = true;
+	quota = archive_item_limit;
+	truncate = true;
+};
+
 function archive_store:append(username, key, value, when, with)
 	if is_stanza(value) then
 		value = st.preserialize(value);
@@ -70,6 +78,8 @@ function archive_store:append(username, key, value, when, with)
 	end
 	if a[key] then
 		table.remove(a, a[key]);
+	elseif #a >= archive_item_limit then
+		return nil, "quota-limit";
 	end
 	local i = #a+1;
 	a[i] = v;
@@ -80,10 +90,18 @@ end
 function archive_store:find(username, query)
 	local items = self.store[username or NULL];
 	if not items then
-		return function () end, 0;
+		if query then
+			if query.before or query.after then
+				return nil, "item-not-found";
+			end
+			if query.total then
+				return function () end, 0;
+			end
+		end
+		return function () end;
 	end
-	local count = #items;
-	local i = 0;
+	local count = nil;
+	local i, last_key = 0;
 	if query then
 		items = array():append(items);
 		if query.key then
@@ -106,24 +124,38 @@ function archive_store:find(username, query)
 				return item.when <= query["end"];
 			end);
 		end
-		count = #items;
+		if query.total then
+			count = #items;
+		end
 		if query.reverse then
 			items:reverse();
 			if query.before then
-				for j = 1, count do
+				local found = false;
+				for j = 1, #items do
 					if (items[j].key or tostring(j)) == query.before then
+						found = true;
 						i = j;
 						break;
 					end
 				end
+				if not found then
+					return nil, "item-not-found";
+				end
 			end
+		elseif query.before then
+			last_key = query.before;
 		elseif query.after then
-			for j = 1, count do
+			local found = false;
+			for j = 1, #items do
 				if (items[j].key or tostring(j)) == query.after then
+					found = true;
 					i = j;
 					break;
 				end
 			end
+			if not found then
+				return nil, "item-not-found";
+			end
 		end
 		if query.limit and #items - i > query.limit then
 			items[i+query.limit+1] = nil;
@@ -132,11 +164,62 @@ function archive_store:find(username, query)
 	return function ()
 		i = i + 1;
 		local item = items[i];
-		if not item then return; end
+		if not item or (last_key and item.key == last_key) then return; end
 		return item.key, item.value(), item.when, item.with;
 	end, count;
 end
 
+function archive_store:get(username, wanted_key)
+	local items = self.store[username or NULL];
+	if not items then return nil, "item-not-found"; end
+	local i = items[wanted_key];
+	if not i then return nil, "item-not-found"; end
+	local item = items[i];
+	return item.value(), item.when, item.with;
+end
+
+function archive_store:set(username, wanted_key, new_value, new_when, new_with)
+	local items = self.store[username or NULL];
+	if not items then return nil, "item-not-found"; end
+	local i = items[wanted_key];
+	if not i then return nil, "item-not-found"; end
+	local item = items[i];
+
+	if is_stanza(new_value) then
+		new_value = st.preserialize(new_value);
+		item.value = envload("return xml"..serialize(new_value), "=(stanza)", { xml = st.deserialize })
+	else
+		item.value = envload("return "..serialize(new_value), "=(data)", {});
+	end
+	if new_when then
+		item.when = new_when;
+	end
+	if new_with then
+		item.with = new_when;
+	end
+	return true;
+end
+
+function archive_store:summary(username, query)
+	local iter, err = self:find(username, query)
+	if not iter then return iter, err; end
+	local counts = {};
+	local earliest = {};
+	local latest = {};
+	for _, _, when, with in iter do
+		counts[with] = (counts[with] or 0) + 1;
+		if earliest[with] == nil then
+			earliest[with] = when;
+		end
+		latest[with] = when;
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+	};
+end
+
 
 function archive_store:delete(username, query)
 	if not query or next(query) == nil then
diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua
index 5b1c3603..8fc92410 100644
--- a/plugins/mod_storage_sql.lua
+++ b/plugins/mod_storage_sql.lua
@@ -1,17 +1,19 @@
 
 -- luacheck: ignore 212/self
 
+local cache = require "util.cache";
 local json = require "util.json";
 local sql = require "util.sql";
 local xml_parse = require "util.xml".parse;
 local uuid = require "util.uuid";
 local resolve_relative_path = require "util.paths".resolve_relative_path;
+local jid_join = require "util.jid".join;
 
 local is_stanza = require"util.stanza".is_stanza;
 local t_concat = table.concat;
 
 local noop = function() end
-local unpack = table.unpack or unpack;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local function iterator(result)
 	return function(result_)
 		local row = result_();
@@ -148,7 +150,13 @@ end
 
 --- Archive store API
 
--- luacheck: ignore 512 431/user 431/store
+local archive_item_limit = module:get_option_number("storage_archive_item_limit");
+local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
+
+local item_count_cache_hit = module:measure("item_count_cache_hit", "rate");
+local item_count_cache_miss = module:measure("item_count_cache_miss", "rate")
+
+-- luacheck: ignore 512 431/user 431/store 431/err
 local map_store = {};
 map_store.__index = map_store;
 map_store.remove = {};
@@ -225,13 +233,91 @@ function map_store:set_keys(username, keydatas)
 	return result;
 end
 
+function map_store:get_all(key)
+	if type(key) ~= "string" or key == "" then
+		return nil, "get_all only supports non-empty string keys";
+	end
+	local ok, result = engine:transaction(function()
+		local query = [[
+		SELECT "user", "type", "value"
+		FROM "prosody"
+		WHERE "host"=? AND "store"=? AND "key"=?
+		]];
+
+		local data;
+		for row in engine:select(query, host, self.store, key) do
+			local key_data, err = deserialize(row[2], row[3]);
+			assert(key_data ~= nil, err);
+			if data == nil then
+				data = {};
+			end
+			data[row[1]] = key_data;
+		end
+
+		return data;
+
+	end);
+	if not ok then return nil, result; end
+	return result;
+end
+
+function map_store:delete_all(key)
+	if type(key) ~= "string" or key == "" then
+		return nil, "delete_all only supports non-empty string keys";
+	end
+	local ok, result = engine:transaction(function()
+		local delete_sql = [[
+		DELETE FROM "prosody"
+		WHERE "host"=? AND "store"=? AND "key"=?;
+		]];
+		engine:delete(delete_sql, host, self.store, key);
+		return true;
+	end);
+	if not ok then return nil, result; end
+	return result;
+end
+
 local archive_store = {}
 archive_store.caps = {
 	total = true;
+	quota = archive_item_limit;
+	truncate = true;
 };
 archive_store.__index = archive_store
 function archive_store:append(username, key, value, when, with)
 	local user,store = username,self.store;
+	local cache_key = jid_join(username, host, store);
+	local item_count = archive_item_count_cache:get(cache_key);
+	if not item_count then
+		item_count_cache_miss();
+		local ok, ret = engine:transaction(function()
+			local count_sql = [[
+			SELECT COUNT(*) FROM "prosodyarchive"
+			WHERE "host"=? AND "user"=? AND "store"=?;
+			]];
+			local result = engine:select(count_sql, host, user, store);
+			if result then
+				for row in result do
+					item_count = row[1];
+				end
+			end
+		end);
+		if not ok or not item_count then
+			module:log("error", "Failed while checking quota for %s: %s", username, ret);
+			return nil, "Failure while checking quota";
+		end
+		archive_item_count_cache:set(cache_key, item_count);
+	else
+		item_count_cache_hit();
+	end
+
+	if archive_item_limit then
+		module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit);
+		if item_count >= archive_item_limit then
+			return nil, "quota-limit";
+		end
+	end
+
 	when = when or os.time();
 	with = with or "";
 	local ok, ret = engine:transaction(function()
@@ -245,12 +331,16 @@ function archive_store:append(username, key, value, when, with)
 		VALUES (?,?,?,?,?,?,?,?);
 		]];
 		if key then
-			engine:delete(delete_sql, host, user or "", store, key);
+			local result = engine:delete(delete_sql, host, user or "", store, key);
+			if result then
+				item_count = item_count - result:affected();
+			end
 		else
 			key = uuid.generate();
 		end
 		local t, encoded_value = assert(serialize(value));
 		engine:insert(insert_sql, host, user or "", store, when, with, key, t, encoded_value);
+		archive_item_count_cache:set(cache_key, item_count+1);
 		return key;
 	end);
 	if not ok then return ok, ret; end
@@ -287,45 +377,48 @@ local function archive_where(query, args, where)
 	end
 end
 local function archive_where_id_range(query, args, where)
-	local args_len = #args
 	-- Before or after specific item, exclusive
+	local id_lookup_sql = [[
+	SELECT "sort_id"
+	FROM "prosodyarchive"
+	WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
+	LIMIT 1;
+	]];
 	if query.after then  -- keys better be unique!
-		where[#where+1] = [[
-		"sort_id" > COALESCE(
-			(
-				SELECT "sort_id"
-				FROM "prosodyarchive"
-				WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
-				LIMIT 1
-			), 0)
-		]];
-		args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.after, args[1], args[2], args[3];
-		args_len = args_len + 4
+		local after_id = nil;
+		for row in engine:select(id_lookup_sql, query.after, args[1], args[2], args[3]) do
+			after_id = row[1];
+		end
+		if not after_id then
+			return nil, "item-not-found";
+		end
+		where[#where+1] = '"sort_id" > ?';
+		args[#args+1] = after_id;
 	end
 	if query.before then
-		where[#where+1] = [[
-		"sort_id" < COALESCE(
-			(
-				SELECT "sort_id"
-				FROM "prosodyarchive"
-				WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
-				LIMIT 1
-			),
-			(
-				SELECT MAX("sort_id")+1
-				FROM "prosodyarchive"
-			)
-		)
-		]]
-		args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.before, args[1], args[2], args[3];
+		local before_id = nil;
+		for row in engine:select(id_lookup_sql, query.before, args[1], args[2], args[3]) do
+			before_id = row[1];
+		end
+		if not before_id then
+			return nil, "item-not-found";
+		end
+		where[#where+1] = '"sort_id" < ?';
+		args[#args+1] = before_id;
 	end
+	return true;
 end
 
 function archive_store:find(username, query)
 	query = query or {};
 	local user,store = username,self.store;
-	local total;
-	local ok, result = engine:transaction(function()
+	local cache_key = jid_join(username, host, self.store);
+	local total = archive_item_count_cache:get(cache_key);
+	(total and item_count_cache_hit or item_count_cache_miss)();
+	if total ~= nil and query.limit == 0 and query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then
+		return noop, total;
+	end
+	local ok, result, err = engine:transaction(function()
 		local sql_query = [[
 		SELECT "key", "type", "value", "when", "with"
 		FROM "prosodyarchive"
@@ -346,12 +439,16 @@ function archive_store:find(username, query)
 					total = row[1];
 				end
 			end
+			if query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then
+				archive_item_count_cache:set(cache_key, total);
+			end
 			if query.limit == 0 then -- Skip the real query
 				return noop, total;
 			end
 		end
 
-		archive_where_id_range(query, args, where);
+		local ok, err = archive_where_id_range(query, args, where);
+		if not ok then return ok, err; end
 
 		if query.limit then
 			args[#args+1] = query.limit;
@@ -361,7 +458,8 @@ function archive_store:find(username, query)
 			and "DESC" or "ASC", query.limit and " LIMIT ?" or "");
 		return engine:select(sql_query, unpack(args));
 	end);
-	if not ok then return ok, result end
+	if not ok then return ok, result; end
+	if not result then return nil, err; end
 	return function()
 		local row = result();
 		if row ~= nil then
@@ -372,6 +470,48 @@ function archive_store:find(username, query)
 	end, total;
 end
 
+function archive_store:summary(username, query)
+	query = query or {};
+	local user,store = username,self.store;
+	local ok, result = engine:transaction(function()
+		local sql_query = [[
+		SELECT DISTINCT "with", COUNT(*), MIN("when"), MAX("when")
+		FROM "prosodyarchive"
+		WHERE %s
+		GROUP BY "with"
+		ORDER BY "sort_id" %s%s;
+		]];
+		local args = { host, user or "", store, };
+		local where = { "\"host\" = ?", "\"user\" = ?", "\"store\" = ?", };
+
+		archive_where(query, args, where);
+
+		archive_where_id_range(query, args, where);
+
+		if query.limit then
+			args[#args+1] = query.limit;
+		end
+
+		sql_query = sql_query:format(t_concat(where, " AND "), query.reverse
+			and "DESC" or "ASC", query.limit and " LIMIT ?" or "");
+		return engine:select(sql_query, unpack(args));
+	end);
+	if not ok then return ok, result end
+	local counts = {};
+	local earliest, latest = {}, {};
+	for row in result do
+		local with, count = row[1], row[2];
+		counts[with] = count;
+		earliest[with] = row[3];
+		latest[with] = row[4];
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+	};
+end
+
 function archive_store:delete(username, query)
 	query = query or {};
 	local user,store = username,self.store;
@@ -384,7 +524,8 @@ function archive_store:delete(username, query)
 			table.remove(where, 2);
 		end
 		archive_where(query, args, where);
-		archive_where_id_range(query, args, where);
+		local ok, err = archive_where_id_range(query, args, where);
+		if not ok then return ok, err; end
 		if query.truncate == nil then
 			sql_query = sql_query:format(t_concat(where, " AND "));
 		else
@@ -423,9 +564,24 @@ function archive_store:delete(username, query)
 		end
 		return engine:delete(sql_query, unpack(args));
 	end);
+	local cache_key = jid_join(username, host, self.store);
+	archive_item_count_cache:set(cache_key, nil);
 	return ok and stmt:affected(), stmt;
 end
 
+function archive_store:users()
+	local ok, result = engine:transaction(function()
+		local select_sql = [[
+		SELECT DISTINCT "user"
+		FROM "prosodyarchive"
+		WHERE "host"=? AND "store"=?;
+		]];
+		return engine:select(select_sql, host, self.store);
+	end);
+	if not ok then error(result); end
+	return iterator(result);
+end
+
 local stores = {
 	keyval = keyval_store;
 	map = map_store;
@@ -610,9 +766,10 @@ function module.load()
 	if prosody.prosodyctl then return; end
 	local engines = module:shared("/*/sql/connections");
 	local params = normalize_params(module:get_option("sql", default_params));
-	engine = engines[sql.db2uri(params)];
+	local db_uri = sql.db2uri(params);
+	engine = engines[db_uri];
 	if not engine then
-		module:log("debug", "Creating new engine");
+		module:log("debug", "Creating new engine %s", db_uri);
 		engine = sql:create_engine(params, function (engine) -- luacheck: ignore 431/engine
 			if module:get_option("sql_manage_tables", true) then
 				-- Automatically create table, ignore failure (table probably already exists)
diff --git a/plugins/mod_tls.lua b/plugins/mod_tls.lua
index 8c0c4e20..151add6f 100644
--- a/plugins/mod_tls.lua
+++ b/plugins/mod_tls.lua
@@ -35,9 +35,10 @@ local host = hosts[module.host];
 
 local ssl_ctx_c2s, ssl_ctx_s2sout, ssl_ctx_s2sin;
 local ssl_cfg_c2s, ssl_cfg_s2sout, ssl_cfg_s2sin;
+local err_c2s, err_s2sin, err_s2sout;
 
 function module.load(reload)
-	local NULL, err = {};
+	local NULL = {};
 	local modhost = module.host;
 	local parent = modhost:match("%.(.*)$");
 
@@ -53,16 +54,20 @@ function module.load(reload)
 	local host_s2s   = rawgetopt(modhost, "s2s_ssl") or parent_s2s;
 
 	module:log("debug", "Creating context for c2s");
-	ssl_ctx_c2s, err, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
-	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err); end
+	local request_client_certs = { verify = { "peer", "client_once", }; };
 
 	module:log("debug", "Creating context for s2sout");
-	ssl_ctx_s2sout, err, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s); -- for outgoing server connections
-	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err); end
+	ssl_ctx_c2s, err_c2s, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
+	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err_c2s); end
 
 	module:log("debug", "Creating context for s2sin");
-	ssl_ctx_s2sin, err, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s); -- for incoming server connections
-	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err); end
+	-- for outgoing server connections
+	ssl_ctx_s2sout, err_s2sout, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s, request_client_certs);
+	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err_s2sout); end
+
+	-- for incoming server connections
+	ssl_ctx_s2sin, err_s2sin, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s, request_client_certs);
+	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err_s2sin); end
 
 	if reload then
 		module:log("info", "Certificates reloaded");
@@ -83,12 +88,21 @@ local function can_do_tls(session)
 		return session.ssl_ctx;
 	end
 	if session.type == "c2s_unauthed" then
+		if not ssl_ctx_c2s and c2s_require_encryption then
+			session.log("error", "No TLS context available for c2s. Earlier error was: %s", err_c2s);
+		end
 		session.ssl_ctx = ssl_ctx_c2s;
 		session.ssl_cfg = ssl_cfg_c2s;
 	elseif session.type == "s2sin_unauthed" and allow_s2s_tls then
+		if not ssl_ctx_s2sin and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sin. Earlier error was: %s", err_s2sin);
+		end
 		session.ssl_ctx = ssl_ctx_s2sin;
 		session.ssl_cfg = ssl_cfg_s2sin;
 	elseif session.direction == "outgoing" and allow_s2s_tls then
+		if not ssl_ctx_s2sout and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sout. Earlier error was: %s", err_s2sout);
+		end
 		session.ssl_ctx = ssl_ctx_s2sout;
 		session.ssl_cfg = ssl_cfg_s2sout;
 	else
diff --git a/plugins/mod_tokenauth.lua b/plugins/mod_tokenauth.lua
new file mode 100644
index 00000000..c04a1aa4
--- /dev/null
+++ b/plugins/mod_tokenauth.lua
@@ -0,0 +1,82 @@
+local id = require "util.id";
+local jid = require "util.jid";
+local base64 = require "util.encodings".base64;
+
+local token_store = module:open_store("auth_tokens", "map");
+
+function create_jid_token(actor_jid, token_jid, token_scope, token_ttl)
+	token_jid = jid.prep(token_jid);
+	if not actor_jid or token_jid ~= actor_jid and not jid.compare(token_jid, actor_jid) then
+		return nil, "not-authorized";
+	end
+
+	local token_username, token_host, token_resource = jid.split(token_jid);
+
+	if token_host ~= module.host then
+		return nil, "invalid-host";
+	end
+
+	local token_info = {
+		owner = actor_jid;
+		created = os.time();
+		expires = token_ttl and (os.time() + token_ttl) or nil;
+		jid = token_jid;
+		session = {
+			username = token_username;
+			host = token_host;
+			resource = token_resource;
+
+			auth_scope = token_scope;
+		};
+	};
+
+	local token_id = id.long();
+	local token = base64.encode("1;"..jid.join(token_username, token_host)..";"..token_id);
+	token_store:set(token_username, token_id, token_info);
+
+	return token, token_info;
+end
+
+local function parse_token(encoded_token)
+	local token = base64.decode(encoded_token);
+	if not token then return nil; end
+	local token_jid, token_id = token:match("^1;([^;]+);(.+)$");
+	if not token_jid then return nil; end
+	local token_user, token_host = jid.split(token_jid);
+	return token_id, token_user, token_host;
+end
+
+function get_token_info(token)
+	local token_id, token_user, token_host = parse_token(token);
+	if not token_id then
+		return nil, "invalid-token-format";
+	end
+	if token_host ~= module.host then
+		return nil, "invalid-host";
+	end
+
+	local token_info, err = token_store:get(token_user, token_id);
+	if not token_info then
+		if err then
+			return nil, "internal-error";
+		end
+		return nil, "not-authorized";
+	end
+
+	if token_info.expires and token_info.expires < os.time() then
+		return nil, "not-authorized";
+	end
+
+	return token_info
+end
+
+function revoke_token(token)
+	local token_id, token_user, token_host = parse_token(token);
+	if not token_id then
+		return nil, "invalid-token-format";
+	end
+	if token_host ~= module.host then
+		return nil, "invalid-host";
+	end
+	return token_store:set(token_user, token_id, nil);
+end
diff --git a/plugins/mod_uptime.lua b/plugins/mod_uptime.lua
index ccd8e511..8a01fb17 100644
--- a/plugins/mod_uptime.lua
+++ b/plugins/mod_uptime.lua
@@ -16,7 +16,7 @@ module:add_feature("jabber:iq:last");
 
 module:hook("iq-get/host/jabber:iq:last:query", function(event)
 	local origin, stanza = event.origin, event.stanza;
-	origin.send(st.reply(stanza):tag("query", {xmlns = "jabber:iq:last", seconds = tostring(os.difftime(os.time(), start_time))}));
+	origin.send(st.reply(stanza):tag("query", {xmlns = "jabber:iq:last", seconds = tostring(("%d"):format(os.difftime(os.time(), start_time)))}));
 	return true;
 end);
 
@@ -42,6 +42,6 @@ function uptime_command_handler ()
 	return { info = uptime_text(), status = "completed" };
 end
 
-local descriptor = adhoc_new("Get uptime", "uptime", uptime_command_handler);
+local descriptor = adhoc_new("Get uptime", "uptime", uptime_command_handler, "any");
 
 module:provides("adhoc", descriptor);
diff --git a/plugins/mod_user_account_management.lua b/plugins/mod_user_account_management.lua
index 615c1ed6..130ed089 100644
--- a/plugins/mod_user_account_management.lua
+++ b/plugins/mod_user_account_management.lua
@@ -53,9 +53,10 @@ local function handle_registration_stanza(event)
 			log("info", "User removed their account: %s@%s", username, host);
 			module:fire_event("user-deregistered", { username = username, host = host, source = "mod_register", session = session });
 		else
-			local username = nodeprep(query:get_child_text("username"));
+			local username = query:get_child_text("username");
 			local password = query:get_child_text("password");
 			if username and password then
+				username = nodeprep(username);
 				if username == session.username then
 					if usermanager_set_password(username, password, session.host, session.resource) then
 						session.send(st.reply(stanza));
diff --git a/plugins/mod_vcard.lua b/plugins/mod_vcard.lua
index b1a4c6e8..c3d6fb8b 100644
--- a/plugins/mod_vcard.lua
+++ b/plugins/mod_vcard.lua
@@ -19,7 +19,7 @@ local function handle_vcard(event)
 	if stanza.attr.type == "get" then
 		local vCard;
 		if to then
-			local node, host = jid_split(to);
+			local node = jid_split(to);
 			vCard = st.deserialize(vcards:get(node)); -- load vCard for user or server
 		else
 			vCard = st.deserialize(vcards:get(session.username));-- load user's own vCard
diff --git a/plugins/mod_vcard_legacy.lua b/plugins/mod_vcard_legacy.lua
index 5e75947a..92b1d5e9 100644
--- a/plugins/mod_vcard_legacy.lua
+++ b/plugins/mod_vcard_legacy.lua
@@ -38,7 +38,7 @@ local simple_map = {
 module:hook("iq-get/bare/vcard-temp:vCard", function (event)
 	local origin, stanza = event.origin, event.stanza;
 	local pep_service = mod_pep.get_pep_service(jid_split(stanza.attr.to) or origin.username);
-	local ok, id, vcard4_item = pep_service:get_last_item("urn:xmpp:vcard4", stanza.attr.from);
+	local ok, _, vcard4_item = pep_service:get_last_item("urn:xmpp:vcard4", stanza.attr.from);
 
 	local vcard_temp = st.stanza("vCard", { xmlns = "vcard-temp" });
 	if ok and vcard4_item then
@@ -105,26 +105,46 @@ module:hook("iq-get/bare/vcard-temp:vCard", function (event)
 					vcard_temp:tag("WORK"):up();
 				end
 				vcard_temp:up();
+			elseif tag.name == "impp" then
+				local uri = tag:get_child_text("uri");
+				if uri and uri:sub(1, 5) == "xmpp:" then
+					vcard_temp:text_tag("JABBERID", uri:sub(6))
+				end
+			elseif tag.name == "org" then
+				vcard_temp:tag("ORG")
+					:text_tag("ORGNAME", tag:get_child_text("text"))
+				:up();
+			end
+		end
+	else
+		local ok, _, nick_item = pep_service:get_last_item("http://jabber.org/protocol/nick", stanza.attr.from);
+		if ok and nick_item then
+			local nickname = nick_item:get_child_text("nick", "http://jabber.org/protocol/nick");
+			if nickname then
+				vcard_temp:text_tag("NICKNAME", nickname);
 			end
 		end
 	end
 
-	local meta_ok, avatar_meta = pep_service:get_items("urn:xmpp:avatar:metadata", stanza.attr.from);
-	local data_ok, avatar_data = pep_service:get_items("urn:xmpp:avatar:data", stanza.attr.from);
+	local ok, avatar_hash, meta = pep_service:get_last_item("urn:xmpp:avatar:metadata", true);
+	if ok and avatar_hash then
 
-	if data_ok then
-		for _, hash in ipairs(avatar_data) do
-			local meta = meta_ok and avatar_meta[hash];
-			local data = avatar_data[hash];
-			local info = meta and meta.tags[1]:get_child("info");
+		local info = meta.tags[1]:get_child("info");
+		if info then
 			vcard_temp:tag("PHOTO");
-			if info and info.attr.type then
+
+			if info.attr.type then
 				vcard_temp:text_tag("TYPE", info.attr.type);
 			end
-			if data then
-				vcard_temp:text_tag("BINVAL", data.tags[1]:get_text());
-			elseif info and info.attr.url then
+
+			if info.attr.url then
 				vcard_temp:text_tag("EXTVAL", info.attr.url);
+			elseif info.attr.id then
+				local data_ok, avatar_data = pep_service:get_items("urn:xmpp:avatar:data", stanza.attr.from, { info.attr.id });
+				if data_ok and avatar_data and avatar_data[info.attr.id]  then
+					local data = avatar_data[info.attr.id];
+					vcard_temp:text_tag("BINVAL", data.tags[1]:get_text());
+				end
 			end
 			vcard_temp:up();
 		end
@@ -140,7 +160,7 @@ local node_defaults = {
 };
 
 function vcard_to_pep(vcard_temp)
-	local avatars = {};
+	local avatar = {};
 
 	local vcard4 = st.stanza("item", { xmlns = "http://jabber.org/protocol/pubsub", id = "current" })
 		:tag("vcard", { xmlns = 'urn:ietf:params:xml:ns:vcard-4.0' });
@@ -216,6 +236,10 @@ function vcard_to_pep(vcard_temp)
 				vcard4:text_tag("text", "work");
 			end
 			vcard4:up():up():up();
+		elseif tag.name == "JABBERID" then
+			vcard4:tag("impp")
+				:text_tag("uri", "xmpp:" .. tag:get_text())
+			:up();
 		elseif tag.name == "PHOTO" then
 			local avatar_type = tag:get_child_text("TYPE");
 			local avatar_payload = tag:get_child_text("BINVAL");
@@ -225,7 +249,9 @@ function vcard_to_pep(vcard_temp)
 				local avatar_raw = base64_decode(avatar_payload);
 				local avatar_hash = sha1(avatar_raw, true);
 
-				local avatar_meta = st.stanza("item", { id = avatar_hash, xmlns = "http://jabber.org/protocol/pubsub" })
+				avatar.hash = avatar_hash;
+
+				avatar.meta = st.stanza("item", { id = avatar_hash, xmlns = "http://jabber.org/protocol/pubsub" })
 					:tag("metadata", { xmlns="urn:xmpp:avatar:metadata" })
 						:tag("info", {
 							bytes = tostring(#avatar_raw),
@@ -233,36 +259,27 @@ function vcard_to_pep(vcard_temp)
 							type = avatar_type,
 						});
 
-				local avatar_data = st.stanza("item", { id = avatar_hash, xmlns = "http://jabber.org/protocol/pubsub" })
+				avatar.data = st.stanza("item", { id = avatar_hash, xmlns = "http://jabber.org/protocol/pubsub" })
 					:tag("data", { xmlns="urn:xmpp:avatar:data" })
 						:text(avatar_payload);
 
-				table.insert(avatars, { hash = avatar_hash, meta = avatar_meta, data = avatar_data });
 			end
 		end
 	end
-	return vcard4, avatars;
+	return vcard4, avatar;
 end
 
-function save_to_pep(pep_service, actor, vcard4, avatars)
-	if avatars then
+function save_to_pep(pep_service, actor, vcard4, avatar)
+	if avatar then
 
 		if pep_service:purge("urn:xmpp:avatar:metadata", actor) then
 			pep_service:purge("urn:xmpp:avatar:data", actor);
 		end
 
-		local avatar_defaults = node_defaults;
-		if #avatars > 1 then
-			avatar_defaults = {};
-			for k,v in pairs(node_defaults) do
-				avatar_defaults[k] = v;
-			end
-			avatar_defaults.max_items = #avatars;
-		end
-		for _, avatar in ipairs(avatars) do
-			local ok, err = pep_service:publish("urn:xmpp:avatar:data", actor, avatar.hash, avatar.data, avatar_defaults);
+		if avatar.data and avatar.meta then
+			local ok, err = assert(pep_service:publish("urn:xmpp:avatar:data", actor, avatar.hash, avatar.data, node_defaults));
 			if ok then
-				ok, err = pep_service:publish("urn:xmpp:avatar:metadata", actor, avatar.hash, avatar.meta, avatar_defaults);
+				ok, err = assert(pep_service:publish("urn:xmpp:avatar:metadata", actor, avatar.hash, avatar.meta, node_defaults));
 			end
 			if not ok then
 				return ok, err;
diff --git a/plugins/mod_websocket.lua b/plugins/mod_websocket.lua
index a613672b..7a5f2d03 100644
--- a/plugins/mod_websocket.lua
+++ b/plugins/mod_websocket.lua
@@ -33,18 +33,10 @@ local frame_buffer_limit = module:get_option_number("websocket_frame_buffer_limi
 local frame_fragment_limit = module:get_option_number("websocket_frame_fragment_limit", 8);
 local stream_close_timeout = module:get_option_number("c2s_close_timeout", 5);
 local consider_websocket_secure = module:get_option_boolean("consider_websocket_secure");
-local cross_domain = module:get_option_set("cross_domain_websocket", {});
-if cross_domain:contains("*") or cross_domain:contains(true) then
-	cross_domain = true;
+local cross_domain = module:get_option("cross_domain_websocket");
+if cross_domain ~= nil then
+	module:log("info", "The 'cross_domain_websocket' option has been deprecated");
 end
-
-local function check_origin(origin)
-	if cross_domain == true then
-		return true;
-	end
-	return cross_domain:contains(origin);
-end
-
 local xmlns_framing = "urn:ietf:params:xml:ns:xmpp-framing";
 local xmlns_streams = "http://etherx.jabber.org/streams";
 local xmlns_client = "jabber:client";
@@ -92,7 +84,7 @@ local function session_close(session, reason)
 					stream_error = reason;
 				end
 			end
-			log("debug", "Disconnecting client, <stream:error> is: %s", tostring(stream_error));
+			log("debug", "Disconnecting client, <stream:error> is: %s", stream_error);
 			session.send(stream_error);
 		end
 
@@ -143,6 +135,11 @@ local function filter_open_close(data)
 	return data;
 end
 
+local default_get_response_body = [[<!DOCTYPE html><html><head><title>Websocket</title></head><body>
+<p>It works! Now point your WebSocket client to this URL to connect to Prosody.</p>
+</body></html>]]
+local websocket_get_response_body = module:get_option_string("websocket_get_response_body", default_get_response_body)
+
 local function validate_frame(frame, max_length)
 	local opcode, length = frame.opcode, frame.length;
 
@@ -207,11 +204,9 @@ function handle_request(event)
 
 	conn.starttls = false; -- Prevent mod_tls from believing starttls can be done
 
-	if not request.headers.sec_websocket_key then
+	if not request.headers.sec_websocket_key or request.method ~= "GET" then
 		response.headers.content_type = "text/html";
-		return [[<!DOCTYPE html><html><head><title>Websocket</title></head><body>
-			<p>It works! Now point your WebSocket client to this URL to connect to Prosody.</p>
-			</body></html>]];
+		return websocket_get_response_body;
 	end
 
 	local wants_xmpp = contains_token(request.headers.sec_websocket_protocol or "", "xmpp");
@@ -221,11 +216,6 @@ function handle_request(event)
 		return 501;
 	end
 
-	if not check_origin(request.headers.origin or "") then
-		module:log("debug", "Origin %s is not allowed by 'cross_domain_websocket' [ %s ]", request.headers.origin or "(missing header)", cross_domain);
-		return 403;
-	end
-
 	local function websocket_close(code, message)
 		conn:write(build_close(code, message));
 		conn:close();
@@ -364,27 +354,4 @@ module:provides("http", {
 
 function module.add_host(module)
 	module:hook("c2s-read-timeout", keepalive, -0.9);
-
-	if cross_domain ~= true then
-		local url = require "socket.url";
-		local ws_url = module:http_url("websocket", "xmpp-websocket");
-		local url_components = url.parse(ws_url);
-		-- The 'Origin' consists of the base URL without path
-		url_components.path = nil;
-		local this_origin = url.build(url_components);
-		local local_cross_domain = module:get_option_set("cross_domain_websocket", { this_origin });
-		if local_cross_domain:contains(true) then
-			module:log("error", "cross_domain_websocket = true only works in the global section");
-			return;
-		end
-
-		-- Don't add / remove something added by another host
-		-- This might be weird with random load order
-		local_cross_domain:exclude(cross_domain);
-		cross_domain:include(local_cross_domain);
-		module:log("debug", "cross_domain = %s", tostring(cross_domain));
-		function module.unload()
-			cross_domain:exclude(local_cross_domain);
-		end
-	end
 end
diff --git a/plugins/muc/hats.lib.lua b/plugins/muc/hats.lib.lua
new file mode 100644
index 00000000..77051af2
--- /dev/null
+++ b/plugins/muc/hats.lib.lua
@@ -0,0 +1,24 @@
+local st = require "util.stanza";
+local muc_util = module:require "muc/util";
+
+local xmlns_hats = "xmpp:prosody.im/protocol/hats:1";
+
+-- Strip any hats claimed by the client (to prevent spoofing)
+muc_util.add_filtered_namespace(xmlns_hats);
+
+module:hook("muc-build-occupant-presence", function (event)
+	local aff_data = event.room:get_affiliation_data(event.occupant.bare_jid);
+	local hats = aff_data and aff_data.hats;
+	if not hats then return; end
+	local hats_el;
+	for hat_id, hat_data in pairs(hats) do
+		if hat_data.active then
+			if not hats_el then
+				hats_el = st.stanza("hats", { xmlns = xmlns_hats });
+			end
+			hats_el:tag("hat", { uri = hat_id, title = hat_data.title }):up();
+		end
+	end
+	if not hats_el then return; end
+	event.stanza:add_direct_child(hats_el);
+end);
diff --git a/plugins/muc/history.lib.lua b/plugins/muc/history.lib.lua
index 0d69c97d..fdf65365 100644
--- a/plugins/muc/history.lib.lua
+++ b/plugins/muc/history.lib.lua
@@ -48,16 +48,18 @@ module:hook("muc-config-form", function(event)
 	table.insert(event.form, {
 		name = "muc#roomconfig_historylength";
 		type = "text-single";
+		datatype = "xs:integer";
 		label = "Maximum number of history messages returned by room";
 		desc = "Specify the maximum number of previous messages that should be sent to users when they join the room";
-		value = tostring(get_historylength(event.room));
+		value = get_historylength(event.room);
 	});
 	table.insert(event.form, {
 		name = 'muc#roomconfig_defaulthistorymessages',
 		type = 'text-single',
+		datatype = "xs:integer";
 		label = 'Default number of history messages returned by room',
 		desc = "Specify the number of previous messages sent to new users when they join the room";
-		value = tostring(get_defaulthistorymessages(event.room))
+		value = get_defaulthistorymessages(event.room);
 	});
 end, 70-5);
 
@@ -180,9 +182,6 @@ module:hook("muc-add-history", function(event)
 	stanza:tag("delay", { -- XEP-0203
 		xmlns = "urn:xmpp:delay", from = room.jid, stamp = stamp
 	}):up();
-	stanza:tag("x", { -- XEP-0091 (deprecated)
-		xmlns = "jabber:x:delay", from = room.jid, stamp = datetime.legacy()
-	}):up();
 	local entry = { stanza = stanza, timestamp = ts };
 	table.insert(history, entry);
 	while #history > get_historylength(room) do table.remove(history, 1) end
@@ -198,7 +197,27 @@ module:hook("muc-broadcast-message", function(event)
 end);
 
 module:hook("muc-message-is-historic", function (event)
-	return event.stanza:get_child("body");
+	local stanza = event.stanza;
+	if stanza:get_child("no-store", "urn:xmpp:hints")
+	or stanza:get_child("no-permanent-store", "urn:xmpp:hints") then
+		-- XXX Experimental XEP
+		return false, "hint";
+	end
+	if stanza:get_child("store", "urn:xmpp:hints") then
+		return true, "hint";
+	end
+	if stanza:get_child("body") then
+		return true;
+	end
+	if stanza:get_child("encryption", "urn:xmpp:eme:0") then
+		-- Since we can't know what an encrypted message contains, we assume it's important
+		-- XXX Experimental XEP
+		return true, "encrypted";
+	end
+	if stanza:get_child(nil, "urn:xmpp:chat-markers:0") then
+		-- XXX Experimental XEP
+		return true, "marker";
+	end
 end, -1);
 
 return {
diff --git a/plugins/muc/language.lib.lua b/plugins/muc/language.lib.lua
index ee80806b..2ee2ba0f 100644
--- a/plugins/muc/language.lib.lua
+++ b/plugins/muc/language.lib.lua
@@ -32,6 +32,7 @@ local function add_form_option(event)
 		label = "Language tag for room (e.g. 'en', 'de', 'fr' etc.)";
 		type = "text-single";
 		desc = "Indicate the primary language spoken in this room";
+		datatype = "xs:language";
 		value = get_language(event.room) or "";
 	});
 end
diff --git a/plugins/muc/lock.lib.lua b/plugins/muc/lock.lib.lua
index 062ab615..32f2647b 100644
--- a/plugins/muc/lock.lib.lua
+++ b/plugins/muc/lock.lib.lua
@@ -43,7 +43,7 @@ end
 module:hook("muc-occupant-pre-join", function(event)
 	if not event.is_new_room and is_locked(event.room) then -- Deny entry
 		module:log("debug", "Room is locked, denying entry");
-		event.origin.send(st.error_reply(event.stanza, "cancel", "item-not-found"));
+		event.origin.send(st.error_reply(event.stanza, "cancel", "item-not-found", nil, module.host));
 		return true;
 	end
 end, -30);
diff --git a/plugins/muc/members_only.lib.lua b/plugins/muc/members_only.lib.lua
index 4194c5c7..79077153 100644
--- a/plugins/muc/members_only.lib.lua
+++ b/plugins/muc/members_only.lib.lua
@@ -113,7 +113,7 @@ module:hook("muc-occupant-pre-join", function(event)
 		local stanza = event.stanza;
 		local affiliation = room:get_affiliation(stanza.attr.from);
 		if valid_affiliations[affiliation or "none"] <= valid_affiliations.none then
-			local reply = st.error_reply(stanza, "auth", "registration-required"):up();
+			local reply = st.error_reply(stanza, "auth", "registration-required", nil, room.jid):up();
 			reply.tags[1].attr.code = "407";
 			event.origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 			return true;
@@ -131,7 +131,7 @@ module:hook("muc-pre-invite", function(event)
 		local inviter_affiliation = room:get_affiliation(stanza.attr.from) or "none";
 		local required_affiliation = room._data.allow_member_invites and "member" or "admin";
 		if valid_affiliations[inviter_affiliation] < valid_affiliations[required_affiliation] then
-			event.origin.send(st.error_reply(stanza, "auth", "forbidden"));
+			event.origin.send(st.error_reply(stanza, "auth", "forbidden", nil, room.jid));
 			return true;
 		end
 	end
diff --git a/plugins/muc/mod_muc.lua b/plugins/muc/mod_muc.lua
index b9c7c859..e9fd1521 100644
--- a/plugins/muc/mod_muc.lua
+++ b/plugins/muc/mod_muc.lua
@@ -86,7 +86,14 @@ room_mt.get_registered_nick = register.get_registered_nick;
 room_mt.get_registered_jid = register.get_registered_jid;
 room_mt.handle_register_iq = register.handle_register_iq;
 
+local presence_broadcast = module:require "muc/presence_broadcast";
+room_mt.get_presence_broadcast = presence_broadcast.get;
+room_mt.set_presence_broadcast = presence_broadcast.set;
+room_mt.get_valid_broadcast_roles = presence_broadcast.get_valid_broadcast_roles;
+
+
 local jid_split = require "util.jid".split;
+local jid_prep = require "util.jid".prep;
 local jid_bare = require "util.jid".bare;
 local st = require "util.stanza";
 local cache = require "util.cache";
@@ -98,6 +105,7 @@ module:depends("disco");
 module:add_identity("conference", "text", module:get_option_string("name", "Prosody Chatrooms"));
 module:add_feature("http://jabber.org/protocol/muc");
 module:depends "muc_unique"
+module:require "muc/hats";
 module:require "muc/lock";
 
 local function is_admin(jid)
@@ -129,7 +137,12 @@ local room_items_cache = {};
 local function room_save(room, forced, savestate)
 	local node = jid_split(room.jid);
 	local is_persistent = persistent.get(room);
-	room_items_cache[room.jid] = room:get_public() and room:get_name() or nil;
+	if room:get_public() then
+		room_items_cache[room.jid] = room:get_name() or "";
+	else
+		room_items_cache[room.jid] = nil;
+	end
+
 	if is_persistent or savestate then
 		persistent_rooms:set(nil, room.jid, true);
 		local data, state = room:freeze(savestate);
@@ -155,7 +168,11 @@ local rooms = cache.new(max_rooms or max_live_rooms, function (jid, room)
 	end
 	module:log("debug", "Evicting room %s", jid);
 	room_eviction();
-	room_items_cache[room.jid] = room:get_public() and room:get_name() or nil;
+	if room:get_public() then
+		room_items_cache[room.jid] = room:get_name() or "";
+	else
+		room_items_cache[room.jid] = nil;
+	end
 	local ok, err = room_save(room, nil, true); -- Force to disk
 	if not ok then
 		module:log("error", "Failed to swap inactive room %s to disk: %s", jid, err);
@@ -185,7 +202,7 @@ end
 
 local function handle_broken_room(room, origin, stanza)
 	module:log("debug", "Returning error from broken room %s", room.jid);
-	origin.send(st.error_reply(stanza, "wait", "internal-server-error"));
+	origin.send(st.error_reply(stanza, "wait", "internal-server-error", nil, room.jid));
 	return true;
 end
 
@@ -264,9 +281,13 @@ local function set_room_defaults(room, lang)
 	room:set_changesubject(module:get_option_boolean("muc_room_default_change_subject", room:get_changesubject()));
 	room:set_historylength(module:get_option_number("muc_room_default_history_length", room:get_historylength()));
 	room:set_language(lang or module:get_option_string("muc_room_default_language"));
+	room:set_presence_broadcast(module:get_option("muc_room_default_presence_broadcast", room:get_presence_broadcast()));
 end
 
 function create_room(room_jid, config)
+	if jid_bare(room_jid) ~= room_jid or not jid_prep(room_jid, true) then
+		return nil, "invalid-jid";
+	end
 	local exists = get_room_from_jid(room_jid);
 	if exists then
 		return nil, "room-exists";
@@ -325,13 +346,14 @@ module:hook("host-disco-items", function(event)
 	module:log("debug", "host-disco-items called");
 	if next(room_items_cache) ~= nil then
 		for jid, room_name in pairs(room_items_cache) do
+			if room_name == "" then room_name = nil; end
 			reply:tag("item", { jid = jid, name = room_name }):up();
 		end
 	else
 		for room in all_rooms() do
 			if not room:get_hidden() then
 				local jid, room_name = room.jid, room:get_name();
-				room_items_cache[jid] = room_name;
+				room_items_cache[jid] = room_name or "";
 				reply:tag("item", { jid = jid, name = room_name }):up();
 			end
 		end
@@ -345,7 +367,7 @@ end, 1);
 module:hook("muc-room-pre-create", function(event)
 	local origin, stanza = event.origin, event.stanza;
 	if not track_room(event.room) then
-		origin.send(st.error_reply(stanza, "wait", "resource-constraint"));
+		origin.send(st.error_reply(stanza, "wait", "resource-constraint", nil, module.host));
 		return true;
 	end
 end, -1000);
@@ -396,7 +418,7 @@ do
 				restrict_room_creation == "local" and
 				select(2, jid_split(user_jid)) == host_suffix
 			) then
-				origin.send(st.error_reply(stanza, "cancel", "not-allowed", "Room creation is restricted"));
+				origin.send(st.error_reply(stanza, "cancel", "not-allowed", "Room creation is restricted", module.host));
 				return true;
 			end
 		end);
@@ -441,7 +463,7 @@ for event_name, method in pairs {
 				room = nil;
 			else
 				if stanza.attr.type ~= "error" then
-					local reply = st.error_reply(stanza, "cancel", "gone", room._data.reason)
+					local reply = st.error_reply(stanza, "cancel", "gone", room._data.reason, module.host)
 					if room._data.newjid then
 						local uri = "xmpp:"..room._data.newjid.."?join";
 						reply:get_child("error"):child_with_name("gone"):text(uri);
@@ -454,17 +476,21 @@ for event_name, method in pairs {
 
 		if room == nil then
 			-- Watch presence to create rooms
-			if stanza.attr.type == nil and stanza.name == "presence" then
+			if not jid_prep(room_jid, true) then
+				origin.send(st.error_reply(stanza, "modify", "jid-malformed", nil, module.host));
+				return true;
+			end
+			if stanza.attr.type == nil and stanza.name == "presence" and stanza:get_child("x", "http://jabber.org/protocol/muc") then
 				room = muclib.new_room(room_jid);
 				return room:handle_first_presence(origin, stanza);
 			elseif stanza.attr.type ~= "error" then
-				origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+				origin.send(st.error_reply(stanza, "cancel", "item-not-found", nil, module.host));
 				return true;
 			else
 				return;
 			end
 		elseif room == false then -- Error loading room
-			origin.send(st.error_reply(stanza, "wait", "resource-constraint"));
+			origin.send(st.error_reply(stanza, "wait", "resource-constraint", nil, module.host));
 			return true;
 		end
 		return room[method](room, origin, stanza);
@@ -483,6 +509,7 @@ do -- Ad-hoc commands
 	local t_concat = table.concat;
 	local adhoc_new = module:require "adhoc".new;
 	local adhoc_initial = require "util.adhoc".new_initial_data_form;
+	local adhoc_simple = require "util.adhoc".new_simple_form;
 	local array = require "util.array";
 	local dataforms_new = require "util.dataforms".new;
 
@@ -513,4 +540,46 @@ do -- Ad-hoc commands
 		"http://prosody.im/protocol/muc#destroy", destroy_rooms_handler, "admin");
 
 	module:provides("adhoc", destroy_rooms_desc);
+
+
+	local set_affiliation_layout = dataforms_new {
+		-- FIXME wordsmith title, instructions, labels etc
+		title = "Set affiliation";
+
+		{ name = "FORM_TYPE", type = "hidden", value = "http://prosody.im/protocol/muc#set-affiliation" };
+		{ name = "room", type = "jid-single", required = true, label = "Room"};
+		{ name = "jid", type = "jid-single", required = true, label = "JID"};
+		{ name = "affiliation", type = "list-single", required = true, label = "Affiliation",
+			options = { "owner"; "admin"; "member"; "none"; "outcast"; },
+		};
+		{ name = "reason", type = "text-single", "Reason", }
+	};
+
+	local set_affiliation_handler = adhoc_simple(set_affiliation_layout, function (fields, errors)
+		if errors then
+			local errmsg = {};
+			for field, err in pairs(errors) do
+				errmsg[#errmsg + 1] = field .. ": " .. err;
+			end
+			return { status = "completed", error = { message = t_concat(errmsg, "\n") } };
+		end
+
+		local room = get_room_from_jid(fields.room);
+		if not room then
+			return { status = "canceled", error = { message =  "No such room"; }; };
+		end
+		local ok, err, condition = room:set_affiliation(true, fields.jid, fields.affiliation, fields.reason);
+
+		if not ok then
+			return { status = "canceled", error = { message =  "Affiliation change failed: "..err..":"..condition; }; };
+		end
+
+		return { status = "completed", info = "Affiliation updated",
+		};
+	end);
+
+	local set_affiliation_desc = adhoc_new("Set affiliation in room",
+		"http://prosody.im/protocol/muc#set-affiliation", set_affiliation_handler, "admin");
+
+	module:provides("adhoc", set_affiliation_desc);
 end
diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua
index 3139c131..6b330131 100644
--- a/plugins/muc/muc.lib.lua
+++ b/plugins/muc/muc.lib.lua
@@ -23,6 +23,7 @@ local resourceprep = require "util.encodings".stringprep.resourceprep;
 local st = require "util.stanza";
 local base64 = require "util.encodings".base64;
 local md5 = require "util.hashes".md5;
+local new_id = require "util.id".medium;
 
 local log = module._log;
 
@@ -39,7 +40,7 @@ function room_mt:__tostring()
 end
 
 function room_mt.save()
-	-- overriden by mod_muc.lua
+	-- overridden by mod_muc.lua
 end
 
 function room_mt:get_occupant_jid(real_jid)
@@ -217,13 +218,13 @@ end
 
 -- Broadcasts an occupant's presence to the whole room
 -- Takes the x element that goes into the stanzas
-function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
+function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason, prev_role, force_unavailable)
 	local base_x = x.base or x;
 	-- Build real jid and (optionally) occupant jid template presences
 	local base_presence do
 		-- Try to use main jid's presence
 		local pr = occupant:get_presence();
-		if pr and (occupant.role ~= nil or pr.attr.type == "unavailable") then
+		if pr and (occupant.role ~= nil or pr.attr.type == "unavailable") and not force_unavailable then
 			base_presence = st.clone(pr);
 		else -- user is leaving but didn't send a leave presence. make one for them
 			base_presence = st.presence {from = occupant.nick; type = "unavailable";};
@@ -236,6 +237,7 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 		occupant = occupant; nick = nick; actor = actor;
 		reason = reason;
 	}
+	module:fire_event("muc-build-occupant-presence", event);
 	module:fire_event("muc-broadcast-presence", event);
 
 	-- Allow muc-broadcast-presence listeners to change things
@@ -279,7 +281,9 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 		self_p = st.clone(base_presence):add_child(self_x);
 	end
 
-	-- General populance
+	local broadcast_roles = self:get_presence_broadcast();
+
+	-- General populace
 	for occupant_nick, n_occupant in self:each_occupant() do
 		if occupant_nick ~= occupant.nick then
 			local pr;
@@ -290,7 +294,13 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 			else
 				pr = get_anon_p();
 			end
-			self:route_to_occupant(n_occupant, pr);
+			if broadcast_roles[occupant.role or "none"] or force_unavailable then
+				self:route_to_occupant(n_occupant, pr);
+			elseif prev_role and broadcast_roles[prev_role] then
+				pr.attr.type = 'unavailable';
+				self:route_to_occupant(n_occupant, pr);
+			end
+
 		end
 	end
 
@@ -303,6 +313,7 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 		-- use their own presences as templates
 		for full_jid, pr in occupant:each_session() do
 			pr = st.clone(pr);
+			module:fire_event("muc-build-occupant-presence", { room = self, occupant = occupant, stanza = pr });
 			pr.attr.to = full_jid;
 			pr:add_child(self_x);
 			self:route_stanza(pr);
@@ -314,6 +325,7 @@ function room_mt:send_occupant_list(to, filter)
 	local to_bare = jid_bare(to);
 	local is_anonymous = false;
 	local whois = self:get_whois();
+	local broadcast_roles = self:get_presence_broadcast();
 	if whois ~= "anyone" then
 		local affiliation = self:get_affiliation(to);
 		if affiliation ~= "admin" and affiliation ~= "owner" then
@@ -323,14 +335,38 @@ function room_mt:send_occupant_list(to, filter)
 			end
 		end
 	end
+	local broadcast_bare_jids = {}; -- Track which bare JIDs we have sent presence for
 	for occupant_jid, occupant in self:each_occupant() do
+		broadcast_bare_jids[occupant.bare_jid] = true;
 		if filter == nil or filter(occupant_jid, occupant) then
 			local x = st.stanza("x", {xmlns='http://jabber.org/protocol/muc#user'});
 			self:build_item_list(occupant, x, is_anonymous and to_bare ~= occupant.bare_jid); -- can always see your own jids
 			local pres = st.clone(occupant:get_presence());
 			pres.attr.to = to;
 			pres:add_child(x);
-			self:route_stanza(pres);
+			module:fire_event("muc-build-occupant-presence", { room = self, occupant = occupant, stanza = pres });
+			if to_bare == occupant.bare_jid or broadcast_roles[occupant.role or "none"] then
+				self:route_stanza(pres);
+			end
+		end
+	end
+	if broadcast_roles.none then
+		-- Broadcast stanzas for affiliated users not currently in the MUC
+		for affiliated_jid, affiliation, affiliation_data in self:each_affiliation() do
+			local nick = affiliation_data and affiliation_data.reserved_nickname;
+			if (nick or not is_anonymous) and not broadcast_bare_jids[affiliated_jid]
+			and (filter == nil or filter(affiliated_jid, nil)) then
+				local from = nick and (self.jid.."/"..nick) or self.jid;
+				local pres = st.presence({ to = to, from = from, type = "unavailable" })
+					:tag("x", { xmlns = 'http://jabber.org/protocol/muc#user' })
+						:tag("item", {
+							affiliation = affiliation;
+							role = "none";
+							nick = nick;
+							jid = not is_anonymous and affiliated_jid or nil }):up()
+						:up();
+				self:route_stanza(pres);
+			end
 		end
 	end
 end
@@ -373,13 +409,14 @@ function room_mt:handle_kickable(origin, stanza) -- luacheck: ignore 212
 	local real_jid = stanza.attr.from;
 	local occupant = self:get_occupant_by_real_jid(real_jid);
 	if occupant == nil then return nil; end
-	local type, condition, text = stanza:get_error();
+	local _, condition, text = stanza:get_error();
 	local error_message = "Kicked: "..(condition and condition:gsub("%-", " ") or "presence error");
 	if text and self:get_whois() == "anyone" then
 		error_message = error_message..": "..text;
 	end
 	occupant:set_session(real_jid, st.presence({type="unavailable"})
 		:tag('status'):text(error_message));
+	local orig_role = occupant.role;
 	local is_last_session = occupant.jid == real_jid;
 	if is_last_session then
 		occupant.role = nil;
@@ -389,9 +426,13 @@ function room_mt:handle_kickable(origin, stanza) -- luacheck: ignore 212
 	if is_last_session then
 		x:tag("status", {code = "333"});
 	end
-	self:publicise_occupant_status(new_occupant or occupant, x);
+	self:publicise_occupant_status(new_occupant or occupant, x, nil, nil, nil, orig_role);
 	if is_last_session then
-		module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 	return true;
 end
@@ -406,7 +447,7 @@ module:hook("muc-occupant-pre-join", function(event)
 	local room, stanza = event.room, event.stanza;
 	local affiliation = room:get_affiliation(stanza.attr.from);
 	if affiliation == "outcast" then
-		local reply = st.error_reply(stanza, "auth", "forbidden"):up();
+		local reply = st.error_reply(stanza, "auth", "forbidden", nil, room.jid):up();
 		reply.tags[1].attr.code = "403";
 		event.origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
@@ -414,28 +455,41 @@ module:hook("muc-occupant-pre-join", function(event)
 end, -10);
 
 module:hook("muc-occupant-pre-join", function(event)
+	local room = event.room;
 	local nick = jid_resource(event.occupant.nick);
 	if not nick:find("%S") then
-		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden"));
+		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden", room.jid));
 		return true;
 	end
 end, 1);
 
 module:hook("muc-occupant-pre-change", function(event)
+	local room = event.room;
 	if not jid_resource(event.dest_occupant.nick):find("%S") then
-		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden"));
+		event.origin.send(st.error_reply(event.stanza, "modify", "not-allowed", "Invisible Nicknames are forbidden", room.jid));
 		return true;
 	end
 end, 1);
 
-function room_mt:handle_first_presence(origin, stanza)
-	if not stanza:get_child("x", "http://jabber.org/protocol/muc") then
-		module:log("debug", "Room creation without <x>, possibly desynced");
+module:hook("muc-occupant-pre-join", function(event)
+	local room = event.room;
+	local nick = jid_resource(event.occupant.nick);
+	if not resourceprep(nick, true) then -- strict
+		event.origin.send(st.error_reply(event.stanza, "modify", "jid-malformed", "Nickname must pass strict validation", room.jid));
+		return true;
+	end
+end, 2);
 
-		origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+module:hook("muc-occupant-pre-change", function(event)
+	local room = event.room;
+	local nick = jid_resource(event.dest_occupant.nick);
+	if not resourceprep(nick, true) then -- strict
+		event.origin.send(st.error_reply(event.stanza, "modify", "jid-malformed", "Nickname must pass strict validation", room.jid));
 		return true;
 	end
+end, 2);
 
+function room_mt:handle_first_presence(origin, stanza)
 	local real_jid = stanza.attr.from;
 	local dest_jid = stanza.attr.to;
 	local bare_jid = jid_bare(real_jid);
@@ -505,7 +559,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 	if orig_occupant == nil and not muc_x and stanza.attr.type == nil then
 		module:log("debug", "Attempted join without <x>, possibly desynced");
 		origin.send(st.error_reply(stanza, "cancel", "item-not-found",
-			"You must join the room before sending presence updates"));
+			"You are not currently connected to this chat", self.jid));
 		return true;
 	end
 
@@ -567,7 +621,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 		and bare_jid ~= jid_bare(dest_occupant.bare_jid) then
 		-- new nick or has different bare real jid
 		log("debug", "%s couldn't join due to nick conflict: %s", real_jid, dest_occupant.nick);
-		local reply = st.error_reply(stanza, "cancel", "conflict"):up();
+		local reply = st.error_reply(stanza, "cancel", "conflict", nil, self.jid):up();
 		reply.tags[1].attr.code = "409";
 		origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
@@ -576,6 +630,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 	-- Send presence stanza about original occupant
 	if orig_occupant ~= nil and orig_occupant ~= dest_occupant then
 		local orig_x = st.stanza("x", {xmlns = "http://jabber.org/protocol/muc#user";});
+		local orig_role = orig_occupant.role;
 		local dest_nick;
 		if dest_occupant == nil then -- Session is leaving
 			log("debug", "session %s is leaving occupant %s", real_jid, orig_occupant.nick);
@@ -613,12 +668,12 @@ function room_mt:handle_normal_presence(origin, stanza)
 				x:tag("status", {code = "303";}):up();
 				x:tag("status", {code = "110";}):up();
 				self:route_stanza(generated_unavail:add_child(x));
-				dest_nick = nil; -- set dest_nick to nil; so general populance doesn't see it for whole orig_occupant
+				dest_nick = nil; -- set dest_nick to nil; so general populace doesn't see it for whole orig_occupant
 			end
 		end
 
 		self:save_occupant(orig_occupant);
-		self:publicise_occupant_status(orig_occupant, orig_x, dest_nick);
+		self:publicise_occupant_status(orig_occupant, orig_x, dest_nick, nil, nil, orig_role);
 
 		if is_last_orig_session then
 			module:fire_event("muc-occupant-left", {
@@ -639,7 +694,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 			-- Send occupant list to newly joined or desynced user
 			self:send_occupant_list(real_jid, function(nick, occupant) -- luacheck: ignore 212
 				-- Don't include self
-				return occupant:get_presence(real_jid) == nil;
+				return (not occupant) or occupant:get_presence(real_jid) == nil;
 			end)
 		end
 		local dest_x = st.stanza("x", {xmlns = "http://jabber.org/protocol/muc#user";});
@@ -650,7 +705,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 		if nick_changed then
 			self_x:tag("status", {code="210"}):up();
 		end
-		self:publicise_occupant_status(dest_occupant, {base=dest_x,self=self_x});
+		self:publicise_occupant_status(dest_occupant, {base=dest_x,self=self_x}, nil, nil, nil, orig_occupant and orig_occupant.role or nil);
 
 		if orig_occupant ~= nil and orig_occupant ~= dest_occupant and not is_last_orig_session then
 			-- If user is swapping and wasn't last original session
@@ -696,7 +751,7 @@ function room_mt:handle_presence_to_occupant(origin, stanza)
 		return self:handle_normal_presence(origin, stanza);
 	elseif type ~= 'result' then -- bad type
 		if type ~= 'visible' and type ~= 'invisible' then -- COMPAT ejabberd can broadcast or forward XEP-0018 presences
-			origin.send(st.error_reply(stanza, "modify", "bad-request")); -- FIXME correct error?
+			origin.send(st.error_reply(stanza, "modify", "bad-request", nil, self.jid)); -- FIXME correct error?
 		end
 	end
 	return true;
@@ -731,11 +786,11 @@ function room_mt:handle_iq_to_occupant(origin, stanza)
 	else -- Type is "get" or "set"
 		local current_nick = self:get_occupant_jid(from);
 		if not current_nick then
-			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat"));
+			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat", self.jid));
 			return true;
 		end
 		if not occupant then -- recipient not in room
-			origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room"));
+			origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room", self.jid));
 			return true;
 		end
 		-- XEP-0410 MUC Self-Ping #1220
@@ -764,12 +819,12 @@ function room_mt:handle_message_to_occupant(origin, stanza)
 	local type = stanza.attr.type;
 	if not current_nick then -- not in room
 		if type ~= "error" then
-			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat"));
+			origin.send(st.error_reply(stanza, "cancel", "not-acceptable", "You are not currently connected to this chat", self.jid));
 		end
 		return true;
 	end
 	if type == "groupchat" then -- groupchat messages not allowed in PM
-		origin.send(st.error_reply(stanza, "modify", "bad-request"));
+		origin.send(st.error_reply(stanza, "modify", "bad-request", nil, self.jid));
 		return true;
 	elseif type == "error" and is_kickable_error(stanza) then
 		log("debug", "%s kicked from %s for sending an error message", current_nick, self.jid);
@@ -778,14 +833,16 @@ function room_mt:handle_message_to_occupant(origin, stanza)
 
 	local o_data = self:get_occupant_by_nick(to);
 	if not o_data then
-		origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room"));
+		origin.send(st.error_reply(stanza, "cancel", "item-not-found", "Recipient not in room", self.jid));
 		return true;
 	end
 	log("debug", "%s sent private message stanza to %s (%s)", from, to, o_data.jid);
 	stanza = muc_util.filter_muc_x(st.clone(stanza));
 	stanza:tag("x", { xmlns = "http://jabber.org/protocol/muc#user" }):up();
 	stanza.attr.from = current_nick;
-	self:route_to_occupant(o_data, stanza)
+	if module:fire_event("muc-private-message", { room = self, origin = origin, stanza = stanza }) ~= false then
+		self:route_to_occupant(o_data, stanza)
+	end
 	-- TODO: Remove x tag?
 	stanza.attr.from = from;
 	return true;
@@ -815,10 +872,12 @@ function room_mt:process_form(origin, stanza)
 	if form.attr.type == "cancel" then
 		origin.send(st.reply(stanza));
 	elseif form.attr.type == "submit" then
+		-- luacheck: ignore 231/errors
 		local fields, errors, present;
 		if form.tags[1] == nil then -- Instant room
 			fields, present = {}, {};
 		else
+			-- FIXME handle form errors
 			fields, errors, present = self:get_form_layout(stanza.attr.from):data(form);
 			if fields.FORM_TYPE ~= "http://jabber.org/protocol/muc#roomconfig" then
 				origin.send(st.error_reply(stanza, "cancel", "bad-request", "Form is not of type room configuration"));
@@ -873,13 +932,18 @@ function room_mt:clear(x)
 	x = x or st.stanza("x", {xmlns='http://jabber.org/protocol/muc#user'});
 	local occupants_updated = {};
 	for nick, occupant in self:each_occupant() do -- luacheck: ignore 213
+		local prev_role = occupant.role;
 		occupant.role = nil;
 		self:save_occupant(occupant);
-		occupants_updated[occupant] = true;
+		occupants_updated[occupant] = prev_role;
 	end
-	for occupant in pairs(occupants_updated) do
-		self:publicise_occupant_status(occupant, x);
-		module:fire_event("muc-occupant-left", { room = self; nick = occupant.nick; occupant = occupant;});
+	for occupant, prev_role in pairs(occupants_updated) do
+		self:publicise_occupant_status(occupant, x, nil, nil, nil, prev_role);
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 end
 
@@ -972,7 +1036,7 @@ function room_mt:handle_admin_query_get_command(origin, stanza)
 	local _aff_rank = valid_affiliations[_aff or "none"];
 	local _rol = item.attr.role;
 	if _aff and _aff_rank and not _rol then
-		-- You need to be at least an admin, and be requesting info about your affifiliation or lower
+		-- You need to be at least an admin, and be requesting info about your affiliation or lower
 		-- e.g. an admin can't ask for a list of owners
 		local affiliation_rank = valid_affiliations[affiliation or "none"];
 		if (affiliation_rank >= valid_affiliations.admin and affiliation_rank >= _aff_rank)
@@ -1049,6 +1113,9 @@ end
 function room_mt:handle_groupchat_to_room(origin, stanza)
 	local from = stanza.attr.from;
 	local occupant = self:get_occupant_by_real_jid(from);
+	if not stanza.attr.id then
+		stanza.attr.id = new_id()
+	end
 	if module:fire_event("muc-occupant-groupchat", {
 		room = self; origin = origin; stanza = stanza; from = from; occupant = occupant;
 	}) then return true; end
@@ -1218,7 +1285,7 @@ function room_mt:route_stanza(stanza) -- luacheck: ignore 212
 end
 
 function room_mt:get_affiliation(jid)
-	local node, host, resource = jid_split(jid);
+	local node, host = jid_split(jid);
 	-- Affiliations are granted, revoked, and maintained based on the user's bare JID.
 	local bare = node and node.."@"..host or host;
 	local result = self._affiliations[bare];
@@ -1241,7 +1308,7 @@ end
 function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 	if not actor then return nil, "modify", "not-acceptable"; end;
 
-	local node, host, resource = jid_split(jid);
+	local node, host = jid_split(jid);
 	if not host then return nil, "modify", "not-acceptable"; end
 	jid = jid_join(node, host); -- Bare
 	local is_host_only = node == nil;
@@ -1297,7 +1364,7 @@ function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 			-- Outcast can be by host.
 			is_host_only and affiliation == "outcast" and select(2, jid_split(occupant.bare_jid)) == host
 		) then
-			-- need to publcize in all cases; as affiliation in <item/> has changed.
+			-- need to publicize in all cases; as affiliation in <item/> has changed.
 			occupants_updated[occupant] = occupant.role;
 			if occupant.role ~= role and (
 				is_downgrade or
@@ -1322,16 +1389,20 @@ function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 
 	if next(occupants_updated) ~= nil then
 		for occupant, old_role in pairs(occupants_updated) do
-			self:publicise_occupant_status(occupant, x, nil, actor, reason);
+			self:publicise_occupant_status(occupant, x, nil, actor, reason, old_role);
 			if occupant.role == nil then
-				module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+				module:fire_event("muc-occupant-left", {
+						room = self;
+						nick = occupant.nick;
+						occupant = occupant;
+					});
 			elseif is_semi_anonymous and
 				(old_role == "moderator" and occupant.role ~= "moderator") or
 				(old_role ~= "moderator" and occupant.role == "moderator") then -- Has gained or lost moderator status
 				-- Send everyone else's presences (as jid visibility has changed)
 				for real_jid in occupant:each_session() do
 					self:send_occupant_list(real_jid, function(occupant_jid, occupant) --luacheck: ignore 212 433
-						return occupant.bare_jid ~= jid;
+						return (not occupant) or occupant.bare_jid ~= jid;
 					end);
 				end
 			end
@@ -1376,6 +1447,42 @@ function room_mt:get_role(nick)
 	return occupant and occupant.role or nil;
 end
 
+function room_mt:may_set_role(actor, occupant, role)
+	local event = {
+		room = self,
+		actor = actor,
+		occupant = occupant,
+		role = role,
+	};
+
+	module:fire_event("muc-pre-set-role", event);
+	if event.allowed ~= nil then
+		return event.allowed, event.error, event.condition;
+	end
+
+	-- Can't do anything to other owners or admins
+	local occupant_affiliation = self:get_affiliation(occupant.bare_jid);
+	if occupant_affiliation == "owner" or occupant_affiliation == "admin" then
+		return nil, "cancel", "not-allowed";
+	end
+
+	-- If you are trying to give or take moderator role you need to be an owner or admin
+	if occupant.role == "moderator" or role == "moderator" then
+		local actor_affiliation = self:get_affiliation(actor);
+		if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then
+			return nil, "cancel", "not-allowed";
+		end
+	end
+
+	-- Need to be in the room and a moderator
+	local actor_occupant = self:get_occupant_by_real_jid(actor);
+	if not actor_occupant or actor_occupant.role ~= "moderator" then
+		return nil, "cancel", "not-allowed";
+	end
+
+	return true;
+end
+
 function room_mt:set_role(actor, occupant_jid, role, reason)
 	if not actor then return nil, "modify", "not-acceptable"; end
 
@@ -1390,24 +1497,9 @@ function room_mt:set_role(actor, occupant_jid, role, reason)
 	if actor == true then
 		actor = nil -- So we can pass it safely to 'publicise_occupant_status' below
 	else
-		-- Can't do anything to other owners or admins
-		local occupant_affiliation = self:get_affiliation(occupant.bare_jid);
-		if occupant_affiliation == "owner" or occupant_affiliation == "admin" then
-			return nil, "cancel", "not-allowed";
-		end
-
-		-- If you are trying to give or take moderator role you need to be an owner or admin
-		if occupant.role == "moderator" or role == "moderator" then
-			local actor_affiliation = self:get_affiliation(actor);
-			if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then
-				return nil, "cancel", "not-allowed";
-			end
-		end
-
-		-- Need to be in the room and a moderator
-		local actor_occupant = self:get_occupant_by_real_jid(actor);
-		if not actor_occupant or actor_occupant.role ~= "moderator" then
-			return nil, "cancel", "not-allowed";
+		local allowed, err, condition = self:may_set_role(actor, occupant, role)
+		if not allowed then
+			return allowed, err, condition;
 		end
 	end
 
@@ -1415,11 +1507,17 @@ function room_mt:set_role(actor, occupant_jid, role, reason)
 	if not role then
 		x:tag("status", {code = "307"}):up();
 	end
+
+	local prev_role = occupant.role;
 	occupant.role = role;
 	self:save_occupant(occupant);
-	self:publicise_occupant_status(occupant, x, nil, actor, reason);
+	self:publicise_occupant_status(occupant, x, nil, actor, reason, prev_role);
 	if role == nil then
-		module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 	return true;
 end
@@ -1441,7 +1539,7 @@ function _M.new_room(jid, config)
 	}, room_mt);
 end
 
-local new_format = module:get_option_boolean("new_muc_storage_format", false);
+local new_format = module:get_option_boolean("new_muc_storage_format", true);
 
 function room_mt:freeze(live)
 	local frozen, state;
@@ -1505,7 +1603,7 @@ function _M.restore_room(frozen, state)
 	else
 		-- New storage format
 		for jid, data in pairs(frozen) do
-			local node, host, resource = jid_split(jid);
+			local _, host, resource = jid_split(jid);
 			if host:sub(1,1) ~= "_" and not resource and type(data) == "string" then
 				-- bare jid: affiliation
 				room._affiliations[jid] = data;
diff --git a/plugins/muc/name.lib.lua b/plugins/muc/name.lib.lua
index 37fe1259..5d73e74d 100644
--- a/plugins/muc/name.lib.lua
+++ b/plugins/muc/name.lib.lua
@@ -7,10 +7,8 @@
 -- COPYING file in the source package for more information.
 --
 
-local jid_split = require "util.jid".split;
-
 local function get_name(room)
-	return room._data.name or jid_split(room.jid);
+	return room._data.name;
 end
 
 local function set_name(room, name)
diff --git a/plugins/muc/password.lib.lua b/plugins/muc/password.lib.lua
index 1f4b2add..6695c0cf 100644
--- a/plugins/muc/password.lib.lua
+++ b/plugins/muc/password.lib.lua
@@ -50,7 +50,7 @@ module:hook("muc-occupant-pre-join", function(event)
 	if get_password(room) ~= password then
 		local from, to = stanza.attr.from, stanza.attr.to;
 		module:log("debug", "%s couldn't join due to invalid password: %s", from, to);
-		local reply = st.error_reply(stanza, "auth", "not-authorized"):up();
+		local reply = st.error_reply(stanza, "auth", "not-authorized", nil, room.jid):up();
 		reply.tags[1].attr.code = "401";
 		event.origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
diff --git a/plugins/muc/presence_broadcast.lib.lua b/plugins/muc/presence_broadcast.lib.lua
new file mode 100644
index 00000000..82a89fee
--- /dev/null
+++ b/plugins/muc/presence_broadcast.lib.lua
@@ -0,0 +1,83 @@
+-- Prosody IM
+-- Copyright (C) 2008-2010 Matthew Wild
+-- Copyright (C) 2008-2010 Waqas Hussain
+-- Copyright (C) 2014 Daurnimator
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local st = require "util.stanza";
+
+local valid_roles = { "none", "visitor", "participant", "moderator" };
+local default_broadcast = {
+	visitor = true;
+	participant = true;
+	moderator = true;
+};
+
+local function get_presence_broadcast(room)
+	return room._data.presence_broadcast or default_broadcast;
+end
+
+local function set_presence_broadcast(room, broadcast_roles)
+	broadcast_roles = broadcast_roles or default_broadcast;
+
+	local changed = false;
+	local old_broadcast_roles = get_presence_broadcast(room);
+	for _, role in ipairs(valid_roles) do
+		if old_broadcast_roles[role] ~= broadcast_roles[role] then
+			changed = true;
+		end
+	end
+
+	if not changed then return false; end
+
+	room._data.presence_broadcast = broadcast_roles;
+
+	for _, occupant in room:each_occupant() do
+		local x = st.stanza("x", {xmlns = "http://jabber.org/protocol/muc#user";});
+		local role = occupant.role or "none";
+		if broadcast_roles[role] and not old_broadcast_roles[role] then
+			-- Presence broadcast is now enabled, so announce existing user
+			room:publicise_occupant_status(occupant, x);
+		elseif old_broadcast_roles[role] and not broadcast_roles[role] then
+			-- Presence broadcast is now disabled, so mark existing user as unavailable
+			room:publicise_occupant_status(occupant, x, nil, nil, nil, nil, true);
+		end
+	end
+
+	return true;
+end
+
+module:hook("muc-config-form", function(event)
+	local values = {};
+	for role, value in pairs(get_presence_broadcast(event.room)) do
+		if value then
+			values[#values + 1] = role;
+		end
+	end
+
+	table.insert(event.form, {
+		name = "muc#roomconfig_presencebroadcast";
+		type = "list-multi";
+		label = "Only show participants with roles:";
+		value = values;
+		options = valid_roles;
+	});
+end, 70-7);
+
+module:hook("muc-config-submitted/muc#roomconfig_presencebroadcast", function(event)
+	local broadcast_roles = {};
+	for _, role in ipairs(event.value) do
+		broadcast_roles[role] = true;
+	end
+	if set_presence_broadcast(event.room, broadcast_roles) then
+		event.status_codes["104"] = true;
+	end
+end);
+
+return {
+	get = get_presence_broadcast;
+	set = set_presence_broadcast;
+};
diff --git a/plugins/muc/register.lib.lua b/plugins/muc/register.lib.lua
index 95ed1a84..d917b396 100644
--- a/plugins/muc/register.lib.lua
+++ b/plugins/muc/register.lib.lua
@@ -15,8 +15,7 @@ local function get_reserved_nicks(room)
 	end
 	module:log("debug", "Refreshing reserved nicks...");
 	local reserved_nicks = {};
-	for jid in room:each_affiliation() do
-		local data = room._affiliation_data[jid];
+	for jid, _, data in room:each_affiliation() do
 		local nick = data and data.reserved_nickname;
 		module:log("debug", "Refreshed for %s: %s", jid, nick);
 		if nick then
@@ -54,7 +53,7 @@ end);
 
 local registration_form = dataforms.new {
 	{ name = "FORM_TYPE", type = "hidden", value = "http://jabber.org/protocol/muc#register" },
-	{ name = "muc#register_roomnick", type = "text-single", label = "Nickname"},
+	{ name = "muc#register_roomnick", type = "text-single", required = true, label = "Nickname"},
 };
 
 local function enforce_nick_policy(event)
@@ -67,7 +66,7 @@ local function enforce_nick_policy(event)
 	local reserved_by = get_registered_jid(room, requested_nick);
 	if reserved_by and reserved_by ~= jid_bare(stanza.attr.from) then
 		module:log("debug", "%s attempted to use nick %s reserved by %s", stanza.attr.from, requested_nick, reserved_by);
-		local reply = st.error_reply(stanza, "cancel", "conflict"):up();
+		local reply = st.error_reply(stanza, "cancel", "conflict", nil, room.jid):up();
 		origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 		return true;
 	end
@@ -80,7 +79,7 @@ local function enforce_nick_policy(event)
 				event.occupant.nick = jid_bare(event.occupant.nick) .. "/" .. nick;
 			elseif event.dest_occupant.nick ~= jid_bare(event.dest_occupant.nick) .. "/" .. nick then
 				module:log("debug", "Attempt by %s to join as %s, but their reserved nick is %s", stanza.attr.from, requested_nick, nick);
-				local reply = st.error_reply(stanza, "cancel", "not-acceptable"):up();
+				local reply = st.error_reply(stanza, "cancel", "not-acceptable", nil, room.jid):up();
 				origin.send(reply:tag("x", {xmlns = "http://jabber.org/protocol/muc"}));
 				return true;
 			end
@@ -104,7 +103,7 @@ local function handle_register_iq(room, origin, stanza)
 	local user_jid = jid_bare(stanza.attr.from)
 	local affiliation = room:get_affiliation(user_jid);
 	if affiliation == "outcast" then
-		origin.send(st.error_reply(stanza, "auth", "forbidden"));
+		origin.send(st.error_reply(stanza, "auth", "forbidden", room.jid));
 		return true;
 	elseif not (affiliation or allow_unaffiliated) then
 		origin.send(st.error_reply(stanza, "auth", "registration-required"));
@@ -135,13 +134,25 @@ local function handle_register_iq(room, origin, stanza)
 			return true;
 		end
 		local form_tag = query:get_child("x", "jabber:x:data");
-		local reg_data = form_tag and registration_form:data(form_tag);
+		if not form_tag then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Missing dataform"));
+			return true;
+		end
+		local form_type, err = dataforms.get_type(form_tag);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error with form: "..err));
+			return true;
+		elseif form_type ~= "http://jabber.org/protocol/muc#register" then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error in form"));
+			return true;
+		end
+		local reg_data = registration_form:data(form_tag);
 		if not reg_data then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error in form"));
 			return true;
 		end
 		-- Is the nickname valid?
-		local desired_nick = resourceprep(reg_data["muc#register_roomnick"]);
+		local desired_nick = resourceprep(reg_data["muc#register_roomnick"], true);
 		if not desired_nick then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid Nickname"));
 			return true;
diff --git a/plugins/muc/subject.lib.lua b/plugins/muc/subject.lib.lua
index 14256bbc..3230817c 100644
--- a/plugins/muc/subject.lib.lua
+++ b/plugins/muc/subject.lib.lua
@@ -94,6 +94,12 @@ module:hook("muc-occupant-groupchat", function(event)
 	local stanza = event.stanza;
 	local subject = stanza:get_child("subject");
 	if subject then
+		if stanza:get_child("body") or stanza:get_child("thread") then
+			-- Note: A message with a <subject/> and a <body/> or a <subject/> and
+			-- a <thread/> is a legitimate message, but it SHALL NOT be interpreted
+			-- as a subject change.
+			return;
+		end
 		local room = event.room;
 		local occupant = event.occupant;
 		-- Role check for subject changes
diff --git a/plugins/muc/util.lib.lua b/plugins/muc/util.lib.lua
index 53a83fae..0877150f 100644
--- a/plugins/muc/util.lib.lua
+++ b/plugins/muc/util.lib.lua
@@ -41,18 +41,22 @@ function _M.is_kickable_error(stanza)
 	return kickable_error_conditions[cond];
 end
 
-local muc_x_filters = {
-	["http://jabber.org/protocol/muc"] = true;
-	["http://jabber.org/protocol/muc#user"] = true;
-}
-local function muc_x_filter(tag)
-	if muc_x_filters[tag.attr.xmlns] then
+local filtered_namespaces = module:shared("filtered-namespaces");
+filtered_namespaces["http://jabber.org/protocol/muc"] = true;
+filtered_namespaces["http://jabber.org/protocol/muc#user"] = true;
+
+local function muc_ns_filter(tag)
+	if filtered_namespaces[tag.attr.xmlns] then
 		return nil;
 	end
 	return tag;
 end
 function _M.filter_muc_x(stanza)
-	return stanza:maptags(muc_x_filter);
+	return stanza:maptags(muc_ns_filter);
+end
+
+function _M.add_filtered_namespace(xmlns)
+	filtered_namespaces[xmlns] = true;
 end
 
 function _M.only_with_min_role(role)