Merge 0.11->trunk

49 files changed, 1526 insertions, 976 deletions

diff --git a/plugins/mod_admin_adhoc.lua b/plugins/mod_admin_adhoc.lua
index 37e77ab0..4fea5a73 100644
--- a/plugins/mod_admin_adhoc.lua
+++ b/plugins/mod_admin_adhoc.lua
@@ -392,6 +392,12 @@ local function session_flags(session, line)
 	if session.cert_identity_status == "valid" then
 		flags[#flags+1] = "authenticated";
 	end
+	if session.dialback_key then
+		flags[#flags+1] = "dialback";
+	end
+	if session.external_auth then
+		flags[#flags+1] = "SASL";
+	end
 	if session.secure then
 		flags[#flags+1] = "encrypted";
 	end
@@ -404,6 +410,12 @@ local function session_flags(session, line)
 	if session.ip and session.ip:match(":") then
 		flags[#flags+1] = "IPv6";
 	end
+	if session.incoming and session.outgoing then
+		flags[#flags+1] = "bidi";
+	elseif session.is_bidi or session.bidi_session then
+		flags[#flags+1] = "bidi";
+	end
+
 	line[#line+1] = "("..t_concat(flags, ", ")..")";
 
 	return t_concat(line, " ");
diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua
index 1cbe27a4..cef79d25 100644
--- a/plugins/mod_admin_telnet.lua
+++ b/plugins/mod_admin_telnet.lua
@@ -22,6 +22,7 @@ local prosody = _G.prosody;
 
 local console_listener = { default_port = 5582; default_mode = "*a"; interface = "127.0.0.1" };
 
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local iterators = require "util.iterators";
 local keys, values = iterators.keys, iterators.values;
 local jid_bare, jid_split, jid_join = import("util.jid", "bare", "prepped_split", "join");
@@ -30,6 +31,9 @@ local cert_verify_identity = require "util.x509".verify_identity;
 local envload = require "util.envload".envload;
 local envloadfile = require "util.envload".envloadfile;
 local has_pposix, pposix = pcall(require, "util.pposix");
+local async = require "util.async";
+local serialize = require "util.serialization".new({ fatal = false, unquoted = true});
+local time = require "util.time";
 
 local commands = module:shared("commands")
 local def_env = module:shared("env");
@@ -47,6 +51,24 @@ end
 
 console = {};
 
+local runner_callbacks = {};
+
+function runner_callbacks:ready()
+	self.data.conn:resume();
+end
+
+function runner_callbacks:waiting()
+	self.data.conn:pause();
+end
+
+function runner_callbacks:error(err)
+	module:log("error", "Traceback[telnet]: %s", err);
+
+	self.data.print("Fatal error while running command, it did not complete");
+	self.data.print("Error: "..tostring(err));
+end
+
+
 function console:new_session(conn)
 	local w = function(s) conn:write(s:gsub("\n", "\r\n")); end;
 	local session = { conn = conn;
@@ -62,6 +84,11 @@ function console:new_session(conn)
 			};
 	session.env = setmetatable({}, default_env_mt);
 
+	session.thread = async.runner(function (line)
+		console:process_line(session, line);
+		session.send(string.char(0));
+	end, runner_callbacks, session);
+
 	-- Load up environment with helper objects
 	for name, t in pairs(def_env) do
 		if type(t) == "table" then
@@ -91,6 +118,11 @@ function console:process_line(session, line)
 
 	session.env._ = line;
 
+	if not useglobalenv and commands[line:lower()] then
+		commands[line:lower()](session, line);
+		return;
+	end
+
 	local chunkname = "=console";
 	local env = (useglobalenv and redirect_output(_G, session)) or session.env or nil
 	local chunk, err = envload("return "..line, chunkname, env);
@@ -105,18 +137,7 @@ function console:process_line(session, line)
 		end
 	end
 
-	local ranok, taskok, message = pcall(chunk);
-
-	if not (ranok or message or useglobalenv) and commands[line:lower()] then
-		commands[line:lower()](session, line);
-		return;
-	end
-
-	if not ranok then
-		session.print("Fatal error while running command, it did not complete");
-		session.print("Error: "..taskok);
-		return;
-	end
+	local taskok, message = chunk();
 
 	if not message then
 		session.print("Result: "..tostring(taskok));
@@ -150,8 +171,7 @@ function console_listener.onincoming(conn, data)
 
 	for line in data:gmatch("[^\n]*[\n\004]") do
 		if session.closed then return end
-		console:process_line(session, line);
-		session.send(string.char(0));
+		session.thread:run(line);
 	end
 	session.partial_data = data:match("[^\n]+$");
 end
@@ -220,6 +240,7 @@ function commands.help(session, data)
 		print [[server - Uptime, version, shutting down, etc.]]
 		print [[port - Commands to manage ports the server is listening on]]
 		print [[dns - Commands to manage and inspect the internal DNS resolver]]
+		print [[xmpp - Commands for sending XMPP stanzas]]
 		print [[config - Reloading the configuration, etc.]]
 		print [[console - Help regarding the console itself]]
 	elseif section == "c2s" then
@@ -227,7 +248,9 @@ function commands.help(session, data)
 		print [[c2s:show_insecure() - Show all unencrypted client connections]]
 		print [[c2s:show_secure() - Show all encrypted client connections]]
 		print [[c2s:show_tls() - Show TLS cipher info for encrypted sessions]]
+		print [[c2s:count() - Count sessions without listing them]]
 		print [[c2s:close(jid) - Close all sessions for the specified JID]]
+		print [[c2s:closeall() - Close all active c2s connections ]]
 	elseif section == "s2s" then
 		print [[s2s:show(domain) - Show all s2s connections for the given domain (or all if no domain given)]]
 		print [[s2s:show_tls(domain) - Show TLS cipher info for encrypted sessions]]
@@ -261,6 +284,8 @@ function commands.help(session, data)
 		print [[dns:setnameserver(nameserver) - Replace the list of name servers with the supplied one]]
 		print [[dns:purge() - Clear the DNS cache]]
 		print [[dns:cache() - Show cached records]]
+	elseif section == "xmpp" then
+		print [[xmpp:ping(localhost, remotehost) -- Sends a ping to a remote XMPP server and reports the response]]
 	elseif section == "config" then
 		print [[config:reload() - Reload the server configuration. Modules may need to be reloaded for changes to take effect.]]
 	elseif section == "console" then
@@ -458,7 +483,12 @@ function def_env.module:list(hosts)
 			end
 		else
 			for _, name in ipairs(modules) do
-				print("    "..name);
+				local status, status_text = modulemanager.get_module(host, name).module:get_status();
+				local status_summary = "";
+				if status == "warn" or status == "error" then
+					status_summary = (" (%s: %s)"):format(status, status_text);
+				end
+				print(("    %s%s"):format(name, status_summary));
 			end
 		end
 	end
@@ -474,9 +504,12 @@ function def_env.config:load(filename, format)
 	return true, "Config loaded";
 end
 
-function def_env.config:get(host, section, key)
+function def_env.config:get(host, key)
+	if key == nil then
+		host, key = "*", host;
+	end
 	local config_get = require "core.configmanager".get
-	return true, tostring(config_get(host, section, key));
+	return true, serialize(config_get(host, key));
 end
 
 function def_env.config:reload()
@@ -505,6 +538,12 @@ local function session_flags(session, line)
 	if session.cert_identity_status == "valid" then
 		line[#line+1] = "(authenticated)";
 	end
+	if session.dialback_key then
+		line[#line+1] = "(dialback)";
+	end
+	if session.external_auth then
+		line[#line+1] = "(SASL)";
+	end
 	if session.secure then
 		line[#line+1] = "(encrypted)";
 	end
@@ -520,6 +559,17 @@ local function session_flags(session, line)
 	if session.remote then
 		line[#line+1] = "(remote)";
 	end
+	if session.incoming and session.outgoing then
+		line[#line+1] = "(bidi)";
+	elseif session.is_bidi or session.bidi_session then
+		line[#line+1] = "(bidi)";
+	end
+	if session.bosh_version then
+		line[#line+1] = "(bosh)";
+	end
+	if session.websocket_request then
+		line[#line+1] = "(websocket)";
+	end
 	return table.concat(line, " ");
 end
 
@@ -534,6 +584,12 @@ local function tls_info(session, line)
 		else
 			line[#line+1] = "(cipher info unavailable)";
 		end
+		if sock.getsniname then
+			local name = sock:getsniname();
+			if name then
+				line[#line+1] = ("(SNI:%q)"):format(name);
+			end
+		end
 	else
 		line[#line+1] = "(insecure)";
 	end
@@ -555,9 +611,16 @@ local function get_jid(session)
 	return jid_join("["..ip.."]:"..clientport, session.host or "["..serverip.."]:"..serverport);
 end
 
+local function get_c2s()
+	local c2s = array.collect(values(prosody.full_sessions));
+	c2s:append(array.collect(values(module:shared"/*/c2s/sessions")));
+	c2s:append(array.collect(values(module:shared"/*/bosh/sessions")));
+	c2s:unique();
+	return c2s;
+end
+
 local function show_c2s(callback)
-	local c2s = array.collect(values(module:shared"/*/c2s/sessions"));
-	c2s:sort(function(a, b)
+	get_c2s():sort(function(a, b)
 		if a.host == b.host then
 			if a.username == b.username then
 				return (a.resource or "") > (b.resource or "");
@@ -571,7 +634,8 @@ local function show_c2s(callback)
 end
 
 function def_env.c2s:count()
-	return true, "Total: "..  iterators.count(values(module:shared"/*/c2s/sessions")) .." clients";
+	local c2s = get_c2s();
+	return true, "Total: "..  #c2s .." clients";
 end
 
 function def_env.c2s:show(match_jid, annotate)
@@ -617,17 +681,36 @@ function def_env.c2s:show_tls(match_jid)
 	return self:show(match_jid, tls_info);
 end
 
-function def_env.c2s:close(match_jid)
+local function build_reason(text, condition)
+	if text or condition then
+		return {
+			text = text,
+			condition = condition or "undefined-condition",
+		};
+	end
+end
+
+function def_env.c2s:close(match_jid, text, condition)
 	local count = 0;
 	show_c2s(function (jid, session)
 		if jid == match_jid or jid_bare(jid) == match_jid then
 			count = count + 1;
-			session:close();
+			session:close(build_reason(text, condition));
 		end
 	end);
 	return true, "Total: "..count.." sessions closed";
 end
 
+function def_env.c2s:closeall(text, condition)
+	local count = 0;
+	--luacheck: ignore 212/jid
+	show_c2s(function (jid, session)
+		count = count + 1;
+		session:close(build_reason(text, condition));
+	end);
+	return true, "Total: "..count.." sessions closed";
+end
+
 
 def_env.s2s = {};
 function def_env.s2s:show(match_jid, annotate)
@@ -828,7 +911,7 @@ function def_env.s2s:showcert(domain)
 		.." presented by "..domain..".");
 end
 
-function def_env.s2s:close(from, to)
+function def_env.s2s:close(from, to, text, condition)
 	local print, count = self.session.print, 0;
 	local s2s_sessions = module:shared"/*/s2s/sessions";
 
@@ -842,23 +925,23 @@ function def_env.s2s:close(from, to)
 	end
 
 	for _, session in pairs(s2s_sessions) do
-		local id = session.type..tostring(session):match("[a-f0-9]+$");
+		local id = session.id or (session.type..tostring(session):match("[a-f0-9]+$"));
 		if (match_id and match_id == id)
 		or (session.from_host == from and session.to_host == to) then
 			print(("Closing connection from %s to %s [%s]"):format(session.from_host, session.to_host, id));
-			(session.close or s2smanager.destroy_session)(session);
+			(session.close or s2smanager.destroy_session)(session, build_reason(text, condition));
 			count = count + 1 ;
 		end
 	end
 	return true, "Closed "..count.." s2s session"..((count == 1 and "") or "s");
 end
 
-function def_env.s2s:closeall(host)
+function def_env.s2s:closeall(host, text, condition)
 	local count = 0;
 	local s2s_sessions = module:shared"/*/s2s/sessions";
 	for _,session in pairs(s2s_sessions) do
 		if not host or session.from_host == host or session.to_host == host then
-			session:close();
+			session:close(build_reason(text, condition));
 			count = count + 1;
 		end
 	end
@@ -1062,13 +1145,28 @@ end
 def_env.xmpp = {};
 
 local st = require "util.stanza";
-function def_env.xmpp:ping(localhost, remotehost)
-	if prosody.hosts[localhost] then
-		module:send(st.iq{ from=localhost, to=remotehost, type="get", id="ping" }
-				:tag("ping", {xmlns="urn:xmpp:ping"}), prosody.hosts[localhost]);
-		return true, "Sent ping";
+local new_id = require "util.id".medium;
+function def_env.xmpp:ping(localhost, remotehost, timeout)
+	localhost = select(2, jid_split(localhost));
+	remotehost = select(2, jid_split(remotehost));
+	if not localhost then
+		return nil, "Invalid sender hostname";
+	elseif not prosody.hosts[localhost] then
+		return nil, "No such local host";
+	end
+	if not remotehost then
+		return nil, "Invalid destination hostname";
+	elseif prosody.hosts[remotehost] then
+		return nil, "Both hosts are local";
+	end
+	local iq = st.iq{ from=localhost, to=remotehost, type="get", id=new_id()}
+			:tag("ping", {xmlns="urn:xmpp:ping"});
+	local time_start = time.now();
+	local ret, err = async.wait(module:context(localhost):send_iq(iq, nil, timeout));
+	if ret then
+		return true, ("pong from %s in %gs"):format(ret.stanza.attr.from, time.now() - time_start);
 	else
-		return nil, "No such host";
+		return false, tostring(err);
 	end
 end
 
@@ -1207,7 +1305,7 @@ local function format_stat(type, value, ref_value)
 	--do return tostring(value) end
 	if type == "duration" then
 		if ref_value < 0.001 then
-			return ("%d µs"):format(value*1000000);
+			return ("%g µs"):format(value*1000000);
 		elseif ref_value < 0.9 then
 			return ("%0.2f ms"):format(value*1000);
 		end
@@ -1495,7 +1593,7 @@ function def_env.stats:show(filter)
 	local stats, changed, extra = require "core.statsmanager".get_stats();
 	local available, displayed = 0, 0;
 	local displayed_stats = new_stats_context(self);
-	for name, value in pairs(stats) do
+	for name, value in iterators.sorted_pairs(stats) do
 		available = available + 1;
 		if not filter or name:match(filter) then
 			displayed = displayed + 1;
diff --git a/plugins/mod_auth_internal_hashed.lua b/plugins/mod_auth_internal_hashed.lua
index 35764afb..be22a8d8 100644
--- a/plugins/mod_auth_internal_hashed.lua
+++ b/plugins/mod_auth_internal_hashed.lua
@@ -9,7 +9,7 @@
 
 local max = math.max;
 
-local getAuthenticationDatabaseSHA1 = require "util.sasl.scram".getAuthenticationDatabaseSHA1;
+local scram_hashers = require "util.sasl.scram".hashers;
 local usermanager = require "core.usermanager";
 local generate_uuid = require "util.uuid".generate;
 local new_sasl = require "util.sasl".new;
@@ -21,7 +21,9 @@ local host = module.host;
 
 local accounts = module:open_store("accounts");
 
-
+local hash_name = module:get_option_string("password_hash", "SHA-1");
+local get_auth_db = assert(scram_hashers[hash_name], "SCRAM-"..hash_name.." not supported by SASL library");
+local scram_name = "scram_"..hash_name:gsub("%-","_"):lower();
 
 -- Default; can be set per-user
 local default_iteration_count = 4096;
@@ -49,7 +51,7 @@ function provider.test_password(username, password)
 		return nil, "Auth failed. Stored salt and iteration count information is not complete.";
 	end
 
-	local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, credentials.salt, credentials.iteration_count);
+	local valid, stored_key, server_key = get_auth_db(password, credentials.salt, credentials.iteration_count);
 
 	local stored_key_hex = to_hex(stored_key);
 	local server_key_hex = to_hex(server_key);
@@ -67,7 +69,7 @@ function provider.set_password(username, password)
 	if account then
 		account.salt = generate_uuid();
 		account.iteration_count = max(account.iteration_count or 0, default_iteration_count);
-		local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, account.salt, account.iteration_count);
+		local valid, stored_key, server_key = get_auth_db(password, account.salt, account.iteration_count);
 		local stored_key_hex = to_hex(stored_key);
 		local server_key_hex = to_hex(server_key);
 
@@ -98,7 +100,7 @@ function provider.create_user(username, password)
 		return accounts:set(username, {});
 	end
 	local salt = generate_uuid();
-	local valid, stored_key, server_key = getAuthenticationDatabaseSHA1(password, salt, default_iteration_count);
+	local valid, stored_key, server_key = get_auth_db(password, salt, default_iteration_count);
 	local stored_key_hex = to_hex(stored_key);
 	local server_key_hex = to_hex(server_key);
 	return accounts:set(username, {
@@ -116,7 +118,7 @@ function provider.get_sasl_handler()
 		plain_test = function(_, username, password, realm)
 			return usermanager.test_password(username, realm, password), true;
 		end,
-		scram_sha_1 = function(_, username)
+		[scram_name] = function(_, username)
 			local credentials = accounts:get(username);
 			if not credentials then return; end
 			if credentials.password then
diff --git a/plugins/mod_blocklist.lua b/plugins/mod_blocklist.lua
index ee48ffad..dad06b62 100644
--- a/plugins/mod_blocklist.lua
+++ b/plugins/mod_blocklist.lua
@@ -67,7 +67,7 @@ local function migrate_privacy_list(username)
 		if item.type == "jid" and item.action == "deny" then
 			local jid = jid_prep(item.value);
 			if not jid then
-				module:log("warn", "Invalid JID in privacy store for user '%s' not migrated: %s", username, tostring(item.value));
+				module:log("warn", "Invalid JID in privacy store for user '%s' not migrated: %s", username, item.value);
 			else
 				migrated_data[jid] = true;
 			end
@@ -162,7 +162,7 @@ local function edit_blocklist(event)
 	local blocklist = cache[username] or get_blocklist(username);
 
 	local new_blocklist = {
-		-- We set the [false] key to someting as a signal not to migrate privacy lists
+		-- We set the [false] key to something as a signal not to migrate privacy lists
 		[false] = blocklist[false] or { created = now; };
 	};
 	if type(blocklist[false]) == "table" then
@@ -189,6 +189,7 @@ local function edit_blocklist(event)
 
 	if is_blocking then
 		for jid in pairs(send_unavailable) do
+			-- Check that this JID isn't already blocked, i.e. this is not a change
 			if not blocklist[jid] then
 				for _, session in pairs(sessions[username].sessions) do
 					if session.presence then
diff --git a/plugins/mod_bosh.lua b/plugins/mod_bosh.lua
index d4701148..b45a9dc2 100644
--- a/plugins/mod_bosh.lua
+++ b/plugins/mod_bosh.lua
@@ -44,19 +44,41 @@ local bosh_max_polling = module:get_option_number("bosh_max_polling", 5);
 local bosh_max_wait = module:get_option_number("bosh_max_wait", 120);
 
 local consider_bosh_secure = module:get_option_boolean("consider_bosh_secure");
-local cross_domain = module:get_option("cross_domain_bosh", false);
+local cross_domain = module:get_option("cross_domain_bosh");
 
-if cross_domain == true then cross_domain = "*"; end
-if type(cross_domain) == "table" then cross_domain = table.concat(cross_domain, ", "); end
+if cross_domain ~= nil then
+	module:log("info", "The 'cross_domain_bosh' option has been deprecated");
+end
 
 local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat;
 
 -- All sessions, and sessions that have no requests open
 local sessions = module:shared("sessions");
 
+local measure_active = module:measure("active_sessions", "amount");
+local measure_inactive = module:measure("inactive_sessions", "amount");
+local report_bad_host = module:measure("bad_host", "rate");
+local report_bad_sid = module:measure("bad_sid", "rate");
+local report_new_sid = module:measure("new_sid", "rate");
+local report_timeout = module:measure("timeout", "rate");
+
+module:hook("stats-update", function ()
+	local active = 0;
+	local inactive = 0;
+	for _, session in pairs(sessions) do
+		if #session.requests > 0 then
+			active = active + 1;
+		else
+			inactive = inactive + 1;
+		end
+	end
+	measure_active(active);
+	measure_inactive(inactive);
+end);
+
 -- Used to respond to idle sessions (those with waiting requests)
 function on_destroy_request(request)
-	log("debug", "Request destroyed: %s", tostring(request));
+	log("debug", "Request destroyed: %s", request);
 	local session = sessions[request.context.sid];
 	if session then
 		local requests = session.requests;
@@ -73,7 +95,7 @@ function on_destroy_request(request)
 			if session.inactive_timer then
 				session.inactive_timer:stop();
 			end
-			session.inactive_timer = module:add_timer(max_inactive, check_inactive, session, request.context,
+			session.inactive_timer = module:add_timer(max_inactive, session_timeout, session, request.context,
 				"BOSH client silent for over "..max_inactive.." seconds");
 			(session.log or log)("debug", "BOSH session marked as inactive (for %ds)", max_inactive);
 		end
@@ -84,31 +106,16 @@ function on_destroy_request(request)
 	end
 end
 
-function check_inactive(now, session, context, reason) -- luacheck: ignore 212/now
+function session_timeout(now, session, context, reason) -- luacheck: ignore 212/now
 	if not session.destroyed then
+		report_timeout();
 		sessions[context.sid] = nil;
 		sm_destroy_session(session, reason);
 	end
 end
 
-local function set_cross_domain_headers(response)
-	local headers = response.headers;
-	headers.access_control_allow_methods = "GET, POST, OPTIONS";
-	headers.access_control_allow_headers = "Content-Type";
-	headers.access_control_max_age = "7200";
-	headers.access_control_allow_origin = cross_domain;
-	return response;
-end
-
-function handle_OPTIONS(event)
-	if cross_domain and event.request.headers.origin then
-		set_cross_domain_headers(event.response);
-	end
-	return "";
-end
-
 function handle_POST(event)
-	log("debug", "Handling new request %s: %s\n----------", tostring(event.request), tostring(event.request.body));
+	log("debug", "Handling new request %s: %s\n----------", event.request, event.request.body);
 
 	local request, response = event.request, event.response;
 	response.on_destroy = on_destroy_request;
@@ -121,10 +128,6 @@ function handle_POST(event)
 	local headers = response.headers;
 	headers.content_type = "text/xml; charset=utf-8";
 
-	if cross_domain and request.headers.origin then
-		set_cross_domain_headers(response);
-	end
-
 	-- stream:feed() calls the stream_callbacks, so all stanzas in
 	-- the body are processed in this next line before it returns.
 	-- In particular, the streamopened() stream callback is where
@@ -205,6 +208,7 @@ function handle_POST(event)
 		return;
 	end
 	module:log("warn", "Unable to associate request with a session (incomplete request?)");
+	report_bad_sid();
 	local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 		["xmlns:stream"] = xmlns_streams, condition = "item-not-found" });
 	return tostring(close_reply) .. "\n";
@@ -220,7 +224,7 @@ local function bosh_reset_stream(session) session.notopen = true; end
 
 local stream_xmlns_attr = { xmlns = "urn:ietf:params:xml:ns:xmpp-streams" };
 local function bosh_close_stream(session, reason)
-	(session.log or log)("info", "BOSH client disconnected: %s", tostring((reason and reason.condition or reason) or "session close"));
+	(session.log or log)("info", "BOSH client disconnected: %s", (reason and reason.condition or reason) or "session close");
 
 	local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 		["xmlns:stream"] = xmlns_streams });
@@ -245,7 +249,7 @@ local function bosh_close_stream(session, reason)
 				close_reply = reason;
 			end
 		end
-		log("info", "Disconnecting client, <stream:error> is: %s", tostring(close_reply));
+		log("info", "Disconnecting client, <stream:error> is: %s", close_reply);
 	end
 
 	local response_body = tostring(close_reply);
@@ -268,17 +272,27 @@ function stream_callbacks.streamopened(context, attr)
 		-- New session request
 		context.notopen = nil; -- Signals that we accept this opening tag
 
+		if not attr.to then
+			log("debug", "BOSH client tried to connect without specifying a host");
+			report_bad_host();
+			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
+				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
+			response:send(tostring(close_reply));
+			return;
+		end
+
 		local to_host = nameprep(attr.to);
 		local wait = tonumber(attr.wait);
 		if not to_host then
-			log("debug", "BOSH client tried to connect to invalid host: %s", tostring(attr.to));
+			log("debug", "BOSH client tried to connect to invalid host: %s", attr.to);
+			report_bad_host();
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "improper-addressing" });
 			response:send(tostring(close_reply));
 			return;
 		end
 		if not rid or (not attr.wait or not wait or wait < 0 or wait % 1 ~= 0) then
-			log("debug", "BOSH client sent invalid rid or wait attributes: rid=%s, wait=%s", tostring(attr.rid), tostring(attr.wait));
+			log("debug", "BOSH client sent invalid rid or wait attributes: rid=%s, wait=%s", attr.rid, attr.wait);
 			local close_reply = st.stanza("body", { xmlns = xmlns_bosh, type = "terminate",
 				["xmlns:stream"] = xmlns_streams, condition = "bad-request" });
 			response:send(tostring(close_reply));
@@ -309,6 +323,7 @@ function stream_callbacks.streamopened(context, attr)
 
 		session.log("debug", "BOSH session created for request from %s", session.ip);
 		log("info", "New BOSH session, assigned it sid '%s'", sid);
+		report_new_sid();
 
 		module:fire_event("bosh-session", { session = session, request = request });
 
@@ -323,7 +338,7 @@ function stream_callbacks.streamopened(context, attr)
 				s.attr.xmlns = "jabber:client";
 			end
 			s = filter("stanzas/out", s);
-			--log("debug", "Sending BOSH data: %s", tostring(s));
+			--log("debug", "Sending BOSH data: %s", s);
 			if not s then return true end
 			t_insert(session.send_buffer, tostring(s));
 
@@ -363,6 +378,7 @@ function stream_callbacks.streamopened(context, attr)
 	if not session then
 		-- Unknown sid
 		log("info", "Client tried to use sid '%s' which we don't know about", sid);
+		report_bad_sid();
 		response:send(tostring(st.stanza("body", { xmlns = xmlns_bosh, type = "terminate", condition = "item-not-found" })));
 		context.notopen = nil;
 		return;
@@ -425,7 +441,7 @@ function stream_callbacks.streamopened(context, attr)
 	end
 end
 
-local function handleerr(err) log("error", "Traceback[bosh]: %s", traceback(tostring(err), 2)); end
+local function handleerr(err) log("error", "Traceback[bosh]: %s", traceback(err, 2)); end
 
 function runner_callbacks:error(err) -- luacheck: ignore 212/self
 	return handleerr(err);
@@ -511,8 +527,6 @@ module:provides("http", {
 	route = {
 		["GET"] = GET_response;
 		["GET /"] = GET_response;
-		["OPTIONS"] = handle_OPTIONS;
-		["OPTIONS /"] = handle_OPTIONS;
 		["POST"] = handle_POST;
 		["POST /"] = handle_POST;
 	};
diff --git a/plugins/mod_c2s.lua b/plugins/mod_c2s.lua
index 15d3a9be..aec0370d 100644
--- a/plugins/mod_c2s.lua
+++ b/plugins/mod_c2s.lua
@@ -56,6 +56,11 @@ local stream_xmlns_attr = {xmlns='urn:ietf:params:xml:ns:xmpp-streams'};
 
 function stream_callbacks.streamopened(session, attr)
 	local send = session.send;
+	if not attr.to then
+		session:close{ condition = "improper-addressing",
+			text = "A 'to' attribute is required on stream headers" };
+		return;
+	end
 	local host = nameprep(attr.to);
 	if not host then
 		session:close{ condition = "improper-addressing",
@@ -97,7 +102,6 @@ function stream_callbacks.streamopened(session, attr)
 			session.compressed = info.compression;
 		else
 			(session.log or log)("info", "Stream encrypted");
-			session.compressed = sock.compression and sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -106,7 +110,13 @@ function stream_callbacks.streamopened(session, attr)
 	if features.tags[1] or session.full_jid then
 		send(features);
 	else
-		(session.log or log)("warn", "No stream features to offer");
+		if session.secure then
+			-- Normally STARTTLS would be offered
+			(session.log or log)("warn", "No stream features to offer on secure session. Check authentication settings.");
+		else
+			-- Here SASL should be offered
+			(session.log or log)("warn", "No stream features to offer on insecure session. Check encryption and security settings.");
+		end
 		session:close{ condition = "undefined-condition", text = "No stream features to proceed with" };
 	end
 end
@@ -121,7 +131,7 @@ function stream_callbacks.error(session, error, data)
 		session.log("debug", "Invalid opening stream header (%s)", (data:gsub("^([^\1]+)\1", "{%1}")));
 		session:close("invalid-namespace");
 	elseif error == "parse-error" then
-		(session.log or log)("debug", "Client XML parse error: %s", tostring(data));
+		(session.log or log)("debug", "Client XML parse error: %s", data);
 		session:close("not-well-formed");
 	elseif error == "stream-error" then
 		local condition, text = "undefined-condition";
@@ -251,8 +261,6 @@ function listener.onconnect(conn)
 		local sock = conn:socket();
 		if sock.info then
 			session.compressed = sock:info"compression";
-		elseif sock.compression then
-			session.compressed = sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -283,7 +291,7 @@ function listener.onconnect(conn)
 			if data then
 				local ok, err = stream:feed(data);
 				if not ok then
-					log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
+					log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
 					session:close("not-well-formed");
 				end
 			end
@@ -327,6 +335,13 @@ function listener.onreadtimeout(conn)
 	end
 end
 
+function listener.ondrain(conn)
+	local session = sessions[conn];
+	if session then
+		return (hosts[session.host] or prosody).events.fire_event("c2s-ondrain", { session = session });
+	end
+end
+
 local function keepalive(event)
 	local session = event.session;
 	if not session.notopen then
diff --git a/plugins/mod_component.lua b/plugins/mod_component.lua
index b41204a2..afcfc68c 100644
--- a/plugins/mod_component.lua
+++ b/plugins/mod_component.lua
@@ -49,6 +49,7 @@ function module.add_host(module)
 	local send;
 
 	local function on_destroy(session, err) --luacheck: ignore 212/err
+		module:set_status("warn", err and ("Disconnected: "..err) or "Disconnected");
 		env.connected = false;
 		env.session = false;
 		send = nil;
@@ -102,6 +103,7 @@ function module.add_host(module)
 		module:log("info", "External component successfully authenticated");
 		session.send(st.stanza("handshake"));
 		module:fire_event("component-authenticated", { session = session });
+		module:set_status("info", "Connected");
 
 		return true;
 	end
@@ -165,11 +167,11 @@ local xmlns_xmpp_streams = "urn:ietf:params:xml:ns:xmpp-streams";
 
 function stream_callbacks.error(session, error, data)
 	if session.destroyed then return; end
-	module:log("warn", "Error processing component stream: %s", tostring(error));
+	module:log("warn", "Error processing component stream: %s", error);
 	if error == "no-stream" then
 		session:close("invalid-namespace");
 	elseif error == "parse-error" then
-		session.log("warn", "External component %s XML parse error: %s", tostring(session.host), tostring(data));
+		session.log("warn", "External component %s XML parse error: %s", session.host, data);
 		session:close("not-well-formed");
 	elseif error == "stream-error" then
 		local condition, text = "undefined-condition";
@@ -206,7 +208,7 @@ function stream_callbacks.streamclosed(session)
 	session:close();
 end
 
-local function handleerr(err) log("error", "Traceback[component]: %s", traceback(tostring(err), 2)); end
+local function handleerr(err) log("error", "Traceback[component]: %s", traceback(err, 2)); end
 function stream_callbacks.handlestanza(session, stanza)
 	-- Namespaces are icky.
 	if not stanza.attr.xmlns and stanza.name == "handshake" then
@@ -266,10 +268,10 @@ local function session_close(session, reason)
 					if reason.extra then
 						stanza:add_child(reason.extra);
 					end
-					module:log("info", "Disconnecting component, <stream:error> is: %s", tostring(stanza));
+					module:log("info", "Disconnecting component, <stream:error> is: %s", stanza);
 					session.send(stanza);
 				elseif reason.name then -- a stanza
-					module:log("info", "Disconnecting component, <stream:error> is: %s", tostring(reason));
+					module:log("info", "Disconnecting component, <stream:error> is: %s", reason);
 					session.send(reason);
 				end
 			end
@@ -310,7 +312,7 @@ function listener.onconnect(conn)
 	function session.data(_, data)
 		local ok, err = stream:feed(data);
 		if ok then return; end
-		module:log("debug", "Received invalid XML (%s) %d bytes: %s", tostring(err), #data, data:sub(1, 300):gsub("[\r\n]+", " "):gsub("[%z\1-\31]", "_"));
+		log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
 		session:close("not-well-formed");
 	end
 
@@ -325,7 +327,7 @@ end
 function listener.ondisconnect(conn, err)
 	local session = sessions[conn];
 	if session then
-		(session.log or log)("info", "component disconnected: %s (%s)", tostring(session.host), tostring(err));
+		(session.log or log)("info", "component disconnected: %s (%s)", session.host, err);
 		if session.host then
 			module:context(session.host):fire_event("component-disconnected", { session = session, reason = err });
 		end
diff --git a/plugins/mod_csi.lua b/plugins/mod_csi.lua
index 84476cac..458ff491 100644
--- a/plugins/mod_csi.lua
+++ b/plugins/mod_csi.lua
@@ -2,8 +2,9 @@ local st = require "util.stanza";
 local xmlns_csi = "urn:xmpp:csi:0";
 local csi_feature = st.stanza("csi", { xmlns = xmlns_csi });
 
+local csi_handler_available = nil;
 module:hook("stream-features", function (event)
-	if event.origin.username then
+	if event.origin.username and csi_handler_available then
 		event.features:add_child(csi_feature);
 	end
 end);
@@ -21,3 +22,14 @@ end
 module:hook("stanza/"..xmlns_csi..":active", refire_event("csi-client-active"));
 module:hook("stanza/"..xmlns_csi..":inactive", refire_event("csi-client-inactive"));
 
+function module.load()
+	if prosody.hosts[module.host].events._handlers["csi-client-active"] then
+		csi_handler_available = true;
+		module:set_status("core", "CSI handler module loaded");
+	else
+		csi_handler_available = false;
+		module:set_status("warn", "No CSI handler module loaded");
+	end
+end
+module:hook("module-loaded", module.load);
+module:hook("module-unloaded", module.load);
diff --git a/plugins/mod_csi_simple.lua b/plugins/mod_csi_simple.lua
index da2dd953..4a87f06c 100644
--- a/plugins/mod_csi_simple.lua
+++ b/plugins/mod_csi_simple.lua
@@ -9,40 +9,7 @@ module:depends"csi"
 local jid = require "util.jid";
 local st = require "util.stanza";
 local dt = require "util.datetime";
-local new_queue = require "util.queue".new;
-
-local function new_pump(output, ...)
-	-- luacheck: ignore 212/self
-	local q = new_queue(...);
-	local flush = true;
-	function q:pause()
-		flush = false;
-	end
-	function q:resume()
-		flush = true;
-		return q:flush();
-	end
-	local push = q.push;
-	function q:push(item)
-		local ok = push(self, item);
-		if not ok then
-			q:flush();
-			output(item, self);
-		elseif flush then
-			return q:flush();
-		end
-		return true;
-	end
-	function q:flush()
-		local item = self:pop();
-		while item do
-			output(item, self);
-			item = self:pop();
-		end
-		return true;
-	end
-	return q;
-end
+local filters = require "util.filters";
 
 local queue_size = module:get_option_number("csi_queue_size", 256);
 
@@ -84,37 +51,98 @@ module:hook("csi-is-stanza-important", function (event)
 	return true;
 end, -1);
 
-module:hook("csi-client-inactive", function (event)
-	local session = event.origin;
-	if session.pump then
-		session.pump:pause();
+local function with_timestamp(stanza, from)
+	if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then
+		stanza = st.clone(stanza);
+		stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = from, stamp = dt.datetime()}));
+	end
+	return stanza;
+end
+
+local function manage_buffer(stanza, session)
+	local ctr = session.csi_counter or 0;
+	if ctr >= queue_size then
+		session.log("debug", "Queue size limit hit, flushing buffer (queue size is %d)", session.csi_counter);
+		session.conn:resume_writes();
+	elseif module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then
+		session.log("debug", "Important stanza, flushing buffer (queue size is %d)", session.csi_counter);
+		session.conn:resume_writes();
 	else
-		local bare_jid = jid.join(session.username, session.host);
-		local send = session.send;
-		session._orig_send = send;
-		local pump = new_pump(session.send, queue_size);
-		pump:pause();
-		session.pump = pump;
-		function session.send(stanza)
-			if session.state == "active" or module:fire_event("csi-is-stanza-important", { stanza = stanza, session = session }) then
-				pump:flush();
-				send(stanza);
-			else
-				if st.is_stanza(stanza) and stanza.attr.xmlns == nil and stanza.name ~= "iq" then
-					stanza = st.clone(stanza);
-					stanza:add_direct_child(st.stanza("delay", {xmlns = "urn:xmpp:delay", from = bare_jid, stamp = dt.datetime()}));
-				end
-				pump:push(stanza);
-			end
-			return true;
-		end
+		stanza = with_timestamp(stanza, jid.join(session.username, session.host))
+	end
+	session.csi_counter = ctr + 1;
+	return stanza;
+end
+
+local function flush_buffer(data, session)
+	if session.csi_flushing then
+		return data;
+	end
+	session.csi_flushing = true;
+	session.log("debug", "Client sent something, flushing buffer once (queue size is %d)", session.csi_counter);
+	session.conn:resume_writes();
+	return data;
+end
+
+function enable_optimizations(session)
+	if session.conn and session.conn.pause_writes then
+		session.conn:pause_writes();
+		filters.add_filter(session, "stanzas/out", manage_buffer);
+		filters.add_filter(session, "bytes/in", flush_buffer);
+	else
+		session.log("warn", "Session connection does not support write pausing");
+	end
+end
+
+function disable_optimizations(session)
+	session.csi_flushing = nil;
+	filters.remove_filter(session, "stanzas/out", manage_buffer);
+	filters.remove_filter(session, "bytes/in", flush_buffer);
+	if session.conn and session.conn.resume_writes then
+		session.conn:resume_writes();
 	end
+end
+
+module:hook("csi-client-inactive", function (event)
+	local session = event.origin;
+	enable_optimizations(session);
 end);
 
 module:hook("csi-client-active", function (event)
 	local session = event.origin;
-	if session.pump then
-		session.pump:resume();
+	disable_optimizations(session);
+end);
+
+module:hook("pre-resource-unbind", function (event)
+	local session = event.session;
+	disable_optimizations(session);
+end, 1);
+
+module:hook("c2s-ondrain", function (event)
+	local session = event.session;
+	if session.state == "inactive" and session.conn and session.conn.pause_writes then
+		session.conn:pause_writes();
+		session.log("debug", "Buffer flushed, resuming inactive mode (queue size was %d)", session.csi_counter);
+		session.csi_counter = 0;
 	end
 end);
 
+function module.load()
+	for _, user_session in pairs(prosody.hosts[module.host].sessions) do
+		for _, session in pairs(user_session.sessions) do
+			if session.state == "inactive" then
+				enable_optimizations(session);
+			end
+		end
+	end
+end
+
+function module.unload()
+	for _, user_session in pairs(prosody.hosts[module.host].sessions) do
+		for _, session in pairs(user_session.sessions) do
+			if session.state == "inactive" then
+				disable_optimizations(session);
+			end
+		end
+	end
+end
diff --git a/plugins/mod_dialback.lua b/plugins/mod_dialback.lua
index eddc3209..f580d948 100644
--- a/plugins/mod_dialback.lua
+++ b/plugins/mod_dialback.lua
@@ -93,6 +93,11 @@ module:hook("stanza/jabber:server:dialback:result", function(event)
 		-- he wants to be identified through dialback
 		-- We need to check the key with the Authoritative server
 		local attr = stanza.attr;
+		if not attr.to or not attr.from then
+			origin.log("debug", "Missing Dialback addressing (from=%q, to=%q)", attr.from, attr.to);
+			origin:close("improper-addressing");
+			return true;
+		end
 		local to, from = nameprep(attr.to), nameprep(attr.from);
 
 		if not hosts[to] then
@@ -102,6 +107,7 @@ module:hook("stanza/jabber:server:dialback:result", function(event)
 			return true;
 		elseif not from then
 			origin:close("improper-addressing");
+			return true;
 		end
 
 		if dwd and origin.secure then
diff --git a/plugins/mod_groups.lua b/plugins/mod_groups.lua
index 646b7408..0c44f481 100644
--- a/plugins/mod_groups.lua
+++ b/plugins/mod_groups.lua
@@ -25,7 +25,7 @@ function inject_roster_contacts(event)
 	local function import_jids_to_roster(group_name)
 		for jid in pairs(groups[group_name]) do
 			-- Add them to roster
-			--module:log("debug", "processing jid %s in group %s", tostring(jid), tostring(group_name));
+			--module:log("debug", "processing jid %s in group %s", jid, group_name);
 			if jid ~= bare_jid then
 				if not roster[jid] then roster[jid] = {}; end
 				roster[jid].subscription = "both";
@@ -99,7 +99,7 @@ function module.load()
 				end
 				members[false][#members[false]+1] = curr_group; -- Is a public group
 			end
-			module:log("debug", "New group: %s", tostring(curr_group));
+			module:log("debug", "New group: %s", curr_group);
 			groups[curr_group] = groups[curr_group] or {};
 		else
 			-- Add JID
@@ -108,7 +108,7 @@ function module.load()
 			local jid;
 			jid = jid_prep(entryjid:match("%S+"));
 			if jid then
-				module:log("debug", "New member of %s: %s", tostring(curr_group), tostring(jid));
+				module:log("debug", "New member of %s: %s", curr_group, jid);
 				groups[curr_group][jid] = name or false;
 				members[jid] = members[jid] or {};
 				members[jid][#members[jid]+1] = curr_group;
diff --git a/plugins/mod_http.lua b/plugins/mod_http.lua
index a1d409bd..081aa7e9 100644
--- a/plugins/mod_http.lua
+++ b/plugins/mod_http.lua
@@ -7,13 +7,16 @@
 --
 
 module:set_global();
-module:depends("http_errors");
+pcall(function ()
+	module:depends("http_errors");
+end);
 
 local portmanager = require "core.portmanager";
 local moduleapi = require "core.moduleapi";
 local url_parse = require "socket.url".parse;
 local url_build = require "socket.url".build;
 local normalize_path = require "util.http".normalize_path;
+local set = require "util.set";
 
 local server = require "net.http.server";
 
@@ -22,6 +25,12 @@ server.set_default_host(module:get_option_string("http_default_host"));
 server.set_option("body_size_limit", module:get_option_number("http_max_content_size"));
 server.set_option("buffer_size_limit", module:get_option_number("http_max_buffer_size"));
 
+-- CORS settigs
+local opt_methods = module:get_option_set("access_control_allow_methods", { "GET", "OPTIONS" });
+local opt_headers = module:get_option_set("access_control_allow_headers", { "Content-Type" });
+local opt_credentials = module:get_option_boolean("access_control_allow_credentials", false);
+local opt_max_age = module:get_option_number("access_control_max_age", 2 * 60 * 60);
+
 local function get_http_event(host, app_path, key)
 	local method, path = key:match("^(%S+)%s+(.+)$");
 	if not method then -- No path specified, default to "" (base path)
@@ -83,6 +92,16 @@ function moduleapi.http_url(module, app_name, default_path)
 	return "http://disabled.invalid/";
 end
 
+local function apply_cors_headers(response, methods, headers, max_age, allow_credentials, origin)
+	response.headers.access_control_allow_methods = tostring(methods);
+	response.headers.access_control_allow_headers = tostring(headers);
+	response.headers.access_control_max_age = tostring(max_age)
+	response.headers.access_control_allow_origin = origin or "*";
+	if allow_credentials then
+		response.headers.access_control_allow_credentials = "true";
+	end
+end
+
 function module.add_host(module)
 	local host = module.host;
 	if host ~= "*" then
@@ -101,9 +120,27 @@ function module.add_host(module)
 		end
 		apps[app_name] = apps[app_name] or {};
 		local app_handlers = apps[app_name];
+
+		local app_methods = opt_methods;
+
+		local function cors_handler(event_data)
+			local request, response = event_data.request, event_data.response;
+			apply_cors_headers(response, app_methods, opt_headers, opt_max_age, opt_credentials, request.headers.origin);
+		end
+
+		local function options_handler(event_data)
+			cors_handler(event_data);
+			return "";
+		end
+
 		for key, handler in pairs(event.item.route or {}) do
 			local event_name = get_http_event(host, app_path, key);
 			if event_name then
+				local method = event_name:match("^%S+");
+				if not app_methods:contains(method) then
+					app_methods = app_methods + set.new{ method };
+				end
+				local options_event_name = event_name:gsub("^%S+", "OPTIONS");
 				if type(handler) ~= "function" then
 					local data = handler;
 					handler = function () return data; end
@@ -119,8 +156,14 @@ function module.add_host(module)
 					module:hook_object_event(server, event_name:sub(1, -2), redir_handler, -1);
 				end
 				if not app_handlers[event_name] then
-					app_handlers[event_name] = handler;
+					app_handlers[event_name] = {
+						main = handler;
+						cors = cors_handler;
+						options = options_handler;
+					};
 					module:hook_object_event(server, event_name, handler);
+					module:hook_object_event(server, event_name, cors_handler, 1);
+					module:hook_object_event(server, options_event_name, options_handler, -1);
 				else
 					module:log("warn", "App %s added handler twice for '%s', ignoring", app_name, event_name);
 				end
@@ -139,8 +182,11 @@ function module.add_host(module)
 	local function http_app_removed(event)
 		local app_handlers = apps[event.item.name];
 		apps[event.item.name] = nil;
-		for event_name, handler in pairs(app_handlers) do
-			module:unhook_object_event(server, event_name, handler);
+		for event_name, handlers in pairs(app_handlers) do
+			module:unhook_object_event(server, event_name, handlers.main);
+			module:unhook_object_event(server, event_name, handlers.cors);
+			local options_event_name = event_name:gsub("^%S+", "OPTIONS");
+			module:unhook_object_event(server, options_event_name, handlers.options);
 		end
 	end
 
@@ -195,9 +241,6 @@ module:provides("net", {
 	listener = server.listener;
 	default_port = 5281;
 	encryption = "ssl";
-	ssl_config = {
-		verify = "none";
-	};
 	multiplex = {
 		pattern = "^[A-Z]";
 	};
diff --git a/plugins/mod_http_errors.lua b/plugins/mod_http_errors.lua
index 13473219..2158b403 100644
--- a/plugins/mod_http_errors.lua
+++ b/plugins/mod_http_errors.lua
@@ -26,21 +26,24 @@ local html = [[
 <meta charset="utf-8">
 <title>{title}</title>
 <style>
-body{
-	margin-top:14%;
-	text-align:center;
-	background-color:#F8F8F8;
-	font-family:sans-serif;
+body {
+	margin-top : 14%;
+	text-align : center;
+	background-color : #F8F8F8;
+	font-family : sans-serif
 }
-h1{
-	font-size:xx-large;
+
+h1 {
+	font-size : xx-large
 }
-p{
-	font-size:x-large;
+
+p {
+	font-size : x-large
 }
+
 p+p {
-	font-size:large;
-	font-family:courier;
+	font-size : large;
+	font-family : courier
 }
 </style>
 </head>
@@ -72,3 +75,15 @@ module:hook_object_event(server, "http-error", function (event)
 	end
 	return get_page(event.code, (show_private and event.private_message) or event.message);
 end);
+
+module:hook_object_event(server, "http-error", function (event)
+	local request, response = event.request, event.response;
+	if request and response and request.path == "/" and response.status_code == 404 then
+		response.headers.content_type = "text/html; charset=utf-8";
+		return render(html, {
+				title = "Prosody is running!";
+				message = "Welcome to the XMPP world!";
+			});
+	end
+end, 1);
+
diff --git a/plugins/mod_http_files.lua b/plugins/mod_http_files.lua
index 1dae0d6d..c357ddfc 100644
--- a/plugins/mod_http_files.lua
+++ b/plugins/mod_http_files.lua
@@ -7,14 +7,9 @@
 --
 
 module:depends("http");
-local server = require"net.http.server";
-local lfs = require "lfs";
 
-local os_date = os.date;
 local open = io.open;
-local stat = lfs.attributes;
-local build_path = require"socket.url".build_path;
-local path_sep = package.config:sub(1,1);
+local fileserver = require"net.http.files";
 
 local base_path = module:get_option_path("http_files_dir", module:get_option_path("http_path"));
 local cache_size = module:get_option_number("http_files_cache_size", 128);
@@ -51,148 +46,56 @@ if not mime_map then
 	end
 end
 
-local forbidden_chars_pattern = "[/%z]";
-if prosody.platform == "windows" then
-	forbidden_chars_pattern = "[/%z\001-\031\127\"*:<>?|]"
+local function get_calling_module()
+	local info = debug.getinfo(3, "S");
+	if not info then return "An unknown module"; end
+	return info.source:match"mod_[^/\\.]+" or info.short_src;
 end
 
-local urldecode = require "util.http".urldecode;
-function sanitize_path(path)
-	if not path then return end
-	local out = {};
-
-	local c = 0;
-	for component in path:gmatch("([^/]+)") do
-		component = urldecode(component);
-		if component:find(forbidden_chars_pattern) then
-			return nil;
-		elseif component == ".." then
-			if c <= 0 then
-				return nil;
-			end
-			out[c] = nil;
-			c = c - 1;
-		elseif component ~= "." then
-			c = c + 1;
-			out[c] = component;
-		end
-	end
-	if path:sub(-1,-1) == "/" then
-		out[c+1] = "";
-	end
-	return "/"..table.concat(out, "/");
-end
-
-local cache = require "util.cache".new(cache_size);
-
+-- COMPAT -- TODO deprecate
 function serve(opts)
 	if type(opts) ~= "table" then -- assume path string
 		opts = { path = opts };
 	end
-	-- luacheck: ignore 431
-	local base_path = opts.path;
-	local dir_indices = opts.index_files or dir_indices;
-	local directory_index = opts.directory_index;
-	local function serve_file(event, path)
-		local request, response = event.request, event.response;
-		local sanitized_path = sanitize_path(path);
-		if path and not sanitized_path then
-			return 400;
-		end
-		path = sanitized_path;
-		local orig_path = sanitize_path(request.path);
-		local full_path = base_path .. (path or ""):gsub("/", path_sep);
-		local attr = stat(full_path:match("^.*[^\\/]")); -- Strip trailing path separator because Windows
-		if not attr then
-			return 404;
-		end
-
-		local request_headers, response_headers = request.headers, response.headers;
-
-		local last_modified = os_date('!%a, %d %b %Y %H:%M:%S GMT', attr.modification);
-		response_headers.last_modified = last_modified;
-
-		local etag = ('"%02x-%x-%x-%x"'):format(attr.dev or 0, attr.ino or 0, attr.size or 0, attr.modification or 0);
-		response_headers.etag = etag;
-
-		local if_none_match = request_headers.if_none_match
-		local if_modified_since = request_headers.if_modified_since;
-		if etag == if_none_match
-		or (not if_none_match and last_modified == if_modified_since) then
-			return 304;
-		end
-
-		local data = cache:get(orig_path);
-		if data and data.etag == etag then
-			response_headers.content_type = data.content_type;
-			data = data.data;
-		elseif attr.mode == "directory" and path then
-			if full_path:sub(-1) ~= "/" then
-				local dir_path = { is_absolute = true, is_directory = true };
-				for dir in orig_path:gmatch("[^/]+") do dir_path[#dir_path+1]=dir; end
-				response_headers.location = build_path(dir_path);
-				return 301;
-			end
-			for i=1,#dir_indices do
-				if stat(full_path..dir_indices[i], "mode") == "file" then
-					return serve_file(event, path..dir_indices[i]);
-				end
-			end
-
-			if directory_index then
-				data = server._events.fire_event("directory-index", { path = request.path, full_path = full_path });
-			end
-			if not data then
-				return 403;
-			end
-			cache:set(orig_path, { data = data, content_type = mime_map.html; etag = etag; });
-			response_headers.content_type = mime_map.html;
-
-		else
-			local f, err = open(full_path, "rb");
-			if not f then
-				module:log("debug", "Could not open %s. Error was %s", full_path, err);
-				return 403;
-			end
-			local ext = full_path:match("%.([^./]+)$");
-			local content_type = ext and mime_map[ext];
-			response_headers.content_type = content_type;
-			if attr.size > cache_max_file_size then
-				response_headers.content_length = attr.size;
-				module:log("debug", "%d > cache_max_file_size", attr.size);
-				return response:send_file(f);
-			else
-				data = f:read("*a");
-				f:close();
-			end
-			cache:set(orig_path, { data = data; content_type = content_type; etag = etag });
-		end
-
-		return response:send(data);
+	if opts.directory_index == nil then
+		opts.directory_index = directory_index;
 	end
-
-	return serve_file;
+	if opts.mime_map == nil then
+		opts.mime_map = mime_map;
+	end
+	if opts.cache_size == nil then
+		opts.cache_size = cache_size;
+	end
+	if opts.cache_max_file_size == nil then
+		opts.cache_max_file_size = cache_max_file_size;
+	end
+	if opts.index_files == nil then
+		opts.index_files = dir_indices;
+	end
+	-- TODO Crank up to warning
+	module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module());
+	return fileserver.serve(opts);
 end
 
 function wrap_route(routes)
+	module:log("debug", "%s should be updated to use 'net.http.files' insead of mod_http_files", get_calling_module());
 	for route,handler in pairs(routes) do
 		if type(handler) ~= "function" then
-			routes[route] = serve(handler);
+			routes[route] = fileserver.serve(handler);
 		end
 	end
 	return routes;
 end
 
-if base_path then
-	module:provides("http", {
-		route = {
-			["GET /*"] = serve {
-				path = base_path;
-				directory_index = directory_index;
-			}
-		};
-	});
-else
-	module:log("debug", "http_files_dir not set, assuming use by some other module");
-end
-
+module:provides("http", {
+	route = {
+		["GET /*"] = fileserver.serve({
+			path = base_path;
+			directory_index = directory_index;
+			mime_map = mime_map;
+			cache_size = cache_size;
+			cache_max_file_size = cache_max_file_size;
+			index_files = dir_indices;
+		});
+	};
+});
diff --git a/plugins/mod_limits.lua b/plugins/mod_limits.lua
index 914d5c44..a1a3b2c0 100644
--- a/plugins/mod_limits.lua
+++ b/plugins/mod_limits.lua
@@ -32,7 +32,7 @@ local function parse_burst(burst, sess_type)
 	end
 	local n_burst = tonumber(burst);
 	if not n_burst then
-		module:log("error", "Unable to parse burst for %s: %q, using default burst interval (%ds)", sess_type, tostring(burst), default_burst);
+		module:log("error", "Unable to parse burst for %s: %q, using default burst interval (%ds)", sess_type, burst, default_burst);
 	end
 	return n_burst or default_burst;
 end
@@ -51,18 +51,18 @@ end
 local default_filter_set = {};
 
 function default_filter_set.bytes_in(bytes, session)
-  local sess_throttle = session.throttle;
-  if sess_throttle then
-    local ok, balance, outstanding = sess_throttle:poll(#bytes, true);
+	local sess_throttle = session.throttle;
+	if sess_throttle then
+		local ok, balance, outstanding = sess_throttle:poll(#bytes, true);
 		if not ok then
-      session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding);
+			session.log("debug", "Session over rate limit (%d) with %d (by %d), pausing", sess_throttle.max, #bytes, outstanding);
 			outstanding = ceil(outstanding);
 			session.conn:pause(); -- Read no more data from the connection until there is no outstanding data
 			local outstanding_data = bytes:sub(-outstanding);
 			bytes = bytes:sub(1, #bytes-outstanding);
 			timer.add_task(limits_resolution, function ()
 				if not session.conn then return; end
-        if sess_throttle:peek(#outstanding_data) then
+				if sess_throttle:peek(#outstanding_data) then
 					session.log("debug", "Resuming paused session");
 					session.conn:resume();
 				end
@@ -84,8 +84,13 @@ local function filter_hook(session)
 	local session_type = session.type:match("^[^_]+");
 	local filter_set, opts = type_filters[session_type], limits[session_type];
 	if opts then
-		session.throttle = throttle.create(opts.bytes_per_second * opts.burst_seconds, opts.burst_seconds);
-		filters.add_filter(session, "bytes/in", filter_set.bytes_in, 1000);
+		if session.conn and session.conn.setlimit then
+			session.conn:setlimit(opts.bytes_per_second);
+			-- Currently no burst support
+		else
+			session.throttle = throttle.create(opts.bytes_per_second * opts.burst_seconds, opts.burst_seconds);
+			filters.add_filter(session, "bytes/in", filter_set.bytes_in, 1000);
+		end
 	end
 end
 
@@ -96,3 +101,25 @@ end
 function module.unload()
 	filters.remove_filter_hook(filter_hook);
 end
+
+function module.add_host(module)
+	local unlimited_jids = module:get_option_inherited_set("unlimited_jids", {});
+
+	if not unlimited_jids:empty() then
+		module:hook("authentication-success", function (event)
+			local session = event.session;
+			local session_type = session.type:match("^[^_]+");
+			local jid = session.username .. "@" .. session.host;
+			if unlimited_jids:contains(jid) then
+				if session.conn and session.conn.setlimit then
+					session.conn:setlimit(0);
+					-- Currently no burst support
+				else
+					local filter_set = type_filters[session_type];
+					filters.remove_filter(session, "bytes/in", filter_set.bytes_in);
+					session.throttle = nil;
+				end
+			end
+		end);
+	end
+end
diff --git a/plugins/mod_mam/mod_mam.lua b/plugins/mod_mam/mod_mam.lua
index b500770f..e9528d52 100644
--- a/plugins/mod_mam/mod_mam.lua
+++ b/plugins/mod_mam/mod_mam.lua
@@ -40,6 +40,9 @@ local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://ja
 local archive_store = module:get_option_string("archive_store", "archive");
 local archive = module:open_store(archive_store, "archive");
 
+local cleanup_after = module:get_option_string("archive_expires_after", "1w");
+local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
 if not archive.find then
 	error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n"
 		.."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information");
@@ -117,10 +120,12 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query, id %s with %s from %s until %s",
-		tostring(qid), qwith or "anyone",
-		qstart and timestamp(qstart) or "the dawn of time",
-		qend and timestamp(qend) or "now");
+	module:log("debug", "Archive query by %s id=%s with=%s when=%s...%s",
+		origin.username,
+		qid or stanza.attr.id,
+		qwith or "*",
+		qstart and timestamp(qstart) or "",
+		qend and timestamp(qend) or "");
 
 	-- RSM stuff
 	local qset = rsm.get(query);
@@ -128,6 +133,9 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	local reverse = qset and qset.before or false;
 	local before, after = qset and qset.before, qset and qset.after;
 	if type(before) ~= "string" then before = nil; end
+	if qset then
+		module:log("debug", "Archive query id=%s rsm=%q", qid or stanza.attr.id, qset);
+	end
 
 	-- Load all the data!
 	local data, err = archive:find(origin.username, {
@@ -140,7 +148,12 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
-		origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
+		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);
+		if err == "item-not-found" then
+			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
+		else
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		end
 		return true;
 	end
 	local total = tonumber(err);
@@ -189,13 +202,13 @@ module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
 		first, last = last, first;
 	end
 
-	-- That's all folks!
-	module:log("debug", "Archive query %s completed", tostring(qid));
-
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 			:add_child(rsm.generate {
 				first = first, last = last, count = total }));
+
+	-- That's all folks!
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);
 	return true;
 end);
 
@@ -213,13 +226,13 @@ local function shall_store(user, who)
 	end
 	local prefs = get_prefs(user);
 	local rule = prefs[who];
-	module:log("debug", "%s's rule for %s is %s", user, who, tostring(rule));
+	module:log("debug", "%s's rule for %s is %s", user, who, rule);
 	if rule ~= nil then
 		return rule;
 	end
 	-- Below could be done by a metatable
 	local default = prefs[false];
-	module:log("debug", "%s's default rule is %s", user, tostring(default));
+	module:log("debug", "%s's default rule is %s", user, default);
 	if default == "roster" then
 		return has_in_roster(user, who);
 	end
@@ -297,7 +310,28 @@ local function message_handler(event, c2s)
 		log("debug", "Archiving stanza: %s", stanza:top_tag());
 
 		-- And stash it
-		local ok = archive:append(store_user, nil, clone_for_storage, time_now(), with);
+		local time = time_now();
+		local ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+		if not ok and err == "quota-limit" then
+			if type(cleanup_after) == "number" then
+				module:log("debug", "User '%s' over quota, cleaning archive", store_user);
+				local cleaned = archive:delete(store_user, {
+					["end"] = (os.time() - cleanup_after);
+				});
+				if cleaned then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+			if not ok and (archive.caps and archive.caps.truncate) then
+				module:log("debug", "User '%s' over quota, truncating archive", store_user);
+				local truncated = archive:delete(store_user, {
+					truncate = archive_item_limit - 1;
+				});
+				if truncated then
+					ok, err = archive:append(store_user, nil, clone_for_storage, time, with);
+				end
+			end
+		end
 		if ok then
 			local clone_for_other_handlers = st.clone(stanza);
 			local id = ok;
@@ -323,8 +357,6 @@ end
 module:hook("pre-message/bare", strip_stanza_id_after_other_events, -1);
 module:hook("pre-message/full", strip_stanza_id_after_other_events, -1);
 
-local cleanup_after = module:get_option_string("archive_expires_after", "1w");
-local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
 if cleanup_after ~= "never" then
 	local cleanup_storage = module:open_store("archive_cleanup");
 	local cleanup_map = module:open_store("archive_cleanup", "map");
@@ -359,8 +391,10 @@ if cleanup_after ~= "never" then
 			last_date:set(username, date);
 		end
 	end
+	local cleanup_time = module:measure("cleanup", "times");
 
 	cleanup_runner = require "util.async".runner(function ()
+		local cleanup_done = cleanup_time();
 		local users = {};
 		local cut_off = datestamp(os.time() - cleanup_after);
 		for date in cleanup_storage:users() do
@@ -388,6 +422,7 @@ if cleanup_after ~= "never" then
 			end
 		end
 		module:log("info", "Deleted %d expired messages for %d users", sum, num_users);
+		cleanup_done();
 	end);
 
 	cleanup_task = module:add_timer(1, function ()
diff --git a/plugins/mod_mimicking.lua b/plugins/mod_mimicking.lua
new file mode 100644
index 00000000..b586a70c
--- /dev/null
+++ b/plugins/mod_mimicking.lua
@@ -0,0 +1,85 @@
+-- Prosody IM
+-- Copyright (C) 2012 Florian Zeitz
+-- Copyright (C) 2019 Kim Alvefur
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local encodings = require "util.encodings";
+assert(encodings.confusable, "This module requires that Prosody be built with ICU");
+local skeleton = encodings.confusable.skeleton;
+
+local usage = require "util.prosodyctl".show_usage;
+local usermanager = require "core.usermanager";
+local storagemanager = require "core.storagemanager";
+
+local skeletons
+function module.load()
+	if module.host ~= "*" then
+		skeletons = module:open_store("skeletons");
+	end
+end
+
+module:hook("user-registered", function(user)
+	local skel = skeleton(user.username);
+	local ok, err = skeletons:set(skel, { username = user.username });
+	if not ok then
+		module:log("error", "Unable to store mimicry data (%q => %q): %s", user.username, skel, err);
+	end
+end);
+
+module:hook("user-deleted", function(user)
+	local skel = skeleton(user.username);
+	local ok, err = skeletons:set(skel, nil);
+	if not ok and err then
+		module:log("error", "Unable to clear mimicry data (%q): %s", skel, err);
+	end
+end);
+
+module:hook("user-registering", function(user)
+	local existing, err = skeletons:get(skeleton(user.username));
+	if existing then
+		module:log("debug", "Attempt to register username '%s' which could be confused with '%s'", user.username, existing.username);
+		user.allowed = false;
+	elseif err then
+		module:log("error", "Unable to check if new username '%s' can be confused with any existing user: %s", err);
+	end
+end);
+
+function module.command(arg)
+	if (arg[1] ~= "bootstrap" or not arg[2]) then
+		usage("mod_mimicking bootstrap <host>", "Initialize username mimicry database");
+		return;
+	end
+
+	local host = arg[2];
+
+	local host_session = prosody.hosts[host];
+	if not host_session then
+		return "No such host";
+	end
+
+	storagemanager.initialize_host(host);
+	usermanager.initialize_host(host);
+
+	skeletons = storagemanager.open(host, "skeletons");
+
+	local count = 0;
+	for user in usermanager.users(host) do
+		local skel = skeleton(user);
+		local existing, err = skeletons:get(skel);
+		if existing and existing.username ~= user then
+			module:log("warn", "Existing usernames '%s' and '%s' are confusable", existing.username, user);
+		elseif err then
+			module:log("error", "Error checking for existing mimicry data (%q = %q): %s", user, skel, err);
+		end
+		local ok, err = skeletons:set(skel, { username = user });
+		if ok then
+			count = count + 1;
+		elseif err then
+			module:log("error", "Unable to store mimicry data (%q => %q): %s", user, skel, err);
+		end
+	end
+	module:log("info", "%d usernames indexed", count);
+end
diff --git a/plugins/mod_muc_mam.lua b/plugins/mod_muc_mam.lua
index a2e3f81b..1df93a18 100644
--- a/plugins/mod_muc_mam.lua
+++ b/plugins/mod_muc_mam.lua
@@ -4,7 +4,7 @@
 -- This file is MIT/X11 licensed.
 
 if module:get_host_type() ~= "component" then
-	module:log("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name);
+	module:log_status("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name);
 	return;
 end
 
@@ -21,6 +21,7 @@ local jid_bare = require "util.jid".bare;
 local jid_split = require "util.jid".split;
 local jid_prep = require "util.jid".prep;
 local dataform = require "util.dataforms".new;
+local get_form_type = require "util.dataforms".get_type;
 
 local mod_muc = module:depends"muc";
 local get_room_from_jid = mod_muc.get_room_from_jid;
@@ -32,6 +33,9 @@ local m_min = math.min;
 local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date");
 local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50);
 
+local cleanup_after = module:get_option_string("muc_log_expires_after", "1w");
+local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60);
+
 local default_history_length = 20;
 local max_history_length = module:get_option_number("max_history_messages", math.huge);
 
@@ -49,6 +53,8 @@ local log_by_default = module:get_option_boolean("muc_log_by_default", true);
 local archive_store = "muc_log";
 local archive = module:open_store(archive_store, "archive");
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", archive.caps and archive.caps.quota or 1000);
+
 if archive.name == "null" or not archive.find then
 	if not archive.find then
 		module:log("error", "Attempt to open archive storage returned a driver without archive API support");
@@ -63,12 +69,15 @@ end
 
 local function archiving_enabled(room)
 	if log_all_rooms then
+		module:log("debug", "Archiving all rooms");
 		return true;
 	end
 	local enabled = room._data.archiving;
 	if enabled == nil then
+		module:log("debug", "Default is %s (for %s)", log_by_default, room.jid);
 		return log_by_default;
 	end
+	module:log("debug", "Logging in room %s is %s", room.jid, enabled);
 	return enabled;
 end
 
@@ -135,7 +144,11 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 	local qstart, qend;
 	local form = query:get_child("x", "jabber:x:data");
 	if form then
-		local err;
+		local form_type, err = get_form_type(form);
+		if form_type ~= xmlns_mam then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Unexpected FORM_TYPE, expected '"..xmlns_mam.."'"));
+			return true;
+		end
 		form, err = query_form:data(form);
 		if err then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
@@ -153,10 +166,11 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 		qstart, qend = vstart, vend;
 	end
 
-	module:log("debug", "Archive query id %s from %s until %s)",
-		tostring(qid),
-		qstart and timestamp(qstart) or "the dawn of time",
-		qend and timestamp(qend) or "now");
+	module:log("debug", "Archive query by %s id=%s when=%s...%s",
+		origin.username,
+		qid or stanza.attr.id,
+		qstart and timestamp(qstart) or "",
+		qend and timestamp(qend) or "");
 
 	-- RSM stuff
 	local qset = rsm.get(query);
@@ -165,6 +179,9 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 
 	local before, after = qset and qset.before, qset and qset.after;
 	if type(before) ~= "string" then before = nil; end
+	if qset then
+		module:log("debug", "Archive query id=%s rsm=%q", qid or stanza.attr.id, qset);
+	end
 
 	-- Load all the data!
 	local data, err = archive:find(room_node, {
@@ -176,7 +193,12 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 	});
 
 	if not data then
-		origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		module:log("debug", "Archive query id=%s failed: %s", qid or stanza.attr.id, err);
+		if err == "item-not-found" then
+			origin.send(st.error_reply(stanza, "modify", "item-not-found"));
+		else
+			origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
+		end
 		return true;
 	end
 	local total = tonumber(err);
@@ -233,13 +255,14 @@ module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 		first, last = last, first;
 	end
 
-	-- That's all folks!
-	module:log("debug", "Archive query %s completed", tostring(qid));
 
 	origin.send(st.reply(stanza)
 		:tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 			:add_child(rsm.generate {
 				first = first, last = last, count = total }));
+
+	-- That's all folks!
+	module:log("debug", "Archive query id=%s completed, %d items returned", qid or stanza.attr.id, complete and count or count - 1);
 	return true;
 end);
 
@@ -274,7 +297,7 @@ module:hook("muc-get-history", function (event)
 	local data, err = archive:find(jid_split(room_jid), query);
 
 	if not data then
-		module:log("error", "Could not fetch history: %s", tostring(err));
+		module:log("error", "Could not fetch history: %s", err);
 		return
 	end
 
@@ -300,7 +323,7 @@ module:hook("muc-get-history", function (event)
 			maxchars = maxchars - chars;
 		end
 		history[i], i = item, i+1;
-		-- module:log("debug", tostring(item));
+		-- module:log("debug", item);
 	end
 	function event.next_stanza()
 		i = i - 1;
@@ -352,7 +375,29 @@ local function save_to_history(self, stanza)
 	end
 
 	-- And stash it
-	local id = archive:append(room_node, nil, stored_stanza, time_now(), with);
+	local time = time_now();
+	local id, err = archive:append(room_node, nil, stored_stanza, time, with);
+
+	if not id and err == "quota-limit" then
+		if type(cleanup_after) == "number" then
+			module:log("debug", "Room '%s' over quota, cleaning archive", room_node);
+			local cleaned = archive:delete(room_node, {
+				["end"] = (os.time() - cleanup_after);
+			});
+			if cleaned then
+				id, err = archive:append(room_node, nil, stored_stanza, time, with);
+			end
+		end
+		if not id and (archive.caps and archive.caps.truncate) then
+			module:log("debug", "User '%s' over quota, truncating archive", room_node);
+			local truncated = archive:delete(room_node, {
+				truncate = archive_item_limit - 1;
+			});
+			if truncated then
+				id, err = archive:append(room_node, nil, stored_stanza, time, with);
+			end
+		end
+	end
 
 	if id then
 		schedule_cleanup(room_node);
@@ -389,14 +434,13 @@ end
 module:add_feature(xmlns_mam);
 
 module:hook("muc-disco#info", function(event)
-	event.reply:tag("feature", {var=xmlns_mam}):up();
+	if archiving_enabled(event.room) then
+		event.reply:tag("feature", {var=xmlns_mam}):up();
+	end
 end);
 
 -- Cleanup
 
-local cleanup_after = module:get_option_string("muc_log_expires_after", "1w");
-local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60);
-
 if cleanup_after ~= "never" then
 	local cleanup_storage = module:open_store("muc_log_cleanup");
 	local cleanup_map = module:open_store("muc_log_cleanup", "map");
diff --git a/plugins/mod_offline.lua b/plugins/mod_offline.lua
index 487098d1..dffe8357 100644
--- a/plugins/mod_offline.lua
+++ b/plugins/mod_offline.lua
@@ -24,11 +24,16 @@ module:hook("message/offline/handle", function(event)
 		node = origin.username;
 	end
 
-	return offline_messages:append(node, nil, stanza, os.time(), "");
+	local ok = offline_messages:append(node, nil, stanza, os.time(), "");
+	if ok then
+		module:log("debug", "Saved to offline storage: %s", stanza:top_tag());
+	end
+	return ok;
 end, -1);
 
 module:hook("message/offline/broadcast", function(event)
 	local origin = event.origin;
+	origin.log("debug", "Broadcasting offline messages");
 
 	local node, host = origin.username, origin.host;
 
@@ -38,6 +43,9 @@ module:hook("message/offline/broadcast", function(event)
 		stanza:tag("delay", {xmlns = "urn:xmpp:delay", from = host, stamp = datetime.datetime(when)}):up(); -- XEP-0203
 		origin.send(stanza);
 	end
-	offline_messages:delete(node);
+	local ok = offline_messages:delete(node);
+	if type(ok) == "number" and ok > 0 then
+		origin.log("debug", "%d offline messages consumed");
+	end
 	return true;
 end, -1);
diff --git a/plugins/mod_pep.lua b/plugins/mod_pep.lua
index 7a4aac2b..c0a85a9d 100644
--- a/plugins/mod_pep.lua
+++ b/plugins/mod_pep.lua
@@ -8,6 +8,7 @@ local calculate_hash = require "util.caps".calculate_hash;
 local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed;
 local cache = require "util.cache";
 local set = require "util.set";
+local new_id = require "util.id".medium;
 local storagemanager = require "core.storagemanager";
 
 local xmlns_pubsub = "http://jabber.org/protocol/pubsub";
@@ -138,9 +139,6 @@ local function get_broadcaster(username)
 		if kind == "retract" then
 			kind = "items"; -- XEP-0060 signals retraction in an <items> container
 		end
-		local message = st.message({ from = user_bare, type = "headline" })
-			:tag("event", { xmlns = xmlns_pubsub_event })
-				:tag(kind, { node = node });
 		if item then
 			item = st.clone(item);
 			item.attr.xmlns = nil; -- Clear the pubsub namespace
@@ -149,10 +147,19 @@ local function get_broadcaster(username)
 					item:maptags(function () return nil; end);
 				end
 			end
+		end
+
+		local id = new_id();
+		local message = st.message({ from = user_bare, type = "headline", id = id })
+			:tag("event", { xmlns = xmlns_pubsub_event })
+				:tag(kind, { node = node });
+
+		if item then
 			message:add_child(item);
 		end
+
 		for jid in pairs(jids) do
-			module:log("debug", "Sending notification to %s from %s: %s", jid, user_bare, tostring(item));
+			module:log("debug", "Sending notification to %s from %s for node %s", jid, user_bare, node);
 			message.attr.to = jid;
 			module:send(message);
 		end
@@ -176,12 +183,12 @@ local function on_node_creation(event)
 end
 
 function get_pep_service(username)
-	module:log("debug", "get_pep_service(%q)", username);
 	local user_bare = jid_join(username, host);
 	local service = services[username];
 	if service then
 		return service;
 	end
+	module:log("debug", "Creating pubsub service for user %q", username);
 	service = pubsub.new({
 		pep_username = username;
 		node_defaults = {
@@ -252,8 +259,6 @@ end
 module:hook("iq/bare/"..xmlns_pubsub..":pubsub", handle_pubsub_iq);
 module:hook("iq/bare/"..xmlns_pubsub_owner..":pubsub", handle_pubsub_iq);
 
-module:add_identity("pubsub", "pep", module:get_option_string("name", "Prosody"));
-module:add_feature("http://jabber.org/protocol/pubsub#publish");
 
 local function get_caps_hash_from_presence(stanza, current)
 	local t = stanza.attr.type;
diff --git a/plugins/mod_pep_simple.lua b/plugins/mod_pep_simple.lua
index f0b5d7ef..11268ab7 100644
--- a/plugins/mod_pep_simple.lua
+++ b/plugins/mod_pep_simple.lua
@@ -14,6 +14,7 @@ local is_contact_subscribed = require "core.rostermanager".is_contact_subscribed
 local pairs = pairs;
 local next = next;
 local type = type;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local calculate_hash = require "util.caps".calculate_hash;
 local core_post_stanza = prosody.core_post_stanza;
 local bare_sessions = prosody.bare_sessions;
@@ -229,13 +230,13 @@ module:hook("iq/bare/http://jabber.org/protocol/pubsub:pubsub", function(event)
 				return true;
 			else --invalid request
 				session.send(st.error_reply(stanza, 'modify', 'bad-request'));
-				module:log("debug", "Invalid request: %s", tostring(payload));
+				module:log("debug", "Invalid request: %s", payload);
 				return true;
 			end
 		else --no presence subscription
 			session.send(st.error_reply(stanza, 'auth', 'not-authorized')
 				:tag('presence-subscription-required', {xmlns='http://jabber.org/protocol/pubsub#errors'}));
-			module:log("debug", "Unauthorized request: %s", tostring(payload));
+			module:log("debug", "Unauthorized request: %s", payload);
 			return true;
 		end
 	end
diff --git a/plugins/mod_ping.lua b/plugins/mod_ping.lua
index 5fff58d1..df24c495 100644
--- a/plugins/mod_ping.lua
+++ b/plugins/mod_ping.lua
@@ -16,18 +16,3 @@ end
 
 module:hook("iq-get/bare/urn:xmpp:ping:ping", ping_handler);
 module:hook("iq-get/host/urn:xmpp:ping:ping", ping_handler);
-
--- Ad-hoc command
-
-local datetime = require "util.datetime".datetime;
-
-function ping_command_handler (self, data, state) -- luacheck: ignore 212
-	local now = datetime();
-	return { info = "Pong\n"..now, status = "completed" };
-end
-
-module:depends "adhoc";
-local adhoc_new = module:require "adhoc".new;
-local descriptor = adhoc_new("Ping", "ping", ping_command_handler);
-module:provides("adhoc", descriptor);
-
diff --git a/plugins/mod_posix.lua b/plugins/mod_posix.lua
index 23df4d23..a2a60dd0 100644
--- a/plugins/mod_posix.lua
+++ b/plugins/mod_posix.lua
@@ -20,7 +20,6 @@ if not have_signal then
 	module:log("warn", "Couldn't load signal library, won't respond to SIGTERM");
 end
 
-local format = require "util.format".format;
 local lfs = require "lfs";
 local stat = lfs.attributes;
 
@@ -113,19 +112,6 @@ local function write_pidfile()
 	end
 end
 
-local syslog_opened;
-function syslog_sink_maker(config) -- luacheck: ignore 212/config
-	if not syslog_opened then
-		pposix.syslog_open("prosody", module:get_option_string("syslog_facility"));
-		syslog_opened = true;
-	end
-	local syslog = pposix.syslog_log;
-	return function (name, level, message, ...)
-		syslog(level, name, format(message, ...));
-	end;
-end
-require "core.loggingmanager".register_sink_type("syslog", syslog_sink_maker);
-
 local daemonize = module:get_option("daemonize", prosody.installed);
 
 local function remove_log_sinks()
diff --git a/plugins/mod_presence.lua b/plugins/mod_presence.lua
index 268a2f0c..f7f458ca 100644
--- a/plugins/mod_presence.lua
+++ b/plugins/mod_presence.lua
@@ -81,8 +81,14 @@ function handle_normal_presence(origin, stanza)
 				res.presence.attr.to = nil;
 			end
 		end
-		for jid in pairs(roster[false].pending) do -- resend incoming subscription requests
-			origin.send(st.presence({type="subscribe", from=jid})); -- TODO add to attribute? Use original?
+		for jid, pending_request in pairs(roster[false].pending) do -- resend incoming subscription requests
+			if type(pending_request) == "table" then
+				local subscribe = st.deserialize(pending_request);
+				subscribe.attr.type, subscribe.attr.from = "subscribe", jid;
+				origin.send(subscribe);
+			else
+				origin.send(st.presence({type="subscribe", from=jid}));
+			end
 		end
 		local request = st.presence({type="subscribe", from=origin.username.."@"..origin.host});
 		for jid, item in pairs(roster) do -- resend outgoing subscription requests
@@ -226,7 +232,7 @@ function handle_inbound_presence_subscriptions_and_probes(origin, stanza, from_b
 		else
 			core_post_stanza(hosts[host], st.presence({from=to_bare, to=from_bare, type="unavailable"}), true); -- acknowledging receipt
 			if not rostermanager.is_contact_pending_in(node, host, from_bare) then
-				if rostermanager.set_contact_pending_in(node, host, from_bare) then
+				if rostermanager.set_contact_pending_in(node, host, from_bare, stanza) then
 					sessionmanager.send_to_available_resources(node, host, stanza);
 				end -- TODO else return error, unable to save
 			end
diff --git a/plugins/mod_proxy65.lua b/plugins/mod_proxy65.lua
index 00833772..29c821e2 100644
--- a/plugins/mod_proxy65.lua
+++ b/plugins/mod_proxy65.lua
@@ -117,7 +117,7 @@ function module.add_host(module)
 				if jid_compare(jid, acl) then allow = true; break; end
 			end
 			if allow then break; end
-			module:log("warn", "Denying use of proxy for %s", tostring(stanza.attr.from));
+			module:log("warn", "Denying use of proxy for %s", stanza.attr.from);
 			origin.send(st.error_reply(stanza, "auth", "forbidden"));
 			return true;
 		end
diff --git a/plugins/mod_pubsub/mod_pubsub.lua b/plugins/mod_pubsub/mod_pubsub.lua
index 855c5fd2..faf08cb2 100644
--- a/plugins/mod_pubsub/mod_pubsub.lua
+++ b/plugins/mod_pubsub/mod_pubsub.lua
@@ -75,14 +75,13 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj)
 	local msg_type = node_obj and node_obj.config.message_type or "headline";
 	local message = st.message({ from = module.host, type = msg_type, id = id })
 		:tag("event", { xmlns = xmlns_pubsub_event })
-			:tag(kind, { node = node })
+			:tag(kind, { node = node });
 
 	if item then
 		message:add_child(item);
 	end
 
 	local summary;
-	-- Compose a sensible textual representation of at least Atom payloads
 	if item and item.tags[1] then
 		local payload = item.tags[1];
 		summary = module:fire_event("pubsub-summary/"..payload.attr.xmlns, {
@@ -101,11 +100,12 @@ function simple_broadcast(kind, node, jids, item, actor, node_obj)
 end
 
 local max_max_items = module:get_option_number("pubsub_max_items", 256);
-function check_node_config(node, actor, new_config) -- luacheck: ignore 212/actor 212/node
+function check_node_config(node, actor, new_config) -- luacheck: ignore 212/node 212/actor
 	if (new_config["max_items"] or 1) > max_max_items then
 		return false;
 	end
-	if new_config["access_model"] ~= "whitelist" and new_config["access_model"] ~= "open" then
+	if new_config["access_model"] ~= "whitelist"
+	and new_config["access_model"] ~= "open" then
 		return false;
 	end
 	return true;
@@ -115,6 +115,7 @@ function is_item_stanza(item)
 	return st.is_stanza(item) and item.attr.xmlns == xmlns_pubsub and item.name == "item";
 end
 
+-- Compose a textual representation of Atom payloads
 module:hook("pubsub-summary/http://www.w3.org/2005/Atom", function (event)
 	local payload = event.payload;
 	local title = payload:get_child_text("title");
diff --git a/plugins/mod_pubsub/pubsub.lib.lua b/plugins/mod_pubsub/pubsub.lib.lua
index 50ef7ddf..23695211 100644
--- a/plugins/mod_pubsub/pubsub.lib.lua
+++ b/plugins/mod_pubsub/pubsub.lib.lua
@@ -7,6 +7,7 @@ local st = require "util.stanza";
 local it = require "util.iterators";
 local uuid_generate = require "util.uuid".generate;
 local dataform = require"util.dataforms".new;
+local errors = require "util.error";
 
 local xmlns_pubsub = "http://jabber.org/protocol/pubsub";
 local xmlns_pubsub_errors = "http://jabber.org/protocol/pubsub#errors";
@@ -34,6 +35,9 @@ local pubsub_errors = {
 };
 local function pubsub_error_reply(stanza, error)
 	local e = pubsub_errors[error];
+	if not e and errors.is_err(error) then
+		e = { error.type, error.condition, error.text, error.pubsub_condition };
+	end
 	local reply = st.error_reply(stanza, t_unpack(e, 1, 3));
 	if e[4] then
 		reply:tag(e[4], { xmlns = xmlns_pubsub_errors }):up();
@@ -185,6 +189,14 @@ local node_metadata_form = dataform {
 		type = "text-single";
 		name = "pubsub#type";
 	};
+	{
+		type = "text-single";
+		name = "pubsub#access_model";
+	};
+	{
+		type = "text-single";
+		name = "pubsub#publish_model";
+	};
 };
 
 local service_method_feature_map = {
@@ -258,6 +270,8 @@ function _M.handle_disco_info_node(event, service)
 			["pubsub#title"] = node_obj.config.title;
 			["pubsub#description"] = node_obj.config.description;
 			["pubsub#type"] = node_obj.config.payload_type;
+			["pubsub#access_model"] = node_obj.config.access_model;
+			["pubsub#publish_model"] = node_obj.config.publish_model;
 		}, "result"));
 	end
 end
@@ -318,14 +332,9 @@ function handlers.get_items(origin, stanza, items, service)
 	for _, id in ipairs(results) do
 		data:add_child(results[id]);
 	end
-	local reply;
-	if data then
-		reply = st.reply(stanza)
-			:tag("pubsub", { xmlns = xmlns_pubsub })
-				:add_child(data);
-	else
-		reply = pubsub_error_reply(stanza, "item-not-found");
-	end
+	local reply = st.reply(stanza)
+		:tag("pubsub", { xmlns = xmlns_pubsub })
+			:add_child(data);
 	origin.send(reply);
 	return true;
 end
@@ -633,14 +642,13 @@ function handlers.set_retract(origin, stanza, retract, service)
 end
 
 function handlers.owner_set_purge(origin, stanza, purge, service)
-	local node, notify = purge.attr.node, purge.attr.notify;
-	notify = (notify == "1") or (notify == "true");
+	local node = purge.attr.node;
 	local reply;
 	if not node then
 		origin.send(pubsub_error_reply(stanza, "nodeid-required"));
 		return true;
 	end
-	local ok, ret = service:purge(node, stanza.attr.from, notify);
+	local ok, ret = service:purge(node, stanza.attr.from, true);
 	if ok then
 		reply = st.reply(stanza);
 	else
diff --git a/plugins/mod_register_ibr.lua b/plugins/mod_register_ibr.lua
index bbe7581d..6de9bc33 100644
--- a/plugins/mod_register_ibr.lua
+++ b/plugins/mod_register_ibr.lua
@@ -25,6 +25,7 @@ end);
 local account_details = module:open_store("account_details");
 
 local field_map = {
+	FORM_TYPE = { name = "FORM_TYPE", type = "hidden", value = "jabber:iq:register" };
 	username = { name = "username", type = "text-single", label = "Username", required = true };
 	password = { name = "password", type = "text-private", label = "Password", required = true };
 	nick = { name = "nick", type = "text-single", label = "Nickname" };
@@ -50,6 +51,7 @@ local registration_form = dataform_new{
 	title = title;
 	instructions = instructions;
 
+	field_map.FORM_TYPE;
 	field_map.username;
 	field_map.password;
 };
@@ -153,7 +155,7 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 		return true;
 	end
 
-	local username, password = nodeprep(data.username), data.password;
+	local username, password = nodeprep(data.username, true), data.password;
 	data.username, data.password = nil, nil;
 	local host = module.host;
 	if not username or username == "" then
@@ -166,7 +168,15 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 	module:fire_event("user-registering", user);
 	if not user.allowed then
 		log("debug", "Registration disallowed by module: %s", user.reason or "no reason given");
-		session.send(st.error_reply(stanza, "modify", "not-acceptable", user.reason));
+		local error_type, error_condition, reason;
+		local err = user.error;
+		if err then
+			error_type, error_condition, reason = err.type, err.condition, err.text;
+		else
+			-- COMPAT pre-util.error
+			error_type, error_condition, reason = user.error_type, user.error_condition, user.reason;
+		end
+		session.send(st.error_reply(stanza, error_type or "modify", error_condition or "not-acceptable", reason));
 		return true;
 	end
 
@@ -176,14 +186,13 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 		return true;
 	end
 
-	-- TODO unable to write file, file may be locked, etc, what's the correct error?
-	local error_reply = st.error_reply(stanza, "wait", "internal-server-error", "Failed to write data to disk.");
-	if usermanager_create_user(username, password, host) then
+	local created, err = usermanager_create_user(username, password, host);
+	if created then
 		data.registered = os.time();
 		if not account_details:set(username, data) then
 			log("debug", "Could not store extra details");
 			usermanager_delete_user(username, host);
-			session.send(error_reply);
+			session.send(st.error_reply(stanza, "wait", "internal-server-error", "Failed to write data to disk."));
 			return true;
 		end
 		session.send(st.reply(stanza)); -- user created!
@@ -192,8 +201,8 @@ module:hook("stanza/iq/jabber:iq:register:query", function(event)
 			username = username, host = host, source = "mod_register",
 			session = session });
 	else
-		log("debug", "Could not create user");
-		session.send(error_reply);
+		log("debug", "Could not create user", err);
+		session.send(st.error_reply(stanza, "cancel", "feature-not-implemented", err));
 	end
 	return true;
 end);
diff --git a/plugins/mod_register_limits.lua b/plugins/mod_register_limits.lua
index 736282a5..fc9bf27a 100644
--- a/plugins/mod_register_limits.lua
+++ b/plugins/mod_register_limits.lua
@@ -13,6 +13,7 @@ local ip_util = require "util.ip";
 local new_ip = ip_util.new_ip;
 local match_ip = ip_util.match;
 local parse_cidr = ip_util.parse_cidr;
+local errors = require "util.error";
 
 local min_seconds_between_registrations = module:get_option_number("min_seconds_between_registrations");
 local whitelist_only = module:get_option_boolean("whitelist_registration_only");
@@ -54,6 +55,24 @@ local function ip_in_set(set, ip)
 	return false;
 end
 
+local err_registry = {
+	blacklisted = {
+		text = "Your IP address is blacklisted";
+		type = "auth";
+		condition = "forbidden";
+	};
+	not_whitelisted = {
+		text = "Your IP address is not whitelisted";
+		type = "auth";
+		condition = "forbidden";
+	};
+	throttled = {
+		reason = "Too many registrations from this IP address recently";
+		type = "wait";
+		condition = "policy-violation";
+	};
+}
+
 module:hook("user-registering", function (event)
 	local session = event.session;
 	local ip = event.ip or session and session.ip;
@@ -63,16 +82,22 @@ module:hook("user-registering", function (event)
 	elseif ip_in_set(blacklisted_ips, ip) then
 		log("debug", "Registration disallowed by blacklist");
 		event.allowed = false;
-		event.reason = "Your IP address is blacklisted";
+		event.error = errors.new("blacklisted", err_registry, event);
 	elseif (whitelist_only and not ip_in_set(whitelisted_ips, ip)) then
 		log("debug", "Registration disallowed by whitelist");
 		event.allowed = false;
-		event.reason = "Your IP address is not whitelisted";
+		event.error = errors.new("not_whitelisted", err_registry, event);
 	elseif throttle_max and not ip_in_set(whitelisted_ips, ip) then
 		if not check_throttle(ip) then
 			log("debug", "Registrations over limit for ip %s", ip or "?");
 			event.allowed = false;
-			event.reason = "Too many registrations from this IP address recently";
+			event.error = errors.new("throttle", err_registry, event);
 		end
 	end
+	if event.error then
+		-- COMPAT pre-util.error
+		event.reason = event.error.text;
+		event.error_type = event.error.type;
+		event.error_condition = event.error.condition;
+	end
 end);
diff --git a/plugins/mod_s2s/mod_s2s.lua b/plugins/mod_s2s/mod_s2s.lua
index aae37b7f..e7ed8797 100644
--- a/plugins/mod_s2s/mod_s2s.lua
+++ b/plugins/mod_s2s/mod_s2s.lua
@@ -27,8 +27,9 @@ local s2s_destroy_session = require "core.s2smanager".destroy_session;
 local uuid_gen = require "util.uuid".generate;
 local fire_global_event = prosody.events.fire_event;
 local runner = require "util.async".runner;
-
-local s2sout = module:require("s2sout");
+local connect = require "net.connect".connect;
+local service = require "net.resolvers.service";
+local errors = require "util.error";
 
 local connect_timeout = module:get_option_number("s2s_timeout", 90);
 local stream_close_timeout = module:get_option_number("s2s_close_timeout", 5);
@@ -45,6 +46,8 @@ local sessions = module:shared("sessions");
 
 local runner_callbacks = {};
 
+local listener = {};
+
 local log = module._log;
 
 module:hook("stats-update", function ()
@@ -77,15 +80,28 @@ local function bounce_sendq(session, reason)
 			(session.log or log)("error", "Attempting to close the dummy origin of s2s error replies, please report this! Traceback: %s", traceback());
 		end;
 	};
+	-- FIXME Allow for more specific error conditions
+	-- TODO use util.error ?
+	local error_type = "cancel";
+	local condition = "remote-server-not-found";
+	local reason_text;
+	if session.had_stream then -- set when a stream is opened by the remote
+		error_type, condition = "wait", "remote-server-timeout";
+	end
+	if errors.is_err(reason) then
+		error_type, condition, reason_text = reason.type, reason.condition, reason.text;
+	elseif type(reason) == "string" then
+		reason_text = reason;
+	end
 	for i, data in ipairs(sendq) do
 		local reply = data[2];
 		if reply and not(reply.attr.xmlns) and bouncy_stanzas[reply.name] then
 			reply.attr.type = "error";
-			reply:tag("error", {type = "cancel", by = session.from_host})
-				:tag("remote-server-not-found", {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"}):up();
-			if reason then
+			reply:tag("error", {type = error_type, by = session.from_host})
+				:tag(condition, {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"}):up();
+			if reason_text then
 				reply:tag("text", {xmlns = "urn:ietf:params:xml:ns:xmpp-stanzas"})
-					:text("Server-to-server connection failed: "..reason):up();
+					:text("Server-to-server connection failed: "..reason_text):up();
 			end
 			core_process_stanza(dummy, reply);
 		end
@@ -127,14 +143,9 @@ function route_to_existing_session(event)
 		elseif host.type == "local" or host.type == "component" then
 			log("error", "Trying to send a stanza to ourselves??")
 			log("error", "Traceback: %s", traceback());
-			log("error", "Stanza: %s", tostring(stanza));
+			log("error", "Stanza: %s", stanza);
 			return false;
 		else
-			-- FIXME
-			if host.from_host ~= from_host then
-				log("error", "WARNING! This might, possibly, be a bug, but it might not...");
-				log("error", "We are going to send from %s instead of %s", host.from_host, from_host);
-			end
 			if host.sends2s(stanza) then
 				return true;
 			end
@@ -147,17 +158,13 @@ function route_to_new_session(event)
 	local from_host, to_host, stanza = event.from_host, event.to_host, event.stanza;
 	log("debug", "opening a new outgoing connection for this stanza");
 	local host_session = s2s_new_outgoing(from_host, to_host);
+	host_session.version = 1;
 
 	-- Store in buffer
 	host_session.bounce_sendq = bounce_sendq;
 	host_session.sendq = { {tostring(stanza), stanza.attr.type ~= "error" and stanza.attr.type ~= "result" and st.reply(stanza)} };
-	log("debug", "stanza [%s] queued until connection complete", tostring(stanza.name));
-	s2sout.initiate_connection(host_session);
-	if (not host_session.connecting) and (not host_session.conn) then
-		log("warn", "Connection to %s failed already, destroying session...", to_host);
-		s2s_destroy_session(host_session, "Connection failed");
-		return false;
-	end
+	log("debug", "stanza [%s] queued until connection complete", stanza.name);
+	connect(service.new(to_host, "xmpp-server", "tcp", { default_port = 5269 }), listener, nil, { session = host_session });
 	return true;
 end
 
@@ -184,7 +191,10 @@ function module.add_host(module)
 			return true;
 		elseif not session.dialback_verifying then
 			session.log("warn", "No SASL EXTERNAL offer and Dialback doesn't seem to be enabled, giving up");
-			session:close();
+			session:close({
+					condition = "unsupported-feature",
+					text = "No viable authentication method offered",
+				}, nil, "No viable authentication method offered by remote server");
 			return false;
 		end
 	end, -1);
@@ -203,7 +213,18 @@ function mark_connected(session)
 	if session.type == "s2sout" then
 		fire_global_event("s2sout-established", event_data);
 		hosts[from].events.fire_event("s2sout-established", event_data);
+
+		if session.incoming then
+			session.send = function(stanza)
+				return hosts[from].events.fire_event("route/remote", { from_host = from, to_host = to, stanza = stanza });
+			end;
+		end
+
 	else
+		if session.outgoing and not hosts[to].s2sout[from] then
+			session.log("debug", "Setting up to handle route from %s to %s", to, from);
+			hosts[to].s2sout[from] = session; -- luacheck: ignore 122
+		end
 		local host_session = hosts[to];
 		session.send = function(stanza)
 			return host_session.events.fire_event("route/remote", { from_host = to, to_host = from, stanza = stanza });
@@ -223,13 +244,6 @@ function mark_connected(session)
 			end
 			session.sendq = nil;
 		end
-
-		if session.resolver then
-			session.resolver._resolver:closeall()
-		end
-		session.resolver = nil;
-		session.ip_hosts = nil;
-		session.srv_hosts = nil;
 	end
 end
 
@@ -241,7 +255,7 @@ function make_authenticated(event)
 				condition = "policy-violation",
 				text = "Encrypted server-to-server communication is required but was not "
 				       ..((session.direction == "outgoing" and "offered") or "used")
-			});
+			}, nil, "Could not establish encrypted connection to remote server");
 		end
 	end
 	if hosts[host] then
@@ -251,15 +265,13 @@ function make_authenticated(event)
 		session.type = "s2sout";
 	elseif session.type == "s2sin_unauthed" then
 		session.type = "s2sin";
-		if host then
-			if not session.hosts[host] then session.hosts[host] = {}; end
-			session.hosts[host].authed = true;
-		end
-	elseif session.type == "s2sin" and host then
+	elseif session.type ~= "s2sin" and session.type ~= "s2sout" then
+		return false;
+	end
+
+	if session.incoming and host then
 		if not session.hosts[host] then session.hosts[host] = {}; end
 		session.hosts[host].authed = true;
-	else
-		return false;
 	end
 	session.log("debug", "connection %s->%s is now authenticated for %s", session.from_host, session.to_host, host);
 
@@ -301,6 +313,7 @@ end
 
 function stream_callbacks._streamopened(session, attr)
 	session.version = tonumber(attr.version) or 0;
+	session.had_stream = true; -- Had a stream opened at least once
 
 	-- TODO: Rename session.secure to session.encrypted
 	if session.secure == false then
@@ -314,7 +327,6 @@ function stream_callbacks._streamopened(session, attr)
 			session.compressed = info.compression;
 		else
 			(session.log or log)("info", "Stream encrypted");
-			session.compressed = sock.compression and sock:compression(); --COMPAT mw/luasec-hg
 		end
 	end
 
@@ -322,7 +334,9 @@ function stream_callbacks._streamopened(session, attr)
 		-- Send a reply stream header
 
 		-- Validate to/from
-		local to, from = nameprep(attr.to), nameprep(attr.from);
+		local to, from = attr.to, attr.from;
+		if to then to = nameprep(attr.to); end
+		if from then from = nameprep(attr.from); end
 		if not to and attr.to then -- COMPAT: Some servers do not reliably set 'to' (especially on stream restarts)
 			session:close({ condition = "improper-addressing", text = "Invalid 'to' address" });
 			return;
@@ -471,11 +485,9 @@ function stream_callbacks.error(session, error, data)
 	end
 end
 
-local listener = {};
-
 --- Session methods
 local stream_xmlns_attr = {xmlns='urn:ietf:params:xml:ns:xmpp-streams'};
-local function session_close(session, reason, remote_reason)
+local function session_close(session, reason, remote_reason, bounce_reason)
 	local log = session.log or log;
 	if session.conn then
 		if session.notopen then
@@ -521,16 +533,16 @@ local function session_close(session, reason, remote_reason)
 
 		-- Authenticated incoming stream may still be sending us stanzas, so wait for </stream:stream> from remote
 		local conn = session.conn;
-		if reason == nil and not session.notopen and session.type == "s2sin" then
+		if reason == nil and not session.notopen and session.incoming then
 			add_task(stream_close_timeout, function ()
 				if not session.destroyed then
 					session.log("warn", "Failed to receive a stream close response, closing connection anyway...");
-					s2s_destroy_session(session, reason);
+					s2s_destroy_session(session, reason, bounce_reason);
 					conn:close();
 				end
 			end);
 		else
-			s2s_destroy_session(session, reason);
+			s2s_destroy_session(session, reason, bounce_reason);
 			conn:close(); -- Close immediately, as this is an outgoing connection or is not authed
 		end
 	end
@@ -595,9 +607,8 @@ local function initialize_session(session)
 		if data then
 			local ok, err = stream:feed(data);
 			if ok then return; end
-			log("warn", "Received invalid XML: %s", data);
-			log("warn", "Problem was: %s", err);
-			session:close("not-well-formed");
+			log("debug", "Received invalid XML (%s) %d bytes: %q", err, #data, data:sub(1, 300));
+			session:close("not-well-formed", nil, "Received invalid XML from remote server");
 		end
 	end
 
@@ -672,11 +683,16 @@ function listener.ondisconnect(conn, err)
 	local session = sessions[conn];
 	if session then
 		sessions[conn] = nil;
+		(session.log or log)("debug", "s2s disconnected: %s->%s (%s)", session.from_host, session.to_host, err or "connection closed");
+		s2s_destroy_session(session, err);
+	end
+end
+
+function listener.onfail(data, err)
+	local session = data and data.session;
+	if session then
 		if err and session.direction == "outgoing" and session.notopen then
 			(session.log or log)("debug", "s2s connection attempt failed: %s", err);
-			if s2sout.attempt_connection(session, err) then
-				return; -- Session lives for now
-			end
 		end
 		(session.log or log)("debug", "s2s disconnected: %s->%s (%s)", session.from_host, session.to_host, err or "connection closed");
 		s2s_destroy_session(session, err);
@@ -700,6 +716,15 @@ function listener.ondetach(conn)
 	sessions[conn] = nil;
 end
 
+function listener.onattach(conn, data)
+	local session = data and data.session;
+	if session then
+		session.conn = conn;
+		sessions[conn] = session;
+		initialize_session(session);
+	end
+end
+
 function check_auth_policy(event)
 	local host, session = event.host, event.session;
 	local must_secure = secure_auth;
@@ -713,9 +738,10 @@ function check_auth_policy(event)
 	if must_secure and (session.cert_chain_status ~= "valid" or session.cert_identity_status ~= "valid") then
 		module:log("warn", "Forbidding insecure connection to/from %s", host or session.ip or "(unknown host)");
 		if session.direction == "incoming" then
-			session:close({ condition = "not-authorized", text = "Your server's certificate is invalid, expired, or not trusted by "..session.to_host });
+			session:close({ condition = "not-authorized", text = "Your server's certificate is invalid, expired, or not trusted by "..session.to_host },
+				nil, "Remote server's certificate is invalid, expired, or not trusted");
 		else -- Close outgoing connections without warning
-			session:close(false);
+			session:close(false, nil, "Remote server's certificate is invalid, expired, or not trusted");
 		end
 		return false;
 	end
@@ -723,8 +749,6 @@ end
 
 module:hook("s2s-check-certificate", check_auth_policy, -1);
 
-s2sout.set_listener(listener);
-
 module:hook("server-stopping", function(event)
 	local reason = event.reason;
 	for _, session in pairs(sessions) do
@@ -739,6 +763,9 @@ module:provides("net", {
 	listener = listener;
 	default_port = 5269;
 	encryption = "starttls";
+	ssl_config = { -- FIXME This is not used atm, see mod_tls
+		verify = { "peer", "client_once", };
+	};
 	multiplex = {
 		pattern = "^<.*:stream.*%sxmlns%s*=%s*(['\"])jabber:server%1.*>";
 	};
diff --git a/plugins/mod_s2s/s2sout.lib.lua b/plugins/mod_s2s/s2sout.lib.lua
deleted file mode 100644
index 5f765da8..00000000
--- a/plugins/mod_s2s/s2sout.lib.lua
+++ /dev/null
@@ -1,349 +0,0 @@
--- Prosody IM
--- Copyright (C) 2008-2010 Matthew Wild
--- Copyright (C) 2008-2010 Waqas Hussain
---
--- This project is MIT/X11 licensed. Please see the
--- COPYING file in the source package for more information.
---
-
---- Module containing all the logic for connecting to a remote server
-
--- luacheck: ignore 432/err
-
-local portmanager = require "core.portmanager";
-local wrapclient = require "net.server".wrapclient;
-local initialize_filters = require "util.filters".initialize;
-local idna_to_ascii = require "util.encodings".idna.to_ascii;
-local new_ip = require "util.ip".new_ip;
-local rfc6724_dest = require "util.rfc6724".destination;
-local socket = require "socket";
-local adns = require "net.adns";
-local t_insert, t_sort, ipairs = table.insert, table.sort, ipairs;
-local local_addresses = require "util.net".local_addresses;
-
-local s2s_destroy_session = require "core.s2smanager".destroy_session;
-
-local default_mode = module:get_option("network_default_read_size", 4096);
-
-local log = module._log;
-
-local sources = {};
-local has_ipv4, has_ipv6;
-
-local dns_timeout = module:get_option_number("dns_timeout", 15);
-local resolvers = module:get_option_set("s2s_dns_resolvers")
-
-local s2sout = {};
-
-local s2s_listener;
-
-
-function s2sout.set_listener(listener)
-	s2s_listener = listener;
-end
-
-local function compare_srv_priorities(a,b)
-	return a.priority < b.priority or (a.priority == b.priority and a.weight > b.weight);
-end
-
-function s2sout.initiate_connection(host_session)
-	local log = host_session.log or log;
-
-	initialize_filters(host_session);
-	host_session.version = 1;
-
-	host_session.resolver = adns.resolver();
-	host_session.resolver._resolver:settimeout(dns_timeout);
-	if resolvers then
-		for resolver in resolvers do
-			host_session.resolver._resolver:addnameserver(resolver);
-		end
-	end
-
-	-- Kick the connection attempting machine into life
-	if not s2sout.attempt_connection(host_session) then
-		-- Intentionally not returning here, the
-		-- session is needed, connected or not
-		s2s_destroy_session(host_session);
-	end
-
-	if not host_session.sends2s then
-		-- A sends2s which buffers data (until the stream is opened)
-		-- note that data in this buffer will be sent before the stream is authed
-		-- and will not be ack'd in any way, successful or otherwise
-		local buffer;
-		function host_session.sends2s(data)
-			if not buffer then
-				buffer = {};
-				host_session.send_buffer = buffer;
-			end
-			log("debug", "Buffering data on unconnected s2sout to %s", host_session.to_host);
-			buffer[#buffer+1] = data;
-			log("debug", "Buffered item %d: %s", #buffer, data);
-		end
-	end
-end
-
-function s2sout.attempt_connection(host_session, err)
-	local to_host = host_session.to_host;
-	local connect_host, connect_port = to_host and idna_to_ascii(to_host), 5269;
-	local log = host_session.log or log;
-
-	if not connect_host then
-		return false;
-	end
-
-	if not err then -- This is our first attempt
-		log("debug", "First attempt to connect to %s, starting with SRV lookup...", to_host);
-		host_session.connecting = true;
-		host_session.resolver:lookup(function (answer)
-			local srv_hosts = { answer = answer };
-			host_session.srv_hosts = srv_hosts;
-			host_session.srv_choice = 0;
-			host_session.connecting = nil;
-			if answer and #answer > 0 then
-				log("debug", "%s has SRV records, handling...", to_host);
-				for _, record in ipairs(answer) do
-					t_insert(srv_hosts, record.srv);
-				end
-				if #srv_hosts == 1 and srv_hosts[1].target == "." then
-					log("debug", "%s does not provide a XMPP service", to_host);
-					s2s_destroy_session(host_session, err); -- Nothing to see here
-					return;
-				end
-				t_sort(srv_hosts, compare_srv_priorities);
-
-				local srv_choice = srv_hosts[1];
-				host_session.srv_choice = 1;
-				if srv_choice then
-					connect_host, connect_port = srv_choice.target or to_host, srv_choice.port or connect_port;
-					log("debug", "Best record found, will connect to %s:%d", connect_host, connect_port);
-				end
-			else
-				log("debug", "%s has no SRV records, falling back to A/AAAA", to_host);
-			end
-			-- Try with SRV, or just the plain hostname if no SRV
-			local ok, err = s2sout.try_connect(host_session, connect_host, connect_port);
-			if not ok then
-				if not s2sout.attempt_connection(host_session, err) then
-					-- No more attempts will be made
-					s2s_destroy_session(host_session, err);
-				end
-			end
-		end, "_xmpp-server._tcp."..connect_host..".", "SRV");
-
-		return true; -- Attempt in progress
-	elseif host_session.ip_hosts then
-		return s2sout.try_connect(host_session, connect_host, connect_port, err);
-	elseif host_session.srv_hosts and #host_session.srv_hosts > host_session.srv_choice then -- Not our first attempt, and we also have SRV
-		host_session.srv_choice = host_session.srv_choice + 1;
-		local srv_choice = host_session.srv_hosts[host_session.srv_choice];
-		connect_host, connect_port = srv_choice.target or to_host, srv_choice.port or connect_port;
-		host_session.log("info", "Connection failed (%s). Attempt #%d: This time to %s:%d", err, host_session.srv_choice, connect_host, connect_port);
-	else
-		host_session.log("info", "Failed in all attempts to connect to %s", host_session.to_host);
-		-- We're out of options
-		return false;
-	end
-
-	if not (connect_host and connect_port) then
-		-- Likely we couldn't resolve DNS
-		log("warn", "Hmm, we're without a host (%s) and port (%s) to connect to for %s, giving up :(", connect_host, connect_port, to_host);
-		return false;
-	end
-
-	return s2sout.try_connect(host_session, connect_host, connect_port);
-end
-
-function s2sout.try_next_ip(host_session)
-	host_session.connecting = nil;
-	host_session.ip_choice = host_session.ip_choice + 1;
-	local ip = host_session.ip_hosts[host_session.ip_choice];
-	local ok, err= s2sout.make_connect(host_session, ip.ip, ip.port);
-	if not ok then
-		if not s2sout.attempt_connection(host_session, err or "closed") then
-			err = err and (": "..err) or "";
-			s2s_destroy_session(host_session, "Connection failed"..err);
-		end
-	end
-end
-
-function s2sout.try_connect(host_session, connect_host, connect_port, err)
-	host_session.connecting = true;
-	local log = host_session.log or log;
-
-	if not err then
-		local IPs = {};
-		host_session.ip_hosts = IPs;
-		-- luacheck: ignore 231/handle4 231/handle6
-		local handle4, handle6;
-		local have_other_result = not(has_ipv4) or not(has_ipv6) or false;
-
-		if has_ipv4 then
-			handle4 = host_session.resolver:lookup(function (reply, err)
-				handle4 = nil;
-
-				if reply and reply[#reply] and reply[#reply].a then
-					for _, ip in ipairs(reply) do
-						log("debug", "DNS reply for %s gives us %s", connect_host, ip.a);
-						IPs[#IPs+1] = new_ip(ip.a, "IPv4");
-					end
-				elseif err then
-					log("debug", "Error in DNS lookup: %s", err);
-				end
-
-				if have_other_result then
-					if #IPs > 0 then
-						rfc6724_dest(host_session.ip_hosts, sources);
-						for i = 1, #IPs do
-							IPs[i] = {ip = IPs[i], port = connect_port};
-						end
-						host_session.ip_choice = 0;
-						s2sout.try_next_ip(host_session);
-					else
-						log("debug", "DNS lookup failed to get a response for %s", connect_host);
-						host_session.ip_hosts = nil;
-						if not s2sout.attempt_connection(host_session, "name resolution failed") then -- Retry if we can
-							log("debug", "No other records to try for %s - destroying", host_session.to_host);
-							err = err and (": "..err) or "";
-							s2s_destroy_session(host_session, "DNS resolution failed"..err); -- End of the line, we can't
-						end
-					end
-				else
-					have_other_result = true;
-				end
-			end, connect_host, "A", "IN");
-		else
-			have_other_result = true;
-		end
-
-		if has_ipv6 then
-			handle6 = host_session.resolver:lookup(function (reply, err)
-				handle6 = nil;
-
-				if reply and reply[#reply] and reply[#reply].aaaa then
-					for _, ip in ipairs(reply) do
-						log("debug", "DNS reply for %s gives us %s", connect_host, ip.aaaa);
-						IPs[#IPs+1] = new_ip(ip.aaaa, "IPv6");
-					end
-				elseif err then
-					log("debug", "Error in DNS lookup: %s", err);
-				end
-
-				if have_other_result then
-					if #IPs > 0 then
-						rfc6724_dest(host_session.ip_hosts, sources);
-						for i = 1, #IPs do
-							IPs[i] = {ip = IPs[i], port = connect_port};
-						end
-						host_session.ip_choice = 0;
-						s2sout.try_next_ip(host_session);
-					else
-						log("debug", "DNS lookup failed to get a response for %s", connect_host);
-						host_session.ip_hosts = nil;
-						if not s2sout.attempt_connection(host_session, "name resolution failed") then -- Retry if we can
-							log("debug", "No other records to try for %s - destroying", host_session.to_host);
-							err = err and (": "..err) or "";
-							s2s_destroy_session(host_session, "DNS resolution failed"..err); -- End of the line, we can't
-						end
-					end
-				else
-					have_other_result = true;
-				end
-			end, connect_host, "AAAA", "IN");
-		else
-			have_other_result = true;
-		end
-		return true;
-	elseif host_session.ip_hosts and #host_session.ip_hosts > host_session.ip_choice then -- Not our first attempt, and we also have IPs left to try
-		s2sout.try_next_ip(host_session);
-	else
-		log("debug", "Out of IP addresses, trying next SRV record (if any)");
-		host_session.ip_hosts = nil;
-		if not s2sout.attempt_connection(host_session, "out of IP addresses") then -- Retry if we can
-			log("debug", "No other records to try for %s - destroying", host_session.to_host);
-			err = err and (": "..err) or "";
-			s2s_destroy_session(host_session, "Connecting failed"..err); -- End of the line, we can't
-			return false;
-		end
-	end
-
-	return true;
-end
-
-function s2sout.make_connect(host_session, connect_host, connect_port)
-	local log = host_session.log or log;
-	log("debug", "Beginning new connection attempt to %s ([%s]:%d)", host_session.to_host, connect_host.addr, connect_port);
-
-	-- Reset secure flag in case this is another
-	-- connection attempt after a failed STARTTLS
-	host_session.secure = nil;
-	host_session.encrypted = nil;
-
-	local conn, handler;
-	local proto = connect_host.proto;
-	if proto == "IPv4" then
-		conn, handler = socket.tcp();
-	elseif proto == "IPv6" and socket.tcp6 then
-		conn, handler = socket.tcp6();
-	else
-		handler = "Unsupported protocol: "..tostring(proto);
-	end
-
-	if not conn then
-		log("warn", "Failed to create outgoing connection, system error: %s", handler);
-		return false, handler;
-	end
-
-	conn:settimeout(0);
-	local success, err = conn:connect(connect_host.addr, connect_port);
-	if not success and err ~= "timeout" then
-		log("warn", "s2s connect() to %s (%s:%d) failed: %s", host_session.to_host, connect_host.addr, connect_port, err);
-		return false, err;
-	end
-
-	conn = wrapclient(conn, connect_host.addr, connect_port, s2s_listener, default_mode);
-	host_session.conn = conn;
-
-	-- Register this outgoing connection so that xmppserver_listener knows about it
-	-- otherwise it will assume it is a new incoming connection
-	s2s_listener.register_outgoing(conn, host_session);
-
-	log("debug", "Connection attempt in progress...");
-	return true;
-end
-
-module:hook_global("service-added", function (event)
-	if event.name ~= "s2s" then return end
-
-	local s2s_sources = portmanager.get_active_services():get("s2s");
-	if not s2s_sources then
-		module:log("warn", "s2s not listening on any ports, outgoing connections may fail");
-		return;
-	end
-	for source, _ in pairs(s2s_sources) do
-		if source == "*" or source == "0.0.0.0" then
-			for _, addr in ipairs(local_addresses("ipv4", true)) do
-				sources[#sources + 1] = new_ip(addr, "IPv4");
-			end
-		elseif source == "::" then
-			for _, addr in ipairs(local_addresses("ipv6", true)) do
-				sources[#sources + 1] = new_ip(addr, "IPv6");
-			end
-		else
-			sources[#sources + 1] = new_ip(source, (source:find(":") and "IPv6") or "IPv4");
-		end
-	end
-	for i = 1,#sources do
-		if sources[i].proto == "IPv6" then
-			has_ipv6 = true;
-		elseif sources[i].proto == "IPv4" then
-			has_ipv4 = true;
-		end
-	end
-	if not (has_ipv4 or has_ipv6)  then
-		module:log("warn", "No local IPv4 or IPv6 addresses detected, outgoing connections may fail");
-	end
-end);
-
-return s2sout;
diff --git a/plugins/mod_s2s_auth_certs.lua b/plugins/mod_s2s_auth_certs.lua
index dd0eb3cb..76e134a7 100644
--- a/plugins/mod_s2s_auth_certs.lua
+++ b/plugins/mod_s2s_auth_certs.lua
@@ -17,9 +17,6 @@ module:hook("s2s-check-certificate", function(event)
 	local chain_valid, errors;
 	if conn.getpeerverification then
 		chain_valid, errors = conn:getpeerverification();
-	elseif conn.getpeerchainvalid then -- COMPAT mw/luasec-hg
-		chain_valid, errors = conn:getpeerchainvalid();
-		errors = (not chain_valid) and { { errors } } or nil;
 	else
 		chain_valid, errors = false, { { "Chain verification not supported by this version of LuaSec" } };
 	end
diff --git a/plugins/mod_s2s_bidi.lua b/plugins/mod_s2s_bidi.lua
new file mode 100644
index 00000000..67a48d8d
--- /dev/null
+++ b/plugins/mod_s2s_bidi.lua
@@ -0,0 +1,38 @@
+-- Prosody IM
+-- Copyright (C) 2019 Kim Alvefur
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local st = require "util.stanza";
+
+local xmlns_bidi_feature = "urn:xmpp:features:bidi"
+local xmlns_bidi = "urn:xmpp:bidi";
+
+module:hook("s2s-stream-features", function(event)
+	local origin, features = event.origin, event.features;
+	if origin.type == "s2sin_unauthed" then
+		features:tag("bidi", { xmlns = xmlns_bidi_feature }):up();
+	end
+end);
+
+module:hook_tag("http://etherx.jabber.org/streams", "features", function (session, stanza)
+	if session.type == "s2sout_unauthed" then
+		local bidi = stanza:get_child("bidi", xmlns_bidi_feature);
+		if bidi then
+			session.incoming = true;
+			session.log("debug", "Requesting bidirectional stream");
+			session.sends2s(st.stanza("bidi", { xmlns = xmlns_bidi }));
+		end
+	end
+end, 200);
+
+module:hook_tag("urn:xmpp:bidi", "bidi", function(session)
+	if session.type == "s2sin_unauthed" then
+		session.log("debug", "Requested bidirectional stream");
+		session.outgoing = true;
+		return true;
+	end
+end);
+
diff --git a/plugins/mod_saslauth.lua b/plugins/mod_saslauth.lua
index fba84ef8..9e9091d3 100644
--- a/plugins/mod_saslauth.lua
+++ b/plugins/mod_saslauth.lua
@@ -12,9 +12,9 @@ local st = require "util.stanza";
 local sm_bind_resource = require "core.sessionmanager".bind_resource;
 local sm_make_authenticated = require "core.sessionmanager".make_authenticated;
 local base64 = require "util.encodings".base64;
+local set = require "util.set";
 
 local usermanager_get_sasl_handler = require "core.usermanager".get_sasl_handler;
-local tostring = tostring;
 
 local secure_auth_only = module:get_option_boolean("c2s_require_encryption", module:get_option_boolean("require_encryption", false));
 local allow_unencrypted_plain_auth = module:get_option_boolean("allow_unencrypted_plain_auth", false)
@@ -67,7 +67,6 @@ local function sasl_process_cdata(session, stanza)
 	local text = stanza[1];
 	if text then
 		text = base64.decode(text);
-		--log("debug", "AUTH: %s", text:gsub("[%z\001-\008\011\012\014-\031]", " "));
 		if not text then
 			session.sasl_handler = nil;
 			session.send(build_reply("failure", "incorrect-encoding"));
@@ -77,7 +76,6 @@ local function sasl_process_cdata(session, stanza)
 	local status, ret, err_msg = session.sasl_handler:process(text);
 	status, ret, err_msg = handle_status(session, status, ret, err_msg);
 	local s = build_reply(status, ret, err_msg);
-	log("debug", "sasl reply: %s", tostring(s));
 	session.send(s);
 	return true;
 end
@@ -248,37 +246,72 @@ module:hook("stream-features", function(event)
 		local sasl_handler = usermanager_get_sasl_handler(module.host, origin)
 		origin.sasl_handler = sasl_handler;
 		if origin.encrypted then
-			-- check wether LuaSec has the nifty binding to the function needed for tls-unique
+			-- check whether LuaSec has the nifty binding to the function needed for tls-unique
 			-- FIXME: would be nice to have this check only once and not for every socket
 			if sasl_handler.add_cb_handler then
 				local socket = origin.conn:socket();
 				if socket.getpeerfinished then
+					log("debug", "Channel binding 'tls-unique' supported");
 					sasl_handler:add_cb_handler("tls-unique", tls_unique);
+				else
+					log("debug", "Channel binding 'tls-unique' not supported (by LuaSec?)");
 				end
 				sasl_handler["userdata"] = {
 					["tls-unique"] = socket;
 				};
+			else
+				log("debug", "Channel binding not supported by SASL handler");
 			end
 		end
 		local mechanisms = st.stanza("mechanisms", mechanisms_attr);
 		local sasl_mechanisms = sasl_handler:mechanisms()
+		local available_mechanisms = set.new();
 		for mechanism in pairs(sasl_mechanisms) do
-			if disabled_mechanisms:contains(mechanism) then
-				log("debug", "Not offering disabled mechanism %s", mechanism);
-			elseif not origin.secure and insecure_mechanisms:contains(mechanism) then
-				log("debug", "Not offering mechanism %s on insecure connection", mechanism);
-			else
-				log("debug", "Offering mechanism %s", mechanism);
+			available_mechanisms:add(mechanism);
+		end
+		log("debug", "SASL mechanisms supported by handler: %s", available_mechanisms);
+
+		local usable_mechanisms = available_mechanisms - disabled_mechanisms;
+
+		local available_disabled = set.intersection(available_mechanisms, disabled_mechanisms);
+		if not available_disabled:empty() then
+			log("debug", "Not offering disabled mechanisms: %s", available_disabled);
+		end
+
+		local available_insecure = set.intersection(available_mechanisms, insecure_mechanisms);
+		if not origin.secure and not available_insecure:empty() then
+			log("debug", "Session is not secure, not offering insecure mechanisms: %s", available_insecure);
+			usable_mechanisms = usable_mechanisms - insecure_mechanisms;
+		end
+
+		if not usable_mechanisms:empty() then
+			log("debug", "Offering usable mechanisms: %s", usable_mechanisms);
+			for mechanism in available_mechanisms do
 				mechanisms:tag("mechanism"):text(mechanism):up();
 			end
-		end
-		if mechanisms[1] then
 			features:add_child(mechanisms);
-		elseif not next(sasl_mechanisms) then
-			log("warn", "No available SASL mechanisms, verify that the configured authentication module is working");
-		else
-			log("warn", "All available authentication mechanisms are either disabled or not suitable for an insecure connection");
+			return;
+		end
+
+		local authmod = module:get_option_string("authentication", "internal_plain");
+		if available_mechanisms:empty() then
+			log("warn", "No available SASL mechanisms, verify that the configured authentication module '%s' is loaded and configured correctly", authmod);
+			return;
 		end
+
+		if not origin.secure and not available_insecure:empty() then
+			if not available_disabled:empty() then
+				log("warn", "All SASL mechanisms provided by authentication module '%s' are forbidden on insecure connections (%s) or disabled (%s)",
+					authmod, available_insecure, available_disabled);
+			else
+				log("warn", "All SASL mechanisms provided by authentication module '%s' are forbidden on insecure connections (%s)",
+					authmod, available_insecure);
+			end
+		elseif not available_disabled:empty() then
+			log("warn", "All SASL mechanisms provided by authentication module '%s' are disabled (%s)",
+				authmod, available_disabled);
+		end
+
 	else
 		features:tag("bind", bind_attr):tag("required"):up():up();
 		features:tag("session", xmpp_session_attr):tag("optional"):up():up();
diff --git a/plugins/mod_stanza_debug.lua b/plugins/mod_stanza_debug.lua
index 6dedb6f7..af98670c 100644
--- a/plugins/mod_stanza_debug.lua
+++ b/plugins/mod_stanza_debug.lua
@@ -1,18 +1,17 @@
 module:set_global();
 
-local tostring = tostring;
 local filters = require "util.filters";
 
 local function log_send(t, session)
 	if t and t ~= "" and t ~= " " then
-		session.log("debug", "SEND: %s", tostring(t));
+		session.log("debug", "SEND: %s", t);
 	end
 	return t;
 end
 
 local function log_recv(t, session)
 	if t and t ~= "" and t ~= " " then
-		session.log("debug", "RECV: %s", tostring(t));
+		session.log("debug", "RECV: %s", t);
 	end
 	return t;
 end
diff --git a/plugins/mod_storage_internal.lua b/plugins/mod_storage_internal.lua
index 42b451bd..c8b902cf 100644
--- a/plugins/mod_storage_internal.lua
+++ b/plugins/mod_storage_internal.lua
@@ -1,12 +1,17 @@
+local cache = require "util.cache";
 local datamanager = require "core.storagemanager".olddm;
 local array = require "util.array";
 local datetime = require "util.datetime";
 local st = require "util.stanza";
 local now = require "util.time".now;
 local id = require "util.id".medium;
+local jid_join = require "util.jid".join;
 
 local host = module.host;
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", 10000);
+local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
+
 local driver = {};
 
 function driver:open(store, typ)
@@ -43,6 +48,12 @@ end
 local archive = {};
 driver.archive = { __index = archive };
 
+archive.caps = {
+	total = true;
+	quota = archive_item_limit;
+	truncate = true;
+};
+
 function archive:append(username, key, value, when, with)
 	when = when or now();
 	if not st.is_stanza(value) then
@@ -54,28 +65,57 @@ function archive:append(username, key, value, when, with)
 	value.attr.stamp = datetime.datetime(when);
 	value.attr.stamp_legacy = datetime.legacy(when);
 
+	local cache_key = jid_join(username, host, self.store);
+	local item_count = archive_item_count_cache:get(cache_key);
+
 	if key then
 		local items, err = datamanager.list_load(username, host, self.store);
 		if not items and err then return items, err; end
+
+		-- Check the quota
+		item_count = items and #items or 0;
+		archive_item_count_cache:set(cache_key, item_count);
+		if item_count >= archive_item_limit then
+			module:log("debug", "%s reached or over quota, not adding to store", username);
+			return nil, "quota-limit";
+		end
+
 		if items then
+			-- Filter out any item with the same key as the one being added
 			items = array(items);
 			items:filter(function (item)
 				return item.key ~= key;
 			end);
+
 			value.key = key;
 			items:push(value);
 			local ok, err = datamanager.list_store(username, host, self.store, items);
 			if not ok then return ok, err; end
+			archive_item_count_cache:set(cache_key, #items);
 			return key;
 		end
 	else
+		if not item_count then -- Item count not cached?
+			-- We need to load the list to get the number of items currently stored
+			local items, err = datamanager.list_load(username, host, self.store);
+			if not items and err then return items, err; end
+			item_count = items and #items or 0;
+			archive_item_count_cache:set(cache_key, item_count);
+		end
+		if item_count >= archive_item_limit then
+			module:log("debug", "%s reached or over quota, not adding to store", username);
+			return nil, "quota-limit";
+		end
 		key = id();
 	end
 
+	module:log("debug", "%s has %d items out of %d limit in store %s", username, item_count, archive_item_limit, self.store);
+
 	value.key = key;
 
 	local ok, err = datamanager.list_append(username, host, self.store, value);
 	if not ok then return ok, err; end
+	archive_item_count_cache:set(cache_key, item_count+1);
 	return key;
 end
 
@@ -84,11 +124,17 @@ function archive:find(username, query)
 	if not items then
 		if err then
 			return items, err;
-		else
-			return function () end, 0;
+		elseif query then
+			if query.before or query.after then
+				return nil, "item-not-found";
+			end
+			if query.total then
+				return function () end, 0;
+			end
 		end
+		return function () end;
 	end
-	local count = #items;
+	local count = nil;
 	local i = 0;
 	if query then
 		items = array(items);
@@ -112,24 +158,36 @@ function archive:find(username, query)
 				return item.when <= query["end"];
 			end);
 		end
-		count = #items;
+		if query.total then
+			count = #items;
+		end
 		if query.reverse then
 			items:reverse();
 			if query.before then
-				for j = 1, count do
+				local found = false;
+				for j = 1, #items do
 					if (items[j].key or tostring(j)) == query.before then
+						found = true;
 						i = j;
 						break;
 					end
 				end
+				if not found then
+					return nil, "item-not-found";
+				end
 			end
 		elseif query.after then
-			for j = 1, count do
+			local found = false;
+			for j = 1, #items do
 				if (items[j].key or tostring(j)) == query.after then
+					found = true;
 					i = j;
 					break;
 				end
 			end
+			if not found then
+				return nil, "item-not-found";
+			end
 		end
 		if query.limit and #items - i > query.limit then
 			items[i+query.limit+1] = nil;
@@ -156,8 +214,37 @@ function archive:dates(username)
 	return array(items):pluck("when"):map(datetime.date):unique();
 end
 
+function archive:summary(username, query)
+	local iter, err = self:find(username, query)
+	if not iter then return iter, err; end
+	local counts = {};
+	local earliest = {};
+	local latest = {};
+	local body = {};
+	for _, stanza, when, with in iter do
+		counts[with] = (counts[with] or 0) + 1;
+		if earliest[with] == nil then
+			earliest[with] = when;
+		end
+		latest[with] = when;
+		body[with] = stanza:get_child_text("body") or body[with];
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+		body = body;
+	};
+end
+
+function archive:users()
+	return datamanager.users(host, self.store, "list");
+end
+
 function archive:delete(username, query)
+	local cache_key = jid_join(username, host, self.store);
 	if not query or next(query) == nil then
+		archive_item_count_cache:set(cache_key, nil);
 		return datamanager.list_store(username, host, self.store, nil);
 	end
 	local items, err = datamanager.list_load(username, host, self.store);
@@ -165,6 +252,7 @@ function archive:delete(username, query)
 		if err then
 			return items, err;
 		end
+		archive_item_count_cache:set(cache_key, 0);
 		-- Store is empty
 		return 0;
 	end
@@ -214,6 +302,7 @@ function archive:delete(username, query)
 	end
 	local ok, err = datamanager.list_store(username, host, self.store, items);
 	if not ok then return ok, err; end
+	archive_item_count_cache:set(cache_key, #items);
 	return count;
 end
 
diff --git a/plugins/mod_storage_memory.lua b/plugins/mod_storage_memory.lua
index 745e394b..8beb8c01 100644
--- a/plugins/mod_storage_memory.lua
+++ b/plugins/mod_storage_memory.lua
@@ -8,6 +8,8 @@ local new_id = require "util.id".medium;
 local auto_purge_enabled = module:get_option_boolean("storage_memory_temporary", false);
 local auto_purge_stores = module:get_option_set("storage_memory_temporary_stores", {});
 
+local archive_item_limit = module:get_option_number("storage_archive_item_limit", 1000);
+
 local memory = setmetatable({}, {
 	__index = function(t, k)
 		local store = module:shared(k)
@@ -51,6 +53,12 @@ archive_store.__index = archive_store;
 
 archive_store.users = _users;
 
+archive_store.caps = {
+	total = true;
+	quota = archive_item_limit;
+	truncate = true;
+};
+
 function archive_store:append(username, key, value, when, with)
 	if is_stanza(value) then
 		value = st.preserialize(value);
@@ -70,6 +78,8 @@ function archive_store:append(username, key, value, when, with)
 	end
 	if a[key] then
 		table.remove(a, a[key]);
+	elseif #a >= archive_item_limit then
+		return nil, "quota-limit";
 	end
 	local i = #a+1;
 	a[i] = v;
@@ -80,9 +90,17 @@ end
 function archive_store:find(username, query)
 	local items = self.store[username or NULL];
 	if not items then
-		return function () end, 0;
+		if query then
+			if query.before or query.after then
+				return nil, "item-not-found";
+			end
+			if query.total then
+				return function () end, 0;
+			end
+		end
+		return function () end;
 	end
-	local count = #items;
+	local count = nil;
 	local i = 0;
 	if query then
 		items = array():append(items);
@@ -106,24 +124,36 @@ function archive_store:find(username, query)
 				return item.when <= query["end"];
 			end);
 		end
-		count = #items;
+		if query.total then
+			count = #items;
+		end
 		if query.reverse then
 			items:reverse();
 			if query.before then
-				for j = 1, count do
+				local found = false;
+				for j = 1, #items do
 					if (items[j].key or tostring(j)) == query.before then
+						found = true;
 						i = j;
 						break;
 					end
 				end
+				if not found then
+					return nil, "item-not-found";
+				end
 			end
 		elseif query.after then
-			for j = 1, count do
+			local found = false;
+			for j = 1, #items do
 				if (items[j].key or tostring(j)) == query.after then
+					found = true;
 					i = j;
 					break;
 				end
 			end
+			if not found then
+				return nil, "item-not-found";
+			end
 		end
 		if query.limit and #items - i > query.limit then
 			items[i+query.limit+1] = nil;
@@ -137,6 +167,26 @@ function archive_store:find(username, query)
 	end, count;
 end
 
+function archive_store:summary(username, query)
+	local iter, err = self:find(username, query)
+	if not iter then return iter, err; end
+	local counts = {};
+	local earliest = {};
+	local latest = {};
+	for _, _, when, with in iter do
+		counts[with] = (counts[with] or 0) + 1;
+		if earliest[with] == nil then
+			earliest[with] = when;
+		end
+		latest[with] = when;
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+	};
+end
+
 
 function archive_store:delete(username, query)
 	if not query or next(query) == nil then
diff --git a/plugins/mod_storage_sql.lua b/plugins/mod_storage_sql.lua
index a449091e..666cee41 100644
--- a/plugins/mod_storage_sql.lua
+++ b/plugins/mod_storage_sql.lua
@@ -1,17 +1,19 @@
 
 -- luacheck: ignore 212/self
 
+local cache = require "util.cache";
 local json = require "util.json";
 local sql = require "util.sql";
 local xml_parse = require "util.xml".parse;
 local uuid = require "util.uuid";
 local resolve_relative_path = require "util.paths".resolve_relative_path;
+local jid_join = require "util.jid".join;
 
 local is_stanza = require"util.stanza".is_stanza;
 local t_concat = table.concat;
 
 local noop = function() end
-local unpack = table.unpack or unpack;
+local unpack = table.unpack or unpack; -- luacheck: ignore 113
 local function iterator(result)
 	return function(result_)
 		local row = result_();
@@ -148,7 +150,10 @@ end
 
 --- Archive store API
 
--- luacheck: ignore 512 431/user 431/store
+local archive_item_limit = module:get_option_number("storage_archive_item_limit");
+local archive_item_count_cache = cache.new(module:get_option("storage_archive_item_limit_cache_size", 1000));
+
+-- luacheck: ignore 512 431/user 431/store 431/err
 local map_store = {};
 map_store.__index = map_store;
 map_store.remove = {};
@@ -228,10 +233,41 @@ end
 local archive_store = {}
 archive_store.caps = {
 	total = true;
+	quota = archive_item_limit;
+	truncate = true;
 };
 archive_store.__index = archive_store
 function archive_store:append(username, key, value, when, with)
 	local user,store = username,self.store;
+	local cache_key = jid_join(username, host, store);
+	local item_count = archive_item_count_cache:get(cache_key);
+	if not item_count then
+		local ok, ret = engine:transaction(function()
+			local count_sql = [[
+			SELECT COUNT(*) FROM "prosodyarchive"
+			WHERE "host"=? AND "user"=? AND "store"=?;
+			]];
+			local result = engine:select(count_sql, host, user, store);
+			if result then
+				for row in result do
+					item_count = row[1];
+				end
+			end
+		end);
+		if not ok or not item_count then
+			module:log("error", "Failed while checking quota for %s: %s", username, ret);
+			return nil, "Failure while checking quota";
+		end
+		archive_item_count_cache:set(cache_key, item_count);
+	end
+
+	if archive_item_limit then
+		module:log("debug", "%s has %d items out of %d limit", username, item_count, archive_item_limit);
+		if item_count >= archive_item_limit then
+			return nil, "quota-limit";
+		end
+	end
+
 	when = when or os.time();
 	with = with or "";
 	local ok, ret = engine:transaction(function()
@@ -245,12 +281,16 @@ function archive_store:append(username, key, value, when, with)
 		VALUES (?,?,?,?,?,?,?,?);
 		]];
 		if key then
-			engine:delete(delete_sql, host, user or "", store, key);
+			local result, err = engine:delete(delete_sql, host, user or "", store, key);
+			if result then
+				item_count = item_count - result:affected();
+			end
 		else
 			key = uuid.generate();
 		end
 		local t, encoded_value = assert(serialize(value));
 		engine:insert(insert_sql, host, user or "", store, when, with, key, t, encoded_value);
+		archive_item_count_cache:set(cache_key, item_count+1);
 		return key;
 	end);
 	if not ok then return ok, ret; end
@@ -287,45 +327,47 @@ local function archive_where(query, args, where)
 	end
 end
 local function archive_where_id_range(query, args, where)
-	local args_len = #args
 	-- Before or after specific item, exclusive
+	local id_lookup_sql = [[
+	SELECT "sort_id"
+	FROM "prosodyarchive"
+	WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
+	LIMIT 1;
+	]];
 	if query.after then  -- keys better be unique!
-		where[#where+1] = [[
-		"sort_id" > COALESCE(
-			(
-				SELECT "sort_id"
-				FROM "prosodyarchive"
-				WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
-				LIMIT 1
-			), 0)
-		]];
-		args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.after, args[1], args[2], args[3];
-		args_len = args_len + 4
+		local after_id = nil;
+		for row in engine:select(id_lookup_sql, query.after, args[1], args[2], args[3]) do
+			after_id = row[1];
+		end
+		if not after_id then
+			return nil, "item-not-found";
+		end
+		where[#where+1] = '"sort_id" > ?';
+		args[#args+1] = after_id;
 	end
 	if query.before then
-		where[#where+1] = [[
-		"sort_id" < COALESCE(
-			(
-				SELECT "sort_id"
-				FROM "prosodyarchive"
-				WHERE "key" = ? AND "host" = ? AND "user" = ? AND "store" = ?
-				LIMIT 1
-			),
-			(
-				SELECT MAX("sort_id")+1
-				FROM "prosodyarchive"
-			)
-		)
-		]]
-		args[args_len+1], args[args_len+2], args[args_len+3], args[args_len+4] = query.before, args[1], args[2], args[3];
+		local before_id = nil;
+		for row in engine:select(id_lookup_sql, query.after, args[1], args[2], args[3]) do
+			before_id = row[1];
+		end
+		if not before_id then
+			return nil, "item-not-found";
+		end
+		where[#where+1] = '"sort_id" < ?';
+		args[#args+1] = before_id;
 	end
+	return true;
 end
 
 function archive_store:find(username, query)
 	query = query or {};
 	local user,store = username,self.store;
-	local total;
-	local ok, result = engine:transaction(function()
+	local cache_key = jid_join(username, host, self.store);
+	local total = archive_item_count_cache:get(cache_key);
+	if total ~= nil and query.limit == 0 and query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then
+		return noop, total;
+	end
+	local ok, result, err = engine:transaction(function()
 		local sql_query = [[
 		SELECT "key", "type", "value", "when", "with"
 		FROM "prosodyarchive"
@@ -346,12 +388,16 @@ function archive_store:find(username, query)
 					total = row[1];
 				end
 			end
+			if query.start == nil and query.with == nil and query["end"] == nil and query.key == nil then
+				archive_item_count_cache:set(cache_key, total);
+			end
 			if query.limit == 0 then -- Skip the real query
 				return noop, total;
 			end
 		end
 
-		archive_where_id_range(query, args, where);
+		local ok, err = archive_where_id_range(query, args, where);
+		if not ok then return ok, err; end
 
 		if query.limit then
 			args[#args+1] = query.limit;
@@ -361,7 +407,8 @@ function archive_store:find(username, query)
 			and "DESC" or "ASC", query.limit and " LIMIT ?" or "");
 		return engine:select(sql_query, unpack(args));
 	end);
-	if not ok then return ok, result end
+	if not ok then return ok, result; end
+	if not result then return nil, err; end
 	return function()
 		local row = result();
 		if row ~= nil then
@@ -372,6 +419,48 @@ function archive_store:find(username, query)
 	end, total;
 end
 
+function archive_store:summary(username, query)
+	query = query or {};
+	local user,store = username,self.store;
+	local ok, result = engine:transaction(function()
+		local sql_query = [[
+		SELECT DISTINCT "with", COUNT(*), MIN("when"), MAX("when")
+		FROM "prosodyarchive"
+		WHERE %s
+		GROUP BY "with"
+		ORDER BY "sort_id" %s%s;
+		]];
+		local args = { host, user or "", store, };
+		local where = { "\"host\" = ?", "\"user\" = ?", "\"store\" = ?", };
+
+		archive_where(query, args, where);
+
+		archive_where_id_range(query, args, where);
+
+		if query.limit then
+			args[#args+1] = query.limit;
+		end
+
+		sql_query = sql_query:format(t_concat(where, " AND "), query.reverse
+			and "DESC" or "ASC", query.limit and " LIMIT ?" or "");
+		return engine:select(sql_query, unpack(args));
+	end);
+	if not ok then return ok, result end
+	local counts = {};
+	local earliest, latest = {}, {};
+	for row in result do
+		local with, count = row[1], row[2];
+		counts[with] = count;
+		earliest[with] = row[3];
+		latest[with] = row[4];
+	end
+	return {
+		counts = counts;
+		earliest = earliest;
+		latest = latest;
+	};
+end
+
 function archive_store:delete(username, query)
 	query = query or {};
 	local user,store = username,self.store;
@@ -384,7 +473,8 @@ function archive_store:delete(username, query)
 			table.remove(where, 2);
 		end
 		archive_where(query, args, where);
-		archive_where_id_range(query, args, where);
+		local ok, err = archive_where_id_range(query, args, where);
+		if not ok then return ok, err; end
 		if query.truncate == nil then
 			sql_query = sql_query:format(t_concat(where, " AND "));
 		else
@@ -423,9 +513,24 @@ function archive_store:delete(username, query)
 		end
 		return engine:delete(sql_query, unpack(args));
 	end);
+	local cache_key = jid_join(username, host, self.store);
+	archive_item_count_cache:set(cache_key, nil);
 	return ok and stmt:affected(), stmt;
 end
 
+function archive_store:users()
+	local ok, result = engine:transaction(function()
+		local select_sql = [[
+		SELECT DISTINCT "user"
+		FROM "prosodyarchive"
+		WHERE "host"=? AND "store"=?;
+		]];
+		return engine:select(select_sql, host, self.store);
+	end);
+	if not ok then error(result); end
+	return iterator(result);
+end
+
 local stores = {
 	keyval = keyval_store;
 	map = map_store;
diff --git a/plugins/mod_tls.lua b/plugins/mod_tls.lua
index eb208e28..b16acd09 100644
--- a/plugins/mod_tls.lua
+++ b/plugins/mod_tls.lua
@@ -35,9 +35,10 @@ local host = hosts[module.host];
 
 local ssl_ctx_c2s, ssl_ctx_s2sout, ssl_ctx_s2sin;
 local ssl_cfg_c2s, ssl_cfg_s2sout, ssl_cfg_s2sin;
+local err_c2s, err_s2sin, err_s2sout;
 
 function module.load()
-	local NULL, err = {};
+	local NULL = {};
 	local modhost = module.host;
 	local parent = modhost:match("%.(.*)$");
 
@@ -53,16 +54,20 @@ function module.load()
 	local host_s2s   = rawgetopt(modhost, "s2s_ssl") or parent_s2s;
 
 	module:log("debug", "Creating context for c2s");
-	ssl_ctx_c2s, err, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
-	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err); end
+	local request_client_certs = { verify = { "peer", "client_once", }; };
 
 	module:log("debug", "Creating context for s2sout");
-	ssl_ctx_s2sout, err, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s); -- for outgoing server connections
-	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err); end
+	ssl_ctx_c2s, err_c2s, ssl_cfg_c2s = create_context(host.host, "server", host_c2s, host_ssl, global_c2s); -- for incoming client connections
+	if not ssl_ctx_c2s then module:log("error", "Error creating context for c2s: %s", err_c2s); end
 
 	module:log("debug", "Creating context for s2sin");
-	ssl_ctx_s2sin, err, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s); -- for incoming server connections
-	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err); end
+	-- for outgoing server connections
+	ssl_ctx_s2sout, err_s2sout, ssl_cfg_s2sout = create_context(host.host, "client", host_s2s, host_ssl, global_s2s, request_client_certs);
+	if not ssl_ctx_s2sout then module:log("error", "Error creating contexts for s2sout: %s", err_s2sout); end
+
+	-- for incoming server connections
+	ssl_ctx_s2sin, err_s2sin, ssl_cfg_s2sin = create_context(host.host, "server", host_s2s, host_ssl, global_s2s, request_client_certs);
+	if not ssl_ctx_s2sin then module:log("error", "Error creating contexts for s2sin: %s", err_s2sin); end
 end
 
 module:hook_global("config-reloaded", module.load);
@@ -77,12 +82,21 @@ local function can_do_tls(session)
 		return session.ssl_ctx;
 	end
 	if session.type == "c2s_unauthed" then
+		if not ssl_ctx_c2s and c2s_require_encryption then
+			session.log("error", "No TLS context available for c2s. Earlier error was: %s", err_c2s);
+		end
 		session.ssl_ctx = ssl_ctx_c2s;
 		session.ssl_cfg = ssl_cfg_c2s;
 	elseif session.type == "s2sin_unauthed" and allow_s2s_tls then
+		if not ssl_ctx_s2sin and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sin. Earlier error was: %s", err_s2sin);
+		end
 		session.ssl_ctx = ssl_ctx_s2sin;
 		session.ssl_cfg = ssl_cfg_s2sin;
 	elseif session.direction == "outgoing" and allow_s2s_tls then
+		if not ssl_ctx_s2sout and s2s_require_encryption then
+			session.log("error", "No TLS context available for s2sout. Earlier error was: %s", err_s2sout);
+		end
 		session.ssl_ctx = ssl_ctx_s2sout;
 		session.ssl_cfg = ssl_cfg_s2sout;
 	else
diff --git a/plugins/mod_user_account_management.lua b/plugins/mod_user_account_management.lua
index 615c1ed6..130ed089 100644
--- a/plugins/mod_user_account_management.lua
+++ b/plugins/mod_user_account_management.lua
@@ -53,9 +53,10 @@ local function handle_registration_stanza(event)
 			log("info", "User removed their account: %s@%s", username, host);
 			module:fire_event("user-deregistered", { username = username, host = host, source = "mod_register", session = session });
 		else
-			local username = nodeprep(query:get_child_text("username"));
+			local username = query:get_child_text("username");
 			local password = query:get_child_text("password");
 			if username and password then
+				username = nodeprep(username);
 				if username == session.username then
 					if usermanager_set_password(username, password, session.host, session.resource) then
 						session.send(st.reply(stanza));
diff --git a/plugins/mod_vcard_legacy.lua b/plugins/mod_vcard_legacy.lua
index 5e75947a..6bae3ee5 100644
--- a/plugins/mod_vcard_legacy.lua
+++ b/plugins/mod_vcard_legacy.lua
@@ -105,6 +105,23 @@ module:hook("iq-get/bare/vcard-temp:vCard", function (event)
 					vcard_temp:tag("WORK"):up();
 				end
 				vcard_temp:up();
+			elseif tag.name == "impp" then
+				local uri = tag:get_child_text("uri");
+				if uri and uri:sub(1, 5) == "xmpp:" then
+					vcard_temp:text_tag("JABBERID", uri:sub(6))
+				end
+			elseif tag.name == "org" then
+				vcard_temp:tag("ORG")
+					:text_tag("ORGNAME", tag:get_child_text("text"))
+				:up();
+			end
+		end
+	else
+		local ok, _, nick_item = pep_service:get_last_item("http://jabber.org/protocol/nick", stanza.attr.from);
+		if ok and nick_item then
+			local nickname = nick_item:get_child_text("nick", "http://jabber.org/protocol/nick");
+			if nickname then
+				vcard_temp:text_tag("NICKNAME", nickname);
 			end
 		end
 	end
@@ -216,6 +233,10 @@ function vcard_to_pep(vcard_temp)
 				vcard4:text_tag("text", "work");
 			end
 			vcard4:up():up():up();
+		elseif tag.name == "JABBERID" then
+			vcard4:tag("impp")
+				:text_tag("uri", "xmpp:" .. tag:get_text())
+			:up();
 		elseif tag.name == "PHOTO" then
 			local avatar_type = tag:get_child_text("TYPE");
 			local avatar_payload = tag:get_child_text("BINVAL");
diff --git a/plugins/mod_websocket.lua b/plugins/mod_websocket.lua
index 686a8981..386a4d60 100644
--- a/plugins/mod_websocket.lua
+++ b/plugins/mod_websocket.lua
@@ -29,18 +29,10 @@ local t_concat = table.concat;
 
 local stream_close_timeout = module:get_option_number("c2s_close_timeout", 5);
 local consider_websocket_secure = module:get_option_boolean("consider_websocket_secure");
-local cross_domain = module:get_option_set("cross_domain_websocket", {});
-if cross_domain:contains("*") or cross_domain:contains(true) then
-	cross_domain = true;
+local cross_domain = module:get_option("cross_domain_websocket");
+if cross_domain ~= nil then
+	module:log("info", "The 'cross_domain_websocket' option has been deprecated");
 end
-
-local function check_origin(origin)
-	if cross_domain == true then
-		return true;
-	end
-	return cross_domain:contains(origin);
-end
-
 local xmlns_framing = "urn:ietf:params:xml:ns:xmpp-framing";
 local xmlns_streams = "http://etherx.jabber.org/streams";
 local xmlns_client = "jabber:client";
@@ -88,7 +80,7 @@ local function session_close(session, reason)
 					stream_error = reason;
 				end
 			end
-			log("debug", "Disconnecting client, <stream:error> is: %s", tostring(stream_error));
+			log("debug", "Disconnecting client, <stream:error> is: %s", stream_error);
 			session.send(stream_error);
 		end
 
@@ -144,7 +136,7 @@ function handle_request(event)
 
 	conn.starttls = false; -- Prevent mod_tls from believing starttls can be done
 
-	if not request.headers.sec_websocket_key then
+	if not request.headers.sec_websocket_key or request.method ~= "GET" then
 		response.headers.content_type = "text/html";
 		return [[<!DOCTYPE html><html><head><title>Websocket</title></head><body>
 			<p>It works! Now point your WebSocket client to this URL to connect to Prosody.</p>
@@ -158,11 +150,6 @@ function handle_request(event)
 		return 501;
 	end
 
-	if not check_origin(request.headers.origin or "") then
-		module:log("debug", "Origin %s is not allowed by 'cross_domain_websocket' [ %s ]", request.headers.origin or "(missing header)", cross_domain);
-		return 403;
-	end
-
 	local function websocket_close(code, message)
 		conn:write(build_close(code, message));
 		conn:close();
@@ -330,27 +317,4 @@ module:provides("http", {
 
 function module.add_host(module)
 	module:hook("c2s-read-timeout", keepalive, -0.9);
-
-	if cross_domain ~= true then
-		local url = require "socket.url";
-		local ws_url = module:http_url("websocket", "xmpp-websocket");
-		local url_components = url.parse(ws_url);
-		-- The 'Origin' consists of the base URL without path
-		url_components.path = nil;
-		local this_origin = url.build(url_components);
-		local local_cross_domain = module:get_option_set("cross_domain_websocket", { this_origin });
-		if local_cross_domain:contains(true) then
-			module:log("error", "cross_domain_websocket = true only works in the global section");
-			return;
-		end
-
-		-- Don't add / remove something added by another host
-		-- This might be weird with random load order
-		local_cross_domain:exclude(cross_domain);
-		cross_domain:include(local_cross_domain);
-		module:log("debug", "cross_domain = %s", tostring(cross_domain));
-		function module.unload()
-			cross_domain:exclude(local_cross_domain);
-		end
-	end
 end
diff --git a/plugins/muc/history.lib.lua b/plugins/muc/history.lib.lua
index 0d69c97d..f9ddabbf 100644
--- a/plugins/muc/history.lib.lua
+++ b/plugins/muc/history.lib.lua
@@ -48,16 +48,18 @@ module:hook("muc-config-form", function(event)
 	table.insert(event.form, {
 		name = "muc#roomconfig_historylength";
 		type = "text-single";
+		datatype = "xs:integer";
 		label = "Maximum number of history messages returned by room";
 		desc = "Specify the maximum number of previous messages that should be sent to users when they join the room";
-		value = tostring(get_historylength(event.room));
+		value = get_historylength(event.room);
 	});
 	table.insert(event.form, {
 		name = 'muc#roomconfig_defaulthistorymessages',
 		type = 'text-single',
+		datatype = "xs:integer";
 		label = 'Default number of history messages returned by room',
 		desc = "Specify the number of previous messages sent to new users when they join the room";
-		value = tostring(get_defaulthistorymessages(event.room))
+		value = get_defaulthistorymessages(event.room);
 	});
 end, 70-5);
 
diff --git a/plugins/muc/language.lib.lua b/plugins/muc/language.lib.lua
index ee80806b..2ee2ba0f 100644
--- a/plugins/muc/language.lib.lua
+++ b/plugins/muc/language.lib.lua
@@ -32,6 +32,7 @@ local function add_form_option(event)
 		label = "Language tag for room (e.g. 'en', 'de', 'fr' etc.)";
 		type = "text-single";
 		desc = "Indicate the primary language spoken in this room";
+		datatype = "xs:language";
 		value = get_language(event.room) or "";
 	});
 end
diff --git a/plugins/muc/mod_muc.lua b/plugins/muc/mod_muc.lua
index 954bae92..166249cc 100644
--- a/plugins/muc/mod_muc.lua
+++ b/plugins/muc/mod_muc.lua
@@ -86,7 +86,14 @@ room_mt.get_registered_nick = register.get_registered_nick;
 room_mt.get_registered_jid = register.get_registered_jid;
 room_mt.handle_register_iq = register.handle_register_iq;
 
+local presence_broadcast = module:require "muc/presence_broadcast";
+room_mt.get_presence_broadcast = presence_broadcast.get;
+room_mt.set_presence_broadcast = presence_broadcast.set;
+room_mt.get_valid_broadcast_roles = presence_broadcast.get_valid_broadcast_roles;
+
+
 local jid_split = require "util.jid".split;
+local jid_prep = require "util.jid".prep;
 local jid_bare = require "util.jid".bare;
 local st = require "util.stanza";
 local cache = require "util.cache";
@@ -263,9 +270,13 @@ local function set_room_defaults(room, lang)
 	room:set_changesubject(module:get_option_boolean("muc_room_default_change_subject", room:get_changesubject()));
 	room:set_historylength(module:get_option_number("muc_room_default_history_length", room:get_historylength()));
 	room:set_language(lang or module:get_option_string("muc_room_default_language"));
+	room:set_presence_broadcast(module:get_option("muc_room_default_presence_broadcast", room:get_presence_broadcast()));
 end
 
 function create_room(room_jid, config)
+	if jid_bare(room_jid) ~= room_jid or not jid_prep(room_jid, true) then
+		return nil, "invalid-jid";
+	end
 	local exists = get_room_from_jid(room_jid);
 	if exists then
 		return nil, "room-exists";
@@ -453,7 +464,11 @@ for event_name, method in pairs {
 
 		if room == nil then
 			-- Watch presence to create rooms
-			if stanza.attr.type == nil and stanza.name == "presence" then
+			if not jid_prep(room_jid, true) then
+				origin.send(st.error_reply(stanza, "modify", "jid-malformed"));
+				return true;
+			end
+			if stanza.attr.type == nil and stanza.name == "presence" and stanza:get_child("x", "http://jabber.org/protocol/muc") then
 				room = muclib.new_room(room_jid);
 				return room:handle_first_presence(origin, stanza);
 			elseif stanza.attr.type ~= "error" then
diff --git a/plugins/muc/muc.lib.lua b/plugins/muc/muc.lib.lua
index 639ecc38..4b462631 100644
--- a/plugins/muc/muc.lib.lua
+++ b/plugins/muc/muc.lib.lua
@@ -23,6 +23,7 @@ local resourceprep = require "util.encodings".stringprep.resourceprep;
 local st = require "util.stanza";
 local base64 = require "util.encodings".base64;
 local md5 = require "util.hashes".md5;
+local new_id = require "util.id".medium;
 
 local log = module._log;
 
@@ -39,7 +40,7 @@ function room_mt:__tostring()
 end
 
 function room_mt.save()
-	-- overriden by mod_muc.lua
+	-- overridden by mod_muc.lua
 end
 
 function room_mt:get_occupant_jid(real_jid)
@@ -217,13 +218,13 @@ end
 
 -- Broadcasts an occupant's presence to the whole room
 -- Takes the x element that goes into the stanzas
-function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
+function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason, prev_role, force_unavailable)
 	local base_x = x.base or x;
 	-- Build real jid and (optionally) occupant jid template presences
 	local base_presence do
 		-- Try to use main jid's presence
 		local pr = occupant:get_presence();
-		if pr and (occupant.role ~= nil or pr.attr.type == "unavailable") then
+		if pr and (occupant.role ~= nil or pr.attr.type == "unavailable") and not force_unavailable then
 			base_presence = st.clone(pr);
 		else -- user is leaving but didn't send a leave presence. make one for them
 			base_presence = st.presence {from = occupant.nick; type = "unavailable";};
@@ -279,7 +280,9 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 		self_p = st.clone(base_presence):add_child(self_x);
 	end
 
-	-- General populance
+	local broadcast_roles = self:get_presence_broadcast();
+
+	-- General populace
 	for occupant_nick, n_occupant in self:each_occupant() do
 		if occupant_nick ~= occupant.nick then
 			local pr;
@@ -290,7 +293,13 @@ function room_mt:publicise_occupant_status(occupant, x, nick, actor, reason)
 			else
 				pr = get_anon_p();
 			end
-			self:route_to_occupant(n_occupant, pr);
+			if broadcast_roles[occupant.role or "none"] or force_unavailable then
+				self:route_to_occupant(n_occupant, pr);
+			elseif prev_role and broadcast_roles[prev_role] then
+				pr.attr.type = 'unavailable';
+				self:route_to_occupant(n_occupant, pr);
+			end
+
 		end
 	end
 
@@ -314,6 +323,7 @@ function room_mt:send_occupant_list(to, filter)
 	local to_bare = jid_bare(to);
 	local is_anonymous = false;
 	local whois = self:get_whois();
+	local broadcast_roles = self:get_presence_broadcast();
 	if whois ~= "anyone" then
 		local affiliation = self:get_affiliation(to);
 		if affiliation ~= "admin" and affiliation ~= "owner" then
@@ -330,7 +340,9 @@ function room_mt:send_occupant_list(to, filter)
 			local pres = st.clone(occupant:get_presence());
 			pres.attr.to = to;
 			pres:add_child(x);
-			self:route_stanza(pres);
+			if to_bare == occupant.bare_jid or broadcast_roles[occupant.role or "none"] then
+				self:route_stanza(pres);
+			end
 		end
 	end
 end
@@ -391,7 +403,11 @@ function room_mt:handle_kickable(origin, stanza) -- luacheck: ignore 212
 	end
 	self:publicise_occupant_status(new_occupant or occupant, x);
 	if is_last_session then
-		module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 	return true;
 end
@@ -428,14 +444,23 @@ module:hook("muc-occupant-pre-change", function(event)
 	end
 end, 1);
 
-function room_mt:handle_first_presence(origin, stanza)
-	if not stanza:get_child("x", "http://jabber.org/protocol/muc") then
-		module:log("debug", "Room creation without <x>, possibly desynced");
+module:hook("muc-occupant-pre-join", function(event)
+	local nick = jid_resource(event.occupant.nick);
+	if not resourceprep(nick, true) then -- strict
+		event.origin.send(st.error_reply(event.stanza, "modify", "jid-malformed", "Nickname must pass strict validation"));
+		return true;
+	end
+end, 2);
 
-		origin.send(st.error_reply(stanza, "cancel", "item-not-found"));
+module:hook("muc-occupant-pre-change", function(event)
+	local nick = jid_resource(event.dest_occupant.nick);
+	if not resourceprep(nick, true) then -- strict
+		event.origin.send(st.error_reply(event.stanza, "modify", "jid-malformed", "Nickname must pass strict validation"));
 		return true;
 	end
+end, 2);
 
+function room_mt:handle_first_presence(origin, stanza)
 	local real_jid = stanza.attr.from;
 	local dest_jid = stanza.attr.to;
 	local bare_jid = jid_bare(real_jid);
@@ -505,7 +530,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 	if orig_occupant == nil and not muc_x and stanza.attr.type == nil then
 		module:log("debug", "Attempted join without <x>, possibly desynced");
 		origin.send(st.error_reply(stanza, "cancel", "item-not-found",
-			"You must join the room before sending presence updates"));
+			"You are not currently connected to this chat"));
 		return true;
 	end
 
@@ -613,7 +638,7 @@ function room_mt:handle_normal_presence(origin, stanza)
 				x:tag("status", {code = "303";}):up();
 				x:tag("status", {code = "110";}):up();
 				self:route_stanza(generated_unavail:add_child(x));
-				dest_nick = nil; -- set dest_nick to nil; so general populance doesn't see it for whole orig_occupant
+				dest_nick = nil; -- set dest_nick to nil; so general populace doesn't see it for whole orig_occupant
 			end
 		end
 
@@ -879,7 +904,11 @@ function room_mt:clear(x)
 	end
 	for occupant in pairs(occupants_updated) do
 		self:publicise_occupant_status(occupant, x);
-		module:fire_event("muc-occupant-left", { room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 end
 
@@ -972,7 +1001,7 @@ function room_mt:handle_admin_query_get_command(origin, stanza)
 	local _aff_rank = valid_affiliations[_aff or "none"];
 	local _rol = item.attr.role;
 	if _aff and _aff_rank and not _rol then
-		-- You need to be at least an admin, and be requesting info about your affifiliation or lower
+		-- You need to be at least an admin, and be requesting info about your affiliation or lower
 		-- e.g. an admin can't ask for a list of owners
 		local affiliation_rank = valid_affiliations[affiliation or "none"];
 		if (affiliation_rank >= valid_affiliations.admin and affiliation_rank >= _aff_rank)
@@ -1049,6 +1078,9 @@ end
 function room_mt:handle_groupchat_to_room(origin, stanza)
 	local from = stanza.attr.from;
 	local occupant = self:get_occupant_by_real_jid(from);
+	if not stanza.attr.id then
+		stanza.attr.id = new_id()
+	end
 	if module:fire_event("muc-occupant-groupchat", {
 		room = self; origin = origin; stanza = stanza; from = from; occupant = occupant;
 	}) then return true; end
@@ -1297,7 +1329,7 @@ function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 			-- Outcast can be by host.
 			is_host_only and affiliation == "outcast" and select(2, jid_split(occupant.bare_jid)) == host
 		) then
-			-- need to publcize in all cases; as affiliation in <item/> has changed.
+			-- need to publicize in all cases; as affiliation in <item/> has changed.
 			occupants_updated[occupant] = occupant.role;
 			if occupant.role ~= role and (
 				is_downgrade or
@@ -1324,7 +1356,11 @@ function room_mt:set_affiliation(actor, jid, affiliation, reason, data)
 		for occupant, old_role in pairs(occupants_updated) do
 			self:publicise_occupant_status(occupant, x, nil, actor, reason);
 			if occupant.role == nil then
-				module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+				module:fire_event("muc-occupant-left", {
+						room = self;
+						nick = occupant.nick;
+						occupant = occupant;
+					});
 			elseif is_semi_anonymous and
 				(old_role == "moderator" and occupant.role ~= "moderator") or
 				(old_role ~= "moderator" and occupant.role == "moderator") then -- Has gained or lost moderator status
@@ -1376,6 +1412,42 @@ function room_mt:get_role(nick)
 	return occupant and occupant.role or nil;
 end
 
+function room_mt:may_set_role(actor, occupant, role)
+	local event = {
+		room = self,
+		actor = actor,
+		occupant = occupant,
+		role = role,
+	};
+
+	module:fire_event("muc-pre-set-role", event);
+	if event.allowed ~= nil then
+		return event.allowed, event.error, event.condition;
+	end
+
+	-- Can't do anything to other owners or admins
+	local occupant_affiliation = self:get_affiliation(occupant.bare_jid);
+	if occupant_affiliation == "owner" or occupant_affiliation == "admin" then
+		return nil, "cancel", "not-allowed";
+	end
+
+	-- If you are trying to give or take moderator role you need to be an owner or admin
+	if occupant.role == "moderator" or role == "moderator" then
+		local actor_affiliation = self:get_affiliation(actor);
+		if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then
+			return nil, "cancel", "not-allowed";
+		end
+	end
+
+	-- Need to be in the room and a moderator
+	local actor_occupant = self:get_occupant_by_real_jid(actor);
+	if not actor_occupant or actor_occupant.role ~= "moderator" then
+		return nil, "cancel", "not-allowed";
+	end
+
+	return true;
+end
+
 function room_mt:set_role(actor, occupant_jid, role, reason)
 	if not actor then return nil, "modify", "not-acceptable"; end
 
@@ -1390,24 +1462,9 @@ function room_mt:set_role(actor, occupant_jid, role, reason)
 	if actor == true then
 		actor = nil -- So we can pass it safely to 'publicise_occupant_status' below
 	else
-		-- Can't do anything to other owners or admins
-		local occupant_affiliation = self:get_affiliation(occupant.bare_jid);
-		if occupant_affiliation == "owner" or occupant_affiliation == "admin" then
-			return nil, "cancel", "not-allowed";
-		end
-
-		-- If you are trying to give or take moderator role you need to be an owner or admin
-		if occupant.role == "moderator" or role == "moderator" then
-			local actor_affiliation = self:get_affiliation(actor);
-			if actor_affiliation ~= "owner" and actor_affiliation ~= "admin" then
-				return nil, "cancel", "not-allowed";
-			end
-		end
-
-		-- Need to be in the room and a moderator
-		local actor_occupant = self:get_occupant_by_real_jid(actor);
-		if not actor_occupant or actor_occupant.role ~= "moderator" then
-			return nil, "cancel", "not-allowed";
+		local allowed, err, condition = self:may_set_role(actor, occupant, role)
+		if not allowed then
+			return allowed, err, condition;
 		end
 	end
 
@@ -1415,11 +1472,17 @@ function room_mt:set_role(actor, occupant_jid, role, reason)
 	if not role then
 		x:tag("status", {code = "307"}):up();
 	end
+
+	local prev_role = occupant.role;
 	occupant.role = role;
 	self:save_occupant(occupant);
-	self:publicise_occupant_status(occupant, x, nil, actor, reason);
+	self:publicise_occupant_status(occupant, x, nil, actor, reason, prev_role);
 	if role == nil then
-		module:fire_event("muc-occupant-left", {room = self; nick = occupant.nick; occupant = occupant;});
+		module:fire_event("muc-occupant-left", {
+				room = self;
+				nick = occupant.nick;
+				occupant = occupant;
+			});
 	end
 	return true;
 end
diff --git a/plugins/muc/presence_broadcast.lib.lua b/plugins/muc/presence_broadcast.lib.lua
new file mode 100644
index 00000000..ace614b3
--- /dev/null
+++ b/plugins/muc/presence_broadcast.lib.lua
@@ -0,0 +1,87 @@
+-- Prosody IM
+-- Copyright (C) 2008-2010 Matthew Wild
+-- Copyright (C) 2008-2010 Waqas Hussain
+-- Copyright (C) 2014 Daurnimator
+--
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local st = require "util.stanza";
+
+local valid_roles = { "visitor", "participant", "moderator" };
+local default_broadcast = {
+	none = true;
+	visitor = true;
+	participant = true;
+	moderator = true;
+};
+
+local function get_presence_broadcast(room)
+	return room._data.presence_broadcast or default_broadcast;
+end
+
+local function set_presence_broadcast(room, broadcast_roles)
+	broadcast_roles = broadcast_roles or default_broadcast;
+
+	-- Ensure that unavailable presence is always sent when role changes to none
+	broadcast_roles.none = true;
+
+	local changed = false;
+	local old_broadcast_roles = get_presence_broadcast(room);
+	for _, role in ipairs(valid_roles) do
+		if old_broadcast_roles[role] ~= broadcast_roles[role] then
+			changed = true;
+		end
+	end
+
+	if not changed then return false; end
+
+	room._data.presence_broadcast = broadcast_roles;
+
+	for _, occupant in room:each_occupant() do
+		local x = st.stanza("x", {xmlns = "http://jabber.org/protocol/muc#user";});
+		local role = occupant.role or "none";
+		if broadcast_roles[role] and not old_broadcast_roles[role] then
+			-- Presence broadcast is now enabled, so announce existing user
+			room:publicise_occupant_status(occupant, x);
+		elseif old_broadcast_roles[role] and not broadcast_roles[role] then
+			-- Presence broadcast is now disabled, so mark existing user as unavailable
+			room:publicise_occupant_status(occupant, x, nil, nil, nil, nil, true);
+		end
+	end
+
+	return true;
+end
+
+module:hook("muc-config-form", function(event)
+	local values = {};
+	for role, value in pairs(get_presence_broadcast(event.room)) do
+		if value then
+			values[#values + 1] = role;
+		end
+	end
+
+	table.insert(event.form, {
+		name = "muc#roomconfig_presencebroadcast";
+		type = "list-multi";
+		label = "Roles for which Presence is Broadcasted";
+		value = values;
+		options = valid_roles;
+	});
+end, 70-7);
+
+module:hook("muc-config-submitted/muc#roomconfig_presencebroadcast", function(event)
+	local broadcast_roles = {};
+	for _, role in ipairs(event.value) do
+		broadcast_roles[role] = true;
+	end
+	if set_presence_broadcast(event.room, broadcast_roles) then
+		event.status_codes["104"] = true;
+	end
+end);
+
+return {
+	get = get_presence_broadcast;
+	set = set_presence_broadcast;
+};
diff --git a/plugins/muc/register.lib.lua b/plugins/muc/register.lib.lua
index 95ed1a84..bf8cd877 100644
--- a/plugins/muc/register.lib.lua
+++ b/plugins/muc/register.lib.lua
@@ -15,8 +15,7 @@ local function get_reserved_nicks(room)
 	end
 	module:log("debug", "Refreshing reserved nicks...");
 	local reserved_nicks = {};
-	for jid in room:each_affiliation() do
-		local data = room._affiliation_data[jid];
+	for jid, _, data in room:each_affiliation() do
 		local nick = data and data.reserved_nickname;
 		module:log("debug", "Refreshed for %s: %s", jid, nick);
 		if nick then
@@ -54,7 +53,7 @@ end);
 
 local registration_form = dataforms.new {
 	{ name = "FORM_TYPE", type = "hidden", value = "http://jabber.org/protocol/muc#register" },
-	{ name = "muc#register_roomnick", type = "text-single", label = "Nickname"},
+	{ name = "muc#register_roomnick", type = "text-single", required = true, label = "Nickname"},
 };
 
 local function enforce_nick_policy(event)
@@ -135,7 +134,19 @@ local function handle_register_iq(room, origin, stanza)
 			return true;
 		end
 		local form_tag = query:get_child("x", "jabber:x:data");
-		local reg_data = form_tag and registration_form:data(form_tag);
+		if not form_tag then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Missing dataform"));
+			return true;
+		end
+		local form_type, err = dataforms.get_type(form_tag);
+		if not form_type then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error with form: "..err));
+			return true;
+		elseif form_type ~= "http://jabber.org/protocol/muc#register" then
+			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error in form"));
+			return true;
+		end
+		local reg_data = registration_form:data(form_tag);
 		if not reg_data then
 			origin.send(st.error_reply(stanza, "modify", "bad-request", "Error in form"));
 			return true;
diff --git a/plugins/muc/subject.lib.lua b/plugins/muc/subject.lib.lua
index 938abf61..c8b99cc7 100644
--- a/plugins/muc/subject.lib.lua
+++ b/plugins/muc/subject.lib.lua
@@ -94,6 +94,12 @@ module:hook("muc-occupant-groupchat", function(event)
 	local stanza = event.stanza;
 	local subject = stanza:get_child("subject");
 	if subject then
+		if stanza:get_child("body") or stanza:get_child("thread") then
+			-- Note: A message with a <subject/> and a <body/> or a <subject/> and
+			-- a <thread/> is a legitimate message, but it SHALL NOT be interpreted
+			-- as a subject change.
+			return;
+		end
 		local room = event.room;
 		local occupant = event.occupant;
 		-- Role check for subject changes