Merge with trunk

30 files changed, 559 insertions, 388 deletions

diff --git a/configure b/configure
index af046223..dd6af49c 100755
--- a/configure
+++ b/configure
@@ -112,7 +112,7 @@ do
         CFLAGS="-Wall -fPIC"
         LDFLAGS="-shared"
         fi
-        if [ "$OSTYPE" = "freebsd" ]
+        if [ "$OSTYPE" = "freebsd" -o "$OSTYPE" = "openbsd" ]
         then LUA_INCDIR="/usr/local/include/lua51"
         LUA_INCDIR_SET=yes
         CFLAGS="-Wall -fPIC -I/usr/local/include"
@@ -122,6 +122,9 @@ do
         LUA_DIR=/usr/local
         LUA_DIR_SET=yes
         fi
+        if [ "$OSTYPE" = "openbsd" ]
+        then LUA_INCDIR="/usr/local/include";
+        fi
       ;;
    --datadir=*)
    	DATADIR="$value"
diff --git a/core/hostmanager.lua b/core/hostmanager.lua
index 9e74cd6b..0dd1d426 100644
--- a/core/hostmanager.lua
+++ b/core/hostmanager.lua
@@ -53,6 +53,17 @@ end
 
 prosody_events.add_handler("server-starting", load_enabled_hosts);
 
+local function host_send(stanza)
+	local name, type = stanza.name, stanza.attr.type;
+	if type == "error" or (name == "iq" and type == "result") then
+		local dest_host_name = select(2, jid_split(stanza.attr.to));
+		local dest_host = hosts[dest_host_name] or { type = "unknown" };
+		log("warn", "Unhandled response sent to %s host %s: %s", dest_host.type, dest_host_name, tostring(stanza));
+		return;
+	end
+	core_route_stanza(nil, stanza);
+end
+
 function activate(host, host_config)
 	if hosts[host] then return nil, "The host "..host.." is already activated"; end
 	host_config = host_config or configmanager.getconfig()[host];
@@ -63,6 +74,7 @@ function activate(host, host_config)
 		events = events_new();
 		dialback_secret = configmanager.get(host, "core", "dialback_secret") or uuid_gen();
 		disallow_s2s = configmanager.get(host, "core", "disallow_s2s");
+		send = host_send;
 	};
 	if not host_config.core.component_module then -- host
 		host_session.type = "local";
diff --git a/core/moduleapi.lua b/core/moduleapi.lua
new file mode 100644
index 00000000..7a4d1fa6
--- /dev/null
+++ b/core/moduleapi.lua
@@ -0,0 +1,294 @@
+-- Prosody IM
+-- Copyright (C) 2008-2012 Matthew Wild
+-- Copyright (C) 2008-2012 Waqas Hussain
+-- 
+-- This project is MIT/X11 licensed. Please see the
+-- COPYING file in the source package for more information.
+--
+
+local config = require "core.configmanager";
+local modulemanager = require "modulemanager";
+local array = require "util.array";
+local set = require "util.set";
+local logger = require "util.logger";
+local pluginloader = require "util.pluginloader";
+
+local multitable_new = require "util.multitable".new;
+
+local t_insert, t_remove, t_concat = table.insert, table.remove, table.concat;
+local error, setmetatable, setfenv, type = error, setmetatable, setfenv, type;
+local ipairs, pairs, select, unpack = ipairs, pairs, select, unpack;
+local tonumber, tostring = tonumber, tostring;
+
+local prosody = prosody;
+local hosts = prosody.hosts;
+
+-- Registry of shared module data
+local shared_data = setmetatable({}, { __mode = "v" });
+
+local NULL = {};
+
+local api = {};
+
+-- Returns the name of the current module
+function api:get_name()
+	return self.name;
+end
+
+-- Returns the host that the current module is serving
+function api:get_host()
+	return self.host;
+end
+
+function api:get_host_type()
+	return hosts[self.host].type;
+end
+
+function api:set_global()
+	self.host = "*";
+	-- Update the logger
+	local _log = logger.init("mod_"..self.name);
+	self.log = function (self, ...) return _log(...); end;
+	self._log = _log;
+end
+
+function api:add_feature(xmlns)
+	self:add_item("feature", xmlns);
+end
+function api:add_identity(category, type, name)
+	self:add_item("identity", {category = category, type = type, name = name});
+end
+function api:add_extension(data)
+	self:add_item("extension", data);
+end
+
+function api:fire_event(...)
+	return (hosts[self.host] or prosody).events.fire_event(...);
+end
+
+function api:hook_object_event(object, event, handler, priority)
+	self.event_handlers[handler] = { name = event, priority = priority, object = object };
+	return object.add_handler(event, handler, priority);
+end
+
+function api:hook(event, handler, priority)
+	return self:hook_object_event((hosts[self.host] or prosody).events, event, handler, priority);
+end
+
+function api:hook_global(event, handler, priority)
+	return self:hook_object_event(prosody.events, event, handler, priority);
+end
+
+function api:hook_stanza(xmlns, name, handler, priority)
+	if not handler and type(name) == "function" then
+		-- If only 2 options then they specified no xmlns
+		xmlns, name, handler, priority = nil, xmlns, name, handler;
+	elseif not (handler and name) then
+		self:log("warn", "Error: Insufficient parameters to module:hook_stanza()");
+		return;
+	end
+	return self:hook("stanza/"..(xmlns and (xmlns..":") or "")..name, function (data) return handler(data.origin, data.stanza, data); end, priority);
+end
+
+function api:require(lib)
+	local f, n = pluginloader.load_code(self.name, lib..".lib.lua");
+	if not f then
+		f, n = pluginloader.load_code(lib, lib..".lib.lua");
+	end
+	if not f then error("Failed to load plugin library '"..lib.."', error: "..n); end -- FIXME better error message
+	setfenv(f, self.environment);
+	return f();
+end
+
+function api:depends(name)
+	if not self.dependencies then
+		self.dependencies = {};
+		self:hook("module-reloaded", function (event)
+			if self.dependencies[event.module] then
+				self:log("info", "Auto-reloading due to reload of %s:%s", event.host, event.module);
+				modulemanager.reload(self.host, self.name);
+				return;
+			end
+		end);
+		self:hook("module-unloaded", function (event)
+			if self.dependencies[event.module] then
+				self:log("info", "Auto-unloading due to unload of %s:%s", event.host, event.module);
+				modulemanager.unload(self.host, self.name);
+			end
+		end);
+	end
+	local mod = modulemanager.get_module(self.host, name) or modulemanager.get_module("*", name);
+	if not mod then
+		local err;
+		mod, err = modulemanager.load(self.host, name);
+		if not mod then
+			return error(("Unable to load required module, mod_%s: %s"):format(name, ((err or "unknown error"):gsub("%-", " ")) ));
+		end
+	end
+	self.dependencies[name] = true;
+	return mod;
+end
+
+-- Returns one or more shared tables at the specified virtual paths
+-- Intentionally does not allow the table at a path to be _set_, it
+-- is auto-created if it does not exist.
+function api:shared(...)
+	local paths = { n = select("#", ...), ... };
+	local data_array = {};
+	local default_path_components = { self.host, self.name };
+	for i = 1, paths.n do
+		local path = paths[i];
+		if path:sub(1,1) ~= "/" then -- Prepend default components
+			local n_components = select(2, path:gsub("/", "%1"));
+			path = (n_components<#default_path_components and "/" or "")..t_concat(default_path_components, "/", 1, #default_path_components-n_components).."/"..path;
+		end
+		local shared = shared_data[path];
+		if not shared then
+			shared = {};
+			shared_data[path] = shared;
+		end
+		t_insert(data_array, shared);
+	end
+	return unpack(data_array);
+end
+
+function api:get_option(name, default_value)
+	local value = config.get(self.host, self.name, name);
+	if value == nil then
+		value = config.get(self.host, "core", name);
+		if value == nil then
+			value = default_value;
+		end
+	end
+	return value;
+end
+
+function api:get_option_string(name, default_value)
+	local value = self:get_option(name, default_value);
+	if type(value) == "table" then
+		if #value > 1 then
+			self:log("error", "Config option '%s' does not take a list, using just the first item", name);
+		end
+		value = value[1];
+	end
+	if value == nil then
+		return nil;
+	end
+	return tostring(value);
+end
+
+function api:get_option_number(name, ...)
+	local value = self:get_option(name, ...);
+	if type(value) == "table" then
+		if #value > 1 then
+			self:log("error", "Config option '%s' does not take a list, using just the first item", name);
+		end
+		value = value[1];
+	end
+	local ret = tonumber(value);
+	if value ~= nil and ret == nil then
+		self:log("error", "Config option '%s' not understood, expecting a number", name);
+	end
+	return ret;
+end
+
+function api:get_option_boolean(name, ...)
+	local value = self:get_option(name, ...);
+	if type(value) == "table" then
+		if #value > 1 then
+			self:log("error", "Config option '%s' does not take a list, using just the first item", name);
+		end
+		value = value[1];
+	end
+	if value == nil then
+		return nil;
+	end
+	local ret = value == true or value == "true" or value == 1 or nil;
+	if ret == nil then
+		ret = (value == false or value == "false" or value == 0);
+		if ret then
+			ret = false;
+		else
+			ret = nil;
+		end
+	end
+	if ret == nil then
+		self:log("error", "Config option '%s' not understood, expecting true/false", name);
+	end
+	return ret;
+end
+
+function api:get_option_array(name, ...)
+	local value = self:get_option(name, ...);
+
+	if value == nil then
+		return nil;
+	end
+	
+	if type(value) ~= "table" then
+		return array{ value }; -- Assume any non-list is a single-item list
+	end
+	
+	return array():append(value); -- Clone
+end
+
+function api:get_option_set(name, ...)
+	local value = self:get_option_array(name, ...);
+	
+	if value == nil then
+		return nil;
+	end
+	
+	return set.new(value);
+end
+
+local module_items = multitable_new();
+function api:add_item(key, value)
+	self.items = self.items or {};
+	self.items[key] = self.items[key] or {};
+	t_insert(self.items[key], value);
+	self:fire_event("item-added/"..key, {source = self, item = value});
+end
+function api:remove_item(key, value)
+	local t = self.items and self.items[key] or NULL;
+	for i = #t,1,-1 do
+		if t[i] == value then
+			t_remove(self.items[key], i);
+			self:fire_event("item-removed/"..key, {source = self, item = value});
+			return value;
+		end
+	end
+end
+
+function api:get_host_items(key)
+	local result = {};
+	for mod_name, module in pairs(modulemanager.get_modules(self.host)) do
+		module = module.module;
+		if module.items then
+			for _, item in ipairs(module.items[key] or NULL) do
+				t_insert(result, item);
+			end
+		end
+	end
+	for mod_name, module in pairs(modulemanager.get_modules("*")) do
+		module = module.module;
+		if module.items then
+			for _, item in ipairs(module.items[key] or NULL) do
+				t_insert(result, item);
+			end
+		end
+	end
+	return result;
+end
+
+function api:handle_items(type, added_cb, removed_cb, existing)
+	self:hook("item-added/"..type, added_cb);
+	self:hook("item-removed/"..type, removed_cb);
+	if existing ~= false then
+		for _, item in ipairs(self:get_host_items(type)) do
+			added_cb({ item = item });
+		end
+	end
+end
+
+return api;
diff --git a/core/modulemanager.lua b/core/modulemanager.lua
index 2d1eeb77..a192e637 100644
--- a/core/modulemanager.lua
+++ b/core/modulemanager.lua
@@ -9,13 +9,10 @@
 local logger = require "util.logger";
 local log = logger.init("modulemanager");
 local config = require "core.configmanager";
-local multitable_new = require "util.multitable".new;
-local st = require "util.stanza";
 local pluginloader = require "util.pluginloader";
 
 local hosts = hosts;
 local prosody = prosody;
-local prosody_events = prosody.events;
 
 local loadfile, pcall, xpcall = loadfile, pcall, xpcall;
 local setmetatable, setfenv, getfenv = setmetatable, setfenv, getfenv;
@@ -45,15 +42,11 @@ local _G = _G;
 
 module "modulemanager"
 
-api = {};
-local api = api; -- Module API container
+local api = _G.require "core.moduleapi"; -- Module API container
 
+-- [host] = { [module] = module_env }
 local modulemap = { ["*"] = {} };
 
-local modulehelpers = setmetatable({}, { __index = _G });
-
-local hooks = multitable_new();
-
 local NULL = {};
 
 -- Load modules when a host is activated
@@ -88,10 +81,39 @@ function load_modules_for_host(host)
 		load(host, module);
 	end
 end
-prosody_events.add_handler("host-activated", load_modules_for_host);
---
+prosody.events.add_handler("host-activated", load_modules_for_host);
 
-function load(host, module_name, config)
+--- Private helpers ---
+
+local function do_unload_module(host, name)
+	local mod = get_module(host, name);
+	if not mod then return nil, "module-not-loaded"; end
+	
+	if module_has_method(mod, "unload") then
+		local ok, err = call_module_method(mod, "unload");
+		if (not ok) and err then
+			log("warn", "Non-fatal error unloading module '%s' on '%s': %s", name, host, err);
+		end
+	end
+	
+	for handler, event in pairs(mod.module.event_handlers) do
+		event.object.remove_handler(event.name, handler);
+	end
+	
+	if mod.module.items then -- remove items
+		for key,t in pairs(mod.module.items) do
+			for i = #t,1,-1 do
+				local value = t[i];
+				t[i] = nil;
+				hosts[host].events.fire_event("item-removed/"..key, {source = mod.module, item = value});
+			end
+		end
+	end
+	modulemap[host][name] = nil;
+	return true;
+end
+
+local function do_load_module(host, module_name)
 	if not (host and module_name) then
 		return nil, "insufficient-parameters";
 	elseif not hosts[host] then
@@ -117,7 +139,9 @@ function load(host, module_name, config)
 	end
 
 	local _log = logger.init(host..":"..module_name);
-	local api_instance = setmetatable({ name = module_name, host = host, path = err, config = config,  _log = _log, log = function (self, ...) return _log(...); end }, { __index = api });
+	local api_instance = setmetatable({ name = module_name, host = host, path = err,
+		_log = _log, log = function (self, ...) return _log(...); end, event_handlers = {} }
+		, { __index = api });
 
 	local pluginenv = setmetatable({ module = api_instance }, { __index = _G });
 	api_instance.environment = pluginenv;
@@ -126,11 +150,12 @@ function load(host, module_name, config)
 	hosts[host].modules = modulemap[host];
 	modulemap[host][module_name] = pluginenv;
 	
-	local success, err = pcall(mod);
-	if success then
+	local ok, err = pcall(mod);
+	if ok then
+		-- Call module's "load"
 		if module_has_method(pluginenv, "load") then
-			success, err = call_module_method(pluginenv, "load");
-			if not success then
+			ok, err = call_module_method(pluginenv, "load");
+			if not ok then
 				log("warn", "Error loading module '%s' on '%s': %s", module_name, host, err or "nil");
 			end
 		end
@@ -143,62 +168,12 @@ function load(host, module_name, config)
 		end
 	else
 		log("error", "Error initializing module '%s' on '%s': %s", module_name, host, err or "nil");
+		do_unload_module(api_instance.host, module_name); -- Ignore error, module may be partially-loaded
 	end
-	if success then
-		(hosts[api_instance.host] or prosody).events.fire_event("module-loaded", { module = module_name, host = host });
-		return true;
-	else -- load failed, unloading
-		unload(api_instance.host, module_name);
-		return nil, err;
-	end
+	return ok and pluginenv, err;
 end
 
-function get_module(host, name)
-	return modulemap[host] and modulemap[host][name];
-end
-
-function is_loaded(host, name)
-	return modulemap[host] and modulemap[host][name] and true;
-end
-
-function unload(host, name, ...)
-	local mod = get_module(host, name);
-	if not mod then return nil, "module-not-loaded"; end
-	
-	if module_has_method(mod, "unload") then
-		local ok, err = call_module_method(mod, "unload");
-		if (not ok) and err then
-			log("warn", "Non-fatal error unloading module '%s' on '%s': %s", name, host, err);
-		end
-	end
-	-- unhook event handlers hooked by module:hook
-	for event, handlers in pairs(hooks:get(host, name) or NULL) do
-		for handler in pairs(handlers or NULL) do
-			(hosts[host] or prosody).events.remove_handler(event, handler);
-		end
-	end
-	-- unhook event handlers hooked by module:hook_global
-	for event, handlers in pairs(hooks:get("*", name) or NULL) do
-		for handler in pairs(handlers or NULL) do
-			prosody.events.remove_handler(event, handler);
-		end
-	end
-	hooks:remove(host, name);
-	if mod.module.items then -- remove items
-		for key,t in pairs(mod.module.items) do
-			for i = #t,1,-1 do
-				local value = t[i];
-				t[i] = nil;
-				hosts[host].events.fire_event("item-removed/"..key, {source = mod.module, item = value});
-			end
-		end
-	end
-	modulemap[host][name] = nil;
-	(hosts[host] or prosody).events.fire_event("module-unloaded", { module = name, host = host });
-	return true;
-end
-
-function reload(host, name, ...)
+local function do_reload_module(host, name)
 	local mod = get_module(host, name);
 	if not mod then return nil, "module-not-loaded"; end
 
@@ -209,7 +184,6 @@ function reload(host, name, ...)
 	end
 
 	local saved;
-
 	if module_has_method(mod, "save") then
 		local ok, ret, err = call_module_method(mod, "save");
 		if ok then
@@ -225,8 +199,8 @@ function reload(host, name, ...)
 		end
 	end
 
-	unload(host, name, ...);
-	local ok, err = load(host, name, ...);
+	do_unload_module(host, name);
+	local ok, err = do_load_module(host, name);
 	if ok then
 		mod = get_module(host, name);
 		if module_has_method(mod, "restore") then
@@ -235,231 +209,62 @@ function reload(host, name, ...)
 				log("warn", "Error restoring module '%s' from '%s': %s", name, host, err);
 			end
 		end
-		return true;
 	end
-	return ok, err;
+	return ok and mod, err;
 end
 
-function module_has_method(module, method)
-	return type(module.module[method]) == "function";
-end
+--- Public API ---
 
-function call_module_method(module, method, ...)
-	if module_has_method(module, method) then
-		local f = module.module[method];
-		return pcall(f, ...);
-	else
-		return false, "no-such-method";
+-- Load a module and fire module-loaded event
+function load(host, name)
+	local mod, err = do_load_module(host, name);
+	if mod then
+		(hosts[mod.module.host] or prosody).events.fire_event("module-loaded", { module = name, host = host });
 	end
+	return mod, err;
 end
 
------ API functions exposed to modules -----------
--- Must all be in api.*
-
--- Returns the name of the current module
-function api:get_name()
-	return self.name;
-end
-
--- Returns the host that the current module is serving
-function api:get_host()
-	return self.host;
-end
-
-function api:get_host_type()
-	return hosts[self.host].type;
-end
-
-function api:set_global()
-	self.host = "*";
-	-- Update the logger
-	local _log = logger.init("mod_"..self.name);
-	self.log = function (self, ...) return _log(...); end;
-	self._log = _log;
-end
-
-function api:add_feature(xmlns)
-	self:add_item("feature", xmlns);
-end
-function api:add_identity(category, type, name)
-	self:add_item("identity", {category = category, type = type, name = name});
-end
-function api:add_extension(data)
-	self:add_item("extension", data);
-end
-
-function api:fire_event(...)
-	return (hosts[self.host] or prosody).events.fire_event(...);
-end
-
-function api:hook(event, handler, priority)
-	hooks:set(self.host, self.name, event, handler, true);
-	(hosts[self.host] or prosody).events.add_handler(event, handler, priority);
-end
-
-function api:hook_global(event, handler, priority)
-	hooks:set("*", self.name, event, handler, true);
-	prosody.events.add_handler(event, handler, priority);
-end
-
-function api:hook_stanza(xmlns, name, handler, priority)
-	if not handler and type(name) == "function" then
-		-- If only 2 options then they specified no xmlns
-		xmlns, name, handler, priority = nil, xmlns, name, handler;
-	elseif not (handler and name) then
-		self:log("warn", "Error: Insufficient parameters to module:hook_stanza()");
-		return;
-	end
-	return api.hook(self, "stanza/"..(xmlns and (xmlns..":") or "")..name, function (data) return handler(data.origin, data.stanza, data); end, priority);
-end
-
-function api:require(lib)
-	local f, n = pluginloader.load_code(self.name, lib..".lib.lua");
-	if not f then
-		f, n = pluginloader.load_code(lib, lib..".lib.lua");
-	end
-	if not f then error("Failed to load plugin library '"..lib.."', error: "..n); end -- FIXME better error message
-	setfenv(f, self.environment);
-	return f();
-end
-
-function api:get_option(name, default_value)
-	local value = config.get(self.host, self.name, name);
-	if value == nil then
-		value = config.get(self.host, "core", name);
-		if value == nil then
-			value = default_value;
-		end
-	end
-	return value;
-end
-
-function api:get_option_string(name, default_value)
-	local value = self:get_option(name, default_value);
-	if type(value) == "table" then
-		if #value > 1 then
-			self:log("error", "Config option '%s' does not take a list, using just the first item", name);
-		end
-		value = value[1];
-	end
-	if value == nil then
-		return nil;
-	end
-	return tostring(value);
-end
-
-function api:get_option_number(name, ...)
-	local value = self:get_option(name, ...);
-	if type(value) == "table" then
-		if #value > 1 then
-			self:log("error", "Config option '%s' does not take a list, using just the first item", name);
-		end
-		value = value[1];
-	end
-	local ret = tonumber(value);
-	if value ~= nil and ret == nil then
-		self:log("error", "Config option '%s' not understood, expecting a number", name);
+-- Unload a module and fire module-unloaded
+function unload(host, name)
+	local ok, err = do_unload_module(host, name);
+	if ok then
+		(hosts[host] or prosody).events.fire_event("module-unloaded", { module = name, host = host });
 	end
-	return ret;
+	return ok, err;
 end
 
-function api:get_option_boolean(name, ...)
-	local value = self:get_option(name, ...);
-	if type(value) == "table" then
-		if #value > 1 then
-			self:log("error", "Config option '%s' does not take a list, using just the first item", name);
-		end
-		value = value[1];
-	end
-	if value == nil then
-		return nil;
-	end
-	local ret = value == true or value == "true" or value == 1 or nil;
-	if ret == nil then
-		ret = (value == false or value == "false" or value == 0);
-		if ret then
-			ret = false;
-		else
-			ret = nil;
-		end
-	end
-	if ret == nil then
-		self:log("error", "Config option '%s' not understood, expecting true/false", name);
+function reload(host, name)
+	local ok, err = do_reload_module(host, name);
+	if ok then
+		(hosts[host] or prosody).events.fire_event("module-reloaded", { module = name, host = host });
+	elseif not is_loaded(host, name) then
+		(hosts[host] or prosody).events.fire_event("module-unloaded", { module = name, host = host });
 	end
-	return ret;
+	return ok, err;
 end
 
-function api:get_option_array(name, ...)
-	local value = self:get_option(name, ...);
-
-	if value == nil then
-		return nil;
-	end
-	
-	if type(value) ~= "table" then
-		return array{ value }; -- Assume any non-list is a single-item list
-	end
-	
-	return array():append(value); -- Clone
+function get_module(host, name)
+	return modulemap[host] and modulemap[host][name];
 end
 
-function api:get_option_set(name, ...)
-	local value = self:get_option_array(name, ...);
-	
-	if value == nil then
-		return nil;
-	end
-	
-	return set.new(value);
+function get_modules(host)
+	return modulemap[host];
 end
 
-local t_remove = _G.table.remove;
-local module_items = multitable_new();
-function api:add_item(key, value)
-	self.items = self.items or {};
-	self.items[key] = self.items[key] or {};
-	t_insert(self.items[key], value);
-	self:fire_event("item-added/"..key, {source = self, item = value});
-end
-function api:remove_item(key, value)
-	local t = self.items and self.items[key] or NULL;
-	for i = #t,1,-1 do
-		if t[i] == value then
-			t_remove(self.items[key], i);
-			self:fire_event("item-removed/"..key, {source = self, item = value});
-			return value;
-		end
-	end
+function is_loaded(host, name)
+	return modulemap[host] and modulemap[host][name] and true;
 end
 
-function api:get_host_items(key)
-	local result = {};
-	for mod_name, module in pairs(modulemap[self.host]) do
-		module = module.module;
-		if module.items then
-			for _, item in ipairs(module.items[key] or NULL) do
-				t_insert(result, item);
-			end
-		end
-	end
-	for mod_name, module in pairs(modulemap["*"]) do
-		module = module.module;
-		if module.items then
-			for _, item in ipairs(module.items[key] or NULL) do
-				t_insert(result, item);
-			end
-		end
-	end
-	return result;
+function module_has_method(module, method)
+	return type(module.module[method]) == "function";
 end
 
-function api:handle_items(type, added_cb, removed_cb, existing)
-	self:hook("item-added/"..type, added_cb);
-	self:hook("item-removed/"..type, removed_cb);
-	if existing ~= false then
-		for _, item in ipairs(self:get_host_items(type)) do
-			added_cb({ item = item });
-		end
+function call_module_method(module, method, ...)
+	if module_has_method(module, method) then
+		local f = module.module[method];
+		return pcall(f, ...);
+	else
+		return false, "no-such-method";
 	end
 end
 
diff --git a/core/s2smanager.lua b/core/s2smanager.lua
index 77a1b040..f44921c3 100644
--- a/core/s2smanager.lua
+++ b/core/s2smanager.lua
@@ -16,14 +16,13 @@ local socket = require "socket";
 local format = string.format;
 local t_insert, t_sort = table.insert, table.sort;
 local get_traceback = debug.traceback;
-local tostring, pairs, ipairs, getmetatable, newproxy, next, error, tonumber, setmetatable
-    = tostring, pairs, ipairs, getmetatable, newproxy, next, error, tonumber, setmetatable;
+local tostring, pairs, ipairs, getmetatable, newproxy, type, error, tonumber, setmetatable
+    = tostring, pairs, ipairs, getmetatable, newproxy, type, error, tonumber, setmetatable;
 
 local idna_to_ascii = require "util.encodings".idna.to_ascii;
 local connlisteners_get = require "net.connlisteners".get;
 local initialize_filters = require "util.filters".initialize;
 local wrapclient = require "net.server".wrapclient;
-local modulemanager = require "core.modulemanager";
 local st = require "stanza";
 local stanza = st.stanza;
 local nameprep = require "util.encodings".stringprep.nameprep;
@@ -295,11 +294,11 @@ function attempt_connection(host_session, err)
 	return try_connect(host_session, connect_host, connect_port);
 end
 
-function try_next_ip(host_session, connect_port)
+function try_next_ip(host_session)
 	host_session.connecting = nil;
 	host_session.ip_choice = host_session.ip_choice + 1;
 	local ip = host_session.ip_hosts[host_session.ip_choice];
-	local ok, err= make_connect(host_session, ip, connect_port);
+	local ok, err= make_connect(host_session, ip.ip, ip.port);
 	if not ok then
 		if not attempt_connection(host_session, err or "closed") then
 			err = err and (": "..err) or "";
@@ -320,6 +319,9 @@ function try_connect(host_session, connect_host, connect_port, err)
 		if not sources then
 			sources =  {};
 			local cfg_sources = config.get("*", "core", "interface") or connlisteners_get("xmppserver").default_interface;
+			if type(cfg_sources) == "string" then
+				cfg_sources = { cfg_sources };
+			end
 			for i, source in ipairs(cfg_sources) do
 				if source == "*" then
 					sources[i] = new_ip("0.0.0.0", "IPv4");
@@ -354,8 +356,11 @@ function try_connect(host_session, connect_host, connect_port, err)
 			if has_other then
 				if #IPs > 0 then
 					rfc3484_dest(host_session.ip_hosts, sources);
+					for i = 1, #IPs do
+						IPs[i] = {ip = IPs[i], port = connect_port};
+					end
 					host_session.ip_choice = 0;
-					try_next_ip(host_session, connect_port);
+					try_next_ip(host_session);
 				else
 					log("debug", "DNS lookup failed to get a response for %s", connect_host);
 					host_session.ip_hosts = nil;
@@ -383,8 +388,11 @@ function try_connect(host_session, connect_host, connect_port, err)
 			if has_other then
 				if #IPs > 0 then
 					rfc3484_dest(host_session.ip_hosts, sources);
+					for i = 1, #IPs do
+						IPs[i] = {ip = IPs[i], port = connect_port};
+					end
 					host_session.ip_choice = 0;
-					try_next_ip(host_session, connect_port);
+					try_next_ip(host_session);
 				else
 					log("debug", "DNS lookup failed to get a response for %s", connect_host);
 					host_session.ip_hosts = nil;
@@ -401,7 +409,7 @@ function try_connect(host_session, connect_host, connect_port, err)
 
 		return true;
 	elseif host_session.ip_hosts and #host_session.ip_hosts > host_session.ip_choice then -- Not our first attempt, and we also have IPs left to try
-		try_next_ip(host_session, connect_port);
+		try_next_ip(host_session);
 	else
 		host_session.ip_hosts = nil;
 		if not attempt_connection(host_session, "out of IP addresses") then -- Retry if we can
diff --git a/core/sessionmanager.lua b/core/sessionmanager.lua
index 1de6c41a..b1ec819f 100644
--- a/core/sessionmanager.lua
+++ b/core/sessionmanager.lua
@@ -16,7 +16,6 @@ local hosts = hosts;
 local full_sessions = full_sessions;
 local bare_sessions = bare_sessions;
 
-local modulemanager = require "core.modulemanager";
 local logger = require "util.logger";
 local log = logger.init("sessionmanager");
 local error = error;
diff --git a/core/usermanager.lua b/core/usermanager.lua
index 0152afd7..9e5a016c 100644
--- a/core/usermanager.lua
+++ b/core/usermanager.lua
@@ -11,6 +11,7 @@ local log = require "util.logger".init("usermanager");
 local type = type;
 local ipairs = ipairs;
 local jid_bare = require "util.jid".bare;
+local jid_prep = require "util.jid".prep;
 local config = require "core.configmanager";
 local hosts = hosts;
 local sasl_new = require "util.sasl".new;
@@ -97,6 +98,7 @@ end
 
 function is_admin(jid, host)
 	if host and not hosts[host] then return false; end
+	if type(jid) ~= "string" then return false; end
 
 	local is_admin;
 	jid = jid_bare(jid);
@@ -108,7 +110,7 @@ function is_admin(jid, host)
 	if host_admins and host_admins ~= global_admins then
 		if type(host_admins) == "table" then
 			for _,admin in ipairs(host_admins) do
-				if admin == jid then
+				if jid_prep(admin) == jid then
 					is_admin = true;
 					break;
 				end
@@ -121,7 +123,7 @@ function is_admin(jid, host)
 	if not is_admin and global_admins then
 		if type(global_admins) == "table" then
 			for _,admin in ipairs(global_admins) do
-				if admin == jid then
+				if jid_prep(admin) == jid then
 					is_admin = true;
 					break;
 				end
diff --git a/net/http.lua b/net/http.lua
index 2b2f7258..6287f408 100644
--- a/net/http.lua
+++ b/net/http.lua
@@ -70,7 +70,7 @@ local function request_reader(request, data, startpos)
 		local function success_cb(r)
 			if request.callback then
 				for k,v in pairs(r) do request[k] = v; end
-				request.callback(r.body, r.code, request);
+				request.callback(r.body, r.code, request, r);
 				request.callback = nil;
 			end
 			destroy_request(request);
@@ -148,7 +148,7 @@ function request(u, ex, callback)
 	req.handler, req.conn = server.wrapclient(conn, req.host, port, listener, "*a", using_https and { mode = "client", protocol = "sslv23" });
 	req.write = function (...) return req.handler:write(...); end
 	
-	req.callback = function (content, code, request) log("debug", "Calling callback, status %s", code or "---"); return select(2, xpcall(function () return callback(content, code, request) end, handleerr)); end
+	req.callback = function (content, code, request, response) log("debug", "Calling callback, status %s", code or "---"); return select(2, xpcall(function () return callback(content, code, request, response) end, handleerr)); end
 	req.reader = request_reader;
 	req.state = "status";
 
diff --git a/net/httpserver.lua b/net/httpserver.lua
index 74f61c56..44e8e24d 100644
--- a/net/httpserver.lua
+++ b/net/httpserver.lua
@@ -7,7 +7,6 @@
 --
 
 
-local server = require "net.server"
 local url_parse = require "socket.url".parse;
 local httpstream_new = require "util.httpstream".new;
 
diff --git a/net/server_event.lua b/net/server_event.lua
index f4745c34..dbf5161f 100644
--- a/net/server_event.lua
+++ b/net/server_event.lua
@@ -295,7 +295,10 @@ do
 	end
 
 	function interface_mt:resume()
-		return self:_lock(self.nointerface, false, self.nowriting);
+		self:_lock(self.nointerface, false, self.nowriting);
+		if not self.eventread then
+			self.eventread = addevent( base, self.conn, EV_READ, self.readcallback, cfg.READ_TIMEOUT );  -- register callback
+		end
 	end
 
 	function interface_mt:counter(c)
@@ -642,6 +645,10 @@ do
 						return -1
 					end
 				end
+				if interface.noreading then
+					interface.eventread = nil;
+					return -1;
+				end
 				return EV_READ, cfg.READ_TIMEOUT
 			end
 		end
diff --git a/net/xmppcomponent_listener.lua b/net/xmppcomponent_listener.lua
index 90293559..dd7b2b91 100644
--- a/net/xmppcomponent_listener.lua
+++ b/net/xmppcomponent_listener.lua
@@ -7,8 +7,6 @@
 --
 
 
-local hosts = _G.hosts;
-
 local t_concat = table.concat;
 local tostring = tostring;
 local type = type;
diff --git a/net/xmppserver_listener.lua b/net/xmppserver_listener.lua
index a92b1a7f..048c6c7d 100644
--- a/net/xmppserver_listener.lua
+++ b/net/xmppserver_listener.lua
@@ -178,7 +178,7 @@ end
 function xmppserver.ondisconnect(conn, err)
 	local session = sessions[conn];
 	if session then
-		if err and err ~= "closed" then
+		if err and err ~= "closed"  and session.type == "s2sout_unauthed" then
 			(session.log or log)("debug", "s2s connection attempt failed: %s", err);
 			if s2s_attempt_connect(session, err) then
 				(session.log or log)("debug", "...so we're going to try another target");
diff --git a/plugins/adhoc/mod_adhoc.lua b/plugins/adhoc/mod_adhoc.lua
index 6d826338..49d07103 100644
--- a/plugins/adhoc/mod_adhoc.lua
+++ b/plugins/adhoc/mod_adhoc.lua
@@ -90,19 +90,13 @@ module:hook("iq/host/"..xmlns_cmd..":command", function (event)
 	end
 end, 500);
 
-local function handle_item_added(item)
+local function adhoc_added(event)
+	local item = event.item;
 	commands[item.node] = item;
 end
 
-module:hook("item-added/adhoc", function (event)
-	return handle_item_added(event.item);
-end, 500);
-
-module:hook("item-removed/adhoc", function (event)
+local function adhoc_removed(event)
 	commands[event.item.node] = nil;
-end, 500);
-
--- Pick up any items that are already added
-for _, item in ipairs(module:get_host_items("adhoc")) do
-	handle_item_added(item);
 end
+
+module:handle_items("adhoc", adhoc_added, adhoc_removed);
diff --git a/plugins/mod_admin_telnet.lua b/plugins/mod_admin_telnet.lua
index b211d271..b118e27a 100644
--- a/plugins/mod_admin_telnet.lua
+++ b/plugins/mod_admin_telnet.lua
@@ -21,12 +21,10 @@ local jid_bare = require "util.jid".bare;
 local set, array = require "util.set", require "util.array";
 local cert_verify_identity = require "util.x509".verify_identity;
 
-local commands = {};
-local def_env = {};
+local commands = module:shared("commands")
+local def_env = module:shared("env");
 local default_env_mt = { __index = def_env };
 
-prosody.console = { commands = commands, env = def_env };
-
 local function redirect_output(_G, session)
 	local env = setmetatable({ print = session.print }, { __index = function (t, k) return rawget(_G, k); end });
 	env.dofile = function(name)
diff --git a/plugins/mod_bosh.lua b/plugins/mod_bosh.lua
index 834b128a..9de27b4b 100644
--- a/plugins/mod_bosh.lua
+++ b/plugins/mod_bosh.lua
@@ -9,7 +9,6 @@
 module.host = "*" -- Global module
 
 local hosts = _G.hosts;
-local lxp = require "lxp";
 local new_xmpp_stream = require "util.xmppstream".new;
 local httpserver = require "net.httpserver";
 local sm = require "core.sessionmanager";
@@ -35,6 +34,7 @@ local BOSH_DEFAULT_POLLING = module:get_option_number("bosh_max_polling", 5);
 local BOSH_DEFAULT_REQUESTS = module:get_option_number("bosh_max_requests", 2);
 
 local consider_bosh_secure = module:get_option_boolean("consider_bosh_secure");
+local auto_cork = module:get_option_boolean("bosh_auto_cork", false);
 
 local default_headers = { ["Content-Type"] = "text/xml; charset=utf-8" };
 
@@ -57,7 +57,7 @@ end
 local trusted_proxies = module:get_option_set("trusted_proxies", {"127.0.0.1"})._items;
 
 local function get_ip_from_request(request)
-	local ip = request.handler:ip();
+	local ip = request.conn:ip();
 	local forwarded_for = request.headers["x-forwarded-for"];
 	if forwarded_for then
 		forwarded_for = forwarded_for..", "..ip;
@@ -91,9 +91,10 @@ function on_destroy_request(request)
 		end
 		
 		-- If this session now has no requests open, mark it as inactive
-		if #requests == 0 and session.bosh_max_inactive and not inactive_sessions[session] then
-			inactive_sessions[session] = os_time();
-			(session.log or log)("debug", "BOSH session marked as inactive at %d", inactive_sessions[session]);
+		local max_inactive = session.bosh_max_inactive;
+		if max_inactive and #requests == 0 then
+			inactive_sessions[session] = os_time() + max_inactive;
+			(session.log or log)("debug", "BOSH session marked as inactive (for %ds)", max_inactive);
 		end
 	end
 end
@@ -119,10 +120,17 @@ function handle_request(method, body, request)
 	request.on_destroy = on_destroy_request;
 	
 	local stream = new_xmpp_stream(request, stream_callbacks);
+	
 	-- stream:feed() calls the stream_callbacks, so all stanzas in
 	-- the body are processed in this next line before it returns.
+	-- In particular, the streamopened() stream callback is where
+	-- much of the session logic happens, because it's where we first
+	-- get to see the 'sid' of this request.
 	stream:feed(body);
 	
+	-- Stanzas (if any) in the request have now been processed, and
+	-- we take care of the high-level BOSH logic here, including
+	-- giving a response or putting the request "on hold".
 	local session = sessions[request.sid];
 	if session then
 		-- Session was marked as inactive, since we have
@@ -213,9 +221,11 @@ local function bosh_close_stream(session, reason)
 		held_request:destroy();
 	end
 	sessions[session.sid]  = nil;
+	inactive_sessions[session] = nil;
 	sm_destroy_session(session);
 end
 
+-- Handle the <body> tag in the request payload.
 function stream_callbacks.streamopened(request, attr)
 	local sid = attr.sid;
 	log("debug", "BOSH body open (sid: %s)", sid or "<none>");
@@ -258,7 +268,7 @@ function stream_callbacks.streamopened(request, attr)
 			end
 			--log("debug", "Sending BOSH data: %s", tostring(s));
 			local oldest_request = r[1];
-			if oldest_request then
+			if oldest_request and (not(auto_cork) or waiting_requests[oldest_request]) then
 				log("debug", "We have an open request, so sending on that");
 				response.body = t_concat({
 					"<body xmlns='http://jabber.org/protocol/httpbind' ",
@@ -338,14 +348,6 @@ function stream_callbacks.streamopened(request, attr)
 		session.rid = rid;
 	end
 	
-	if session.notopen then
-		local features = st.stanza("stream:features");
-		hosts[session.host].events.fire_event("stream-features", { origin = session, features = features });
-		fire_event("stream-features", session, features);
-		session.send(features);
-		session.notopen = nil;
-	end
-	
 	if attr.type == "terminate" then
 		-- Client wants to end this session, which we'll do
 		-- after processing any stanzas in this request
@@ -355,6 +357,14 @@ function stream_callbacks.streamopened(request, attr)
 	request.notopen = nil; -- Signals that we accept this opening tag
 	t_insert(session.requests, request);
 	request.sid = sid;
+
+	if session.notopen then
+		local features = st.stanza("stream:features");
+		hosts[session.host].events.fire_event("stream-features", { origin = session, features = features });
+		fire_event("stream-features", session, features);
+		session.send(features);
+		session.notopen = nil;
+	end
 end
 
 function stream_callbacks.handlestanza(request, stanza)
@@ -402,17 +412,13 @@ function on_timer()
 	
 	now = now - 3;
 	local n_dead_sessions = 0;
-	for session, inactive_since in pairs(inactive_sessions) do
-		if session.bosh_max_inactive then
-			if now - inactive_since > session.bosh_max_inactive then
-				(session.log or log)("debug", "BOSH client inactive too long, destroying session at %d", now);
-				sessions[session.sid]  = nil;
-				inactive_sessions[session] = nil;
-				n_dead_sessions = n_dead_sessions + 1;
-				dead_sessions[n_dead_sessions] = session;
-			end
-		else
+	for session, close_after in pairs(inactive_sessions) do
+		if close_after < now then
+			(session.log or log)("debug", "BOSH client inactive too long, destroying session at %d", now);
+			sessions[session.sid]  = nil;
 			inactive_sessions[session] = nil;
+			n_dead_sessions = n_dead_sessions + 1;
+			dead_sessions[n_dead_sessions] = session;
 		end
 	end
 
diff --git a/plugins/mod_component.lua b/plugins/mod_component.lua
index 97c46a8c..f7d09930 100644
--- a/plugins/mod_component.lua
+++ b/plugins/mod_component.lua
@@ -10,8 +10,6 @@ if module:get_host_type() ~= "component" then
 	error("Don't load mod_component manually, it should be for a component, please see http://prosody.im/doc/components", 0);
 end
 
-local hosts = _G.hosts;
-
 local t_concat = table.concat;
 
 local sha1 = require "util.hashes".sha1;
@@ -23,6 +21,7 @@ local main_session, send;
 
 local function on_destroy(session, err)
 	if main_session == session then
+		connected = false;
 		main_session = nil;
 		send = nil;
 		session.on_destroy = nil;
@@ -83,6 +82,7 @@ function handle_component_auth(event)
 	
 	-- If component not already created for this host, create one now
 	if not main_session then
+		connected = true;
 		send = session.send;
 		main_session = session;
 		session.on_destroy = on_destroy;
diff --git a/plugins/mod_compression.lua b/plugins/mod_compression.lua
index 82403016..62c0fa2e 100644
--- a/plugins/mod_compression.lua
+++ b/plugins/mod_compression.lua
@@ -16,12 +16,8 @@ local xmlns_stream = "http://etherx.jabber.org/streams";
 local compression_stream_feature = st.stanza("compression", {xmlns=xmlns_compression_feature}):tag("method"):text("zlib"):up();
 local add_filter = require "util.filters".add_filter;
 
-local compression_level = module:get_option("compression_level");
--- if not defined assume admin wants best compression
-if compression_level == nil then compression_level = 9 end;
+local compression_level = module:get_option_number("compression_level", 7);
 
-
-compression_level = tonumber(compression_level);
 if not compression_level or compression_level < 1 or compression_level > 9 then
 	module:log("warn", "Invalid compression level in config: %s", tostring(compression_level));
 	module:log("warn", "Module loading aborted. Compression won't be available.");
diff --git a/plugins/mod_tls.lua b/plugins/mod_tls.lua
index cace2d69..707ae8f5 100644
--- a/plugins/mod_tls.lua
+++ b/plugins/mod_tls.lua
@@ -75,7 +75,7 @@ end);
 module:hook_stanza("http://etherx.jabber.org/streams", "features", function (session, stanza)
 	module:log("debug", "Received features element");
 	if can_do_tls(session) and stanza:child_with_ns(xmlns_starttls) then
-		module:log("%s is offering TLS, taking up the offer...", session.to_host);
+		module:log("debug", "%s is offering TLS, taking up the offer...", session.to_host);
 		session.sends2s("<starttls xmlns='"..xmlns_starttls.."'/>");
 		return true;
 	end
diff --git a/plugins/mod_watchregistrations.lua b/plugins/mod_watchregistrations.lua
index ee51566b..42f9c017 100644
--- a/plugins/mod_watchregistrations.lua
+++ b/plugins/mod_watchregistrations.lua
@@ -18,7 +18,7 @@ module:hook("user-registered", function (user)
 	module:log("debug", "Notifying of new registration");
 	local message = st.message{ type = "chat", from = host }
 		:tag("body")
-			:text(registration_alert:gsub("%$(%w+)", function (v)
+			:text(registration_notification:gsub("%$(%w+)", function (v)
 				return user[v] or user.session and user.session[v] or nil;
 			end));
 	for _, jid in ipairs(registration_watchers) do
diff --git a/prosodyctl b/prosodyctl
index 9f8724d2..f0ba68ce 100755
--- a/prosodyctl
+++ b/prosodyctl
@@ -540,7 +540,7 @@ end
 
 function commands.reload(arg)
 	if arg[1] == "--help" then
-		show_usage([[reload]], [[Reload prosody configuration file]]);
+		show_usage([[reload]], [[Reload Prosody's configuration and re-open log files]]);
 		return 1;
 	end
 
diff --git a/tools/migration/migrator/jabberd14.lua b/tools/migration/migrator/jabberd14.lua
index 47004a52..54c45d94 100644
--- a/tools/migration/migrator/jabberd14.lua
+++ b/tools/migration/migrator/jabberd14.lua
@@ -67,15 +67,12 @@ local parse_xml = (function()
 end)();
 
 local function load_xml(path)
-	if path then
-		local f, err = io_open(path);
-		if not f then return f, err; end
-		local data = f:read("*a");
-		f:close();
-		if data then
-			return parse_xml(data);
-		end
-	end
+	local f, err = io_open(path);
+	if not f then return f, err; end
+	local data = f:read("*a");
+	f:close();
+	if not data then return; end
+	return parse_xml(data);
 end
 
 local function load_spool_file(host, filename, path)
diff --git a/util/array.lua b/util/array.lua
index 5dbd3037..5dc604ba 100644
--- a/util/array.lua
+++ b/util/array.lua
@@ -9,6 +9,11 @@
 local t_insert, t_sort, t_remove, t_concat
     = table.insert, table.sort, table.remove, table.concat;
 
+local setmetatable = setmetatable;
+local math_random = math.random;
+local pairs, ipairs = pairs, ipairs;
+local tostring = tostring;
+
 local array = {};
 local array_base = {};
 local array_methods = {};
@@ -25,6 +30,15 @@ end
 
 setmetatable(array, { __call = new_array });
 
+-- Read-only methods
+function array_methods:random()
+	return self[math_random(1,#self)];
+end
+
+-- These methods can be called two ways:
+--   array.method(existing_array, [params [, ...]]) -- Create new array for result
+--   existing_array:method([params, ...]) -- Transform existing array into result
+--
 function array_base.map(outa, ina, func)
 	for k,v in ipairs(ina) do
 		outa[k] = func(v);
@@ -60,15 +74,18 @@ function array_base.sort(outa, ina, ...)
 	return outa;
 end
 
---- These methods only mutate
-function array_methods:random()
-	return self[math.random(1,#self)];
+function array_base.pluck(outa, ina, key)
+	for i=1,#ina do
+		outa[i] = ina[i][key];
+	end
+	return outa;
 end
 
+--- These methods only mutate the array
 function array_methods:shuffle(outa, ina)
 	local len = #self;
 	for i=1,#self do
-		local r = math.random(i,len);
+		local r = math_random(i,len);
 		self[i], self[r] = self[r], self[i];
 	end
 	return self;
@@ -91,10 +108,24 @@ function array_methods:append(array)
 	return self;
 end
 
-array_methods.push = table.insert;
-array_methods.pop = table.remove;
-array_methods.concat = table.concat;
-array_methods.length = function (t) return #t; end
+function array_methods:push(x)
+	t_insert(self, x);
+	return self;
+end
+
+function array_methods:pop(x)
+	local v = self[x];
+	t_remove(self, x);
+	return v;
+end
+
+function array_methods:concat(sep)
+	return t_concat(array.map(self, tostring), sep);
+end
+
+function array_methods:length()
+	return #self;
+end
 
 --- These methods always create a new array
 function array.collect(f, s, var)
@@ -102,7 +133,7 @@ function array.collect(f, s, var)
 	while true do
 		var = f(s, var);
 	        if var == nil then break; end
-		table.insert(t, var);
+		t_insert(t, var);
 	end
 	return setmetatable(t, array_mt);
 end
diff --git a/util/dataforms.lua b/util/dataforms.lua
index e4d24cf6..d4a1865c 100644
--- a/util/dataforms.lua
+++ b/util/dataforms.lua
@@ -120,12 +120,18 @@ function form_t.data(layout, stanza)
 			end
 		end
 
-		local reader = field_readers[field.type];
-		local verifier = field.verifier or field_verifiers[field.type];
-		if reader then
-			data[field.name] = reader(tag);
-			if verifier then
-				errors[field.name] = verifier(data[field.name], tag, field.required);
+		if not tag then
+			if field.required then
+				errors[field.name] = "Required value missing";
+			end
+		else
+			local reader = field_readers[field.type];
+			local verifier = field.verifier or field_verifiers[field.type];
+			if reader then
+				data[field.name] = reader(tag);
+				if verifier then
+					errors[field.name] = verifier(data[field.name], tag, field.required);
+				end
 			end
 		end
 	end
@@ -161,7 +167,7 @@ field_readers["jid-single"] =
 
 field_verifiers["jid-single"] =
 	function (data, field_tag, required)
-		if #data == 0 and required then
+		if ((not data) or (#data == 0)) and required then
 			return "Required value missing";
 		end
 		if not jid_prep(data) then
@@ -246,7 +252,7 @@ field_readers["boolean"] =
 field_verifiers["boolean"] =
 	function (data, field_tag, required)
 		data = field_readers["text-single"](field_tag);
-		if #data == 0 and required then
+		if ((not data) or (#data == 0)) and required then
 			return "Required value missing";
 		end
 		if data ~= "1" and data ~= "true" and data ~= "0" and data ~= "false" then
diff --git a/util/datamanager.lua b/util/datamanager.lua
index d5e9c88c..a5d676cc 100644
--- a/util/datamanager.lua
+++ b/util/datamanager.lua
@@ -1,7 +1,7 @@
 -- Prosody IM
 -- Copyright (C) 2008-2010 Matthew Wild
 -- Copyright (C) 2008-2010 Waqas Hussain
--- 
+--
 -- This project is MIT/X11 licensed. Please see the
 -- COPYING file in the source package for more information.
 --
@@ -20,7 +20,7 @@ local error = error;
 local next = next;
 local t_insert = table.insert;
 local append = require "util.serialization".append;
-local path_separator = "/"; if os.getenv("WINDIR") then path_separator = "\\" end
+local path_separator = assert ( package.config:match ( "^([^\n]+)" ) , "package.config not in standard form" ) -- Extract directory seperator from package.config (an undocumented string that comes with lua)
 local lfs = require "lfs";
 local prosody = prosody;
 local raw_mkdir;
@@ -72,7 +72,7 @@ local function callback(username, host, datastore, data)
 		username, host, datastore, data = f(username, host, datastore, data);
 		if username == false then break; end
 	end
-	
+
 	return username, host, datastore, data;
 end
 function add_callback(func)
diff --git a/util/debug.lua b/util/debug.lua
index 22d02bf2..2170a6d1 100644
--- a/util/debug.lua
+++ b/util/debug.lua
@@ -9,6 +9,7 @@ local censored_names = {
 };
 
 local function get_locals_table(level)
+	level = level + 1; -- Skip this function itself
 	local locals = {};
 	for local_num = 1, math.huge do
 		local name, value = debug.getlocal(level, local_num);
diff --git a/util/helpers.lua b/util/helpers.lua
index 11356176..a8c8c612 100644
--- a/util/helpers.lua
+++ b/util/helpers.lua
@@ -28,7 +28,7 @@ function log_events(events, name, logger)
 end
 
 function revert_log_events(events)
-	events.fire_event, events[events.fire_event] = events[events.fire_event], nil; -- :)
+	events.fire_event, events[events.fire_event] = events[events.fire_event], nil; -- :))
 end
 
 function get_upvalue(f, get_name)
diff --git a/util/iterators.lua b/util/iterators.lua
index 2a87e97a..aa0b172b 100644
--- a/util/iterators.lua
+++ b/util/iterators.lua
@@ -140,7 +140,7 @@ end
 -- Treat the return of an iterator as key,value pairs,
 -- and build a table
 function it2table(f, s, var)
-	local t, var = {};
+	local t, var2 = {};
 	while true do
 		var, var2 = f(s, var);
 	        if var == nil then break; end
diff --git a/util/json.lua b/util/json.lua
index 5d0b0876..efc602f0 100644
--- a/util/json.lua
+++ b/util/json.lua
@@ -1,6 +1,6 @@
 
 local type = type;
-local t_insert, t_concat, t_remove = table.insert, table.concat, table.remove;
+local t_insert, t_concat, t_remove, t_sort = table.insert, table.concat, table.remove, table.sort;
 local s_char = string.char;
 local tostring, tonumber = tostring, tonumber;
 local pairs, ipairs = pairs, ipairs;
@@ -79,11 +79,25 @@ function tablesave(o, buffer)
 	if next(__hash) ~= nil or next(hash) ~= nil or next(__array) == nil then
 		t_insert(buffer, "{");
 		local mark = #buffer;
-		for k,v in pairs(hash) do
-			stringsave(k, buffer);
-			t_insert(buffer, ":");
-			simplesave(v, buffer);
-			t_insert(buffer, ",");
+		if buffer.ordered then
+			local keys = {};
+			for k in pairs(hash) do
+				t_insert(keys, k);
+			end
+			t_sort(keys);
+			for _,k in ipairs(keys) do
+				stringsave(k, buffer);
+				t_insert(buffer, ":");
+				simplesave(hash[k], buffer);
+				t_insert(buffer, ",");
+			end
+		else
+			for k,v in pairs(hash) do
+				stringsave(k, buffer);
+				t_insert(buffer, ":");
+				simplesave(v, buffer);
+				t_insert(buffer, ",");
+			end
 		end
 		if next(__hash) ~= nil then
 			t_insert(buffer, "\"__hash\":[");
@@ -129,6 +143,11 @@ function json.encode(obj)
 	simplesave(obj, t);
 	return t_concat(t);
 end
+function json.encode_ordered(obj)
+	local t = { ordered = true };
+	simplesave(obj, t);
+	return t_concat(t);
+end
 
 -----------------------------------
 
diff --git a/util/stanza.lua b/util/stanza.lua
index de83977f..600212a4 100644
--- a/util/stanza.lua
+++ b/util/stanza.lua
@@ -258,11 +258,6 @@ function stanza_mt.get_error(stanza)
 	return type, condition or "undefined-condition", text;
 end
 
-function stanza_mt.__add(s1, s2)
-	return s1:add_direct_child(s2);
-end
-
-
 do
 	local id = 0;
 	function new_id()
diff --git a/util/throttle.lua b/util/throttle.lua
index 8b62e797..2e901158 100644
--- a/util/throttle.lua
+++ b/util/throttle.lua
@@ -1,5 +1,6 @@
 
 local gettime = require "socket".gettime;
+local setmetatable = setmetatable;
 
 module "throttle"
 
@@ -33,7 +34,7 @@ function throttle:poll(cost, split)
 		if split then
 			self.balance = 0;
 		end
-		return false, balance, (cost-self.balance);
+		return false, balance, (cost-balance);
 	end
 end