14 files changed, 594 insertions, 457 deletions

diff --git a/tools/cfgdump.lua b/tools/cfgdump.lua
new file mode 100755
index 00000000..8c331a58
--- /dev/null
+++ b/tools/cfgdump.lua
@@ -0,0 +1,128 @@
+#!/usr/bin/env lua
+
+-- cfgdump.lua prosody.cfg.lua [[host] option]
+
+local s_format, print = string.format, print;
+local printf = function(fmt, ...) return print(s_format(fmt, ...)); end
+local it = require "util.iterators";
+local function sort_anything(a, b)
+	local typeof_a, typeof_b = type(a), type(b);
+	if typeof_a ~= typeof_b then return typeof_a < typeof_b end
+	return a < b -- should work for everything in a config file
+end
+local serialization = require "util.serialization";
+local serialize = serialization.new and serialization.new({
+	unquoted = true, table_iterator = function(t) return it.sorted_pairs(t, sort_anything); end,
+}) or serialization.serialize;
+local configmanager = require"core.configmanager";
+local startup = require "util.startup";
+
+startup.set_function_metatable();
+local config_filename, onlyhost, onlyoption = ...;
+
+local ok, _, err = configmanager.load(config_filename or "./prosody.cfg.lua", "lua");
+assert(ok, err);
+
+if onlyhost then
+	if not onlyoption then
+		onlyhost, onlyoption = "*", onlyhost;
+	end
+	if onlyhost ~= "*" then
+		local component_module = configmanager.get(onlyhost, "component_module");
+
+		if component_module == "component" then
+			printf("Component %q", onlyhost);
+		elseif component_module then
+			printf("Component %q %q", onlyhost, component_module);
+		else
+			printf("VirtualHost %q", onlyhost);
+		end
+	end
+	printf("%s = %s", onlyoption or "?", serialize(configmanager.get(onlyhost, onlyoption)));
+	return;
+end
+
+local config = configmanager.getconfig();
+
+
+for host, hostcfg in it.sorted_pairs(config) do
+	local fixed = {};
+	for option, value in it.sorted_pairs(hostcfg) do
+		fixed[option] = value;
+		if option:match("ports?$") or option:match("interfaces?$") then
+			if option:match("s$") then
+				if type(value) ~= "table" then
+					fixed[option] = { value };
+				end
+			else
+				if type(value) == "table" and #value > 1 then
+					fixed[option] = nil;
+					fixed[option.."s"] = value;
+				end
+			end
+		end
+	end
+	config[host] = fixed;
+end
+
+local globals = config["*"]; config["*"] = nil;
+
+local function printsection(section)
+	local out, n = {}, 1;
+	for k,v in it.sorted_pairs(section) do
+		out[n], n = s_format("%s = %s", k, serialize(v)), n + 1;
+	end
+	table.sort(out);
+	print(table.concat(out, "\n"));
+end
+
+print("-------------- Prosody Exported Configuration File -------------");
+print();
+print("------------------------ Global section ------------------------");
+print();
+printsection(globals);
+print();
+
+local has_components = nil;
+
+print("------------------------ Virtual hosts -------------------------");
+
+for host, hostcfg in it.sorted_pairs(config) do
+	setmetatable(hostcfg, nil);
+	hostcfg.defined = nil;
+
+	if hostcfg.component_module == nil then
+		print();
+		printf("VirtualHost %q", host);
+		printsection(hostcfg);
+	else
+		has_components = true
+	end
+end
+
+print();
+
+if has_components then
+print("------------------------- Components ---------------------------");
+
+	for host, hostcfg in it.sorted_pairs(config) do
+		local component_module = hostcfg.component_module;
+		hostcfg.component_module = nil;
+
+		if component_module then
+			print();
+			if component_module == "component" then
+				printf("Component %q", host);
+			else
+				printf("Component %q %q", host, component_module);
+				hostcfg.component_module = nil;
+				hostcfg.load_global_modules = nil;
+			end
+			printsection(hostcfg);
+		end
+	end
+end
+
+print()
+print("------------------------- End of File --------------------------");
+
diff --git a/tools/form2table.lua b/tools/form2table.lua
new file mode 100644
index 00000000..49a6972b
--- /dev/null
+++ b/tools/form2table.lua
@@ -0,0 +1,48 @@
+-- Read an XML dataform and spit out a serialized Lua table of it
+
+local function from_stanza(stanza)
+	local layout = {
+		title = stanza:get_child_text("title");
+		instructions = stanza:get_child_text("instructions");
+	};
+	for tag in stanza:childtags("field") do
+		local field = {
+			name = tag.attr.var;
+			type = tag.attr.type;
+			label = tag.attr.label;
+			desc = tag:get_child_text("desc");
+			required = tag:get_child("required") and true or nil;
+			value = tag:get_child_text("value");
+			options = nil;
+		};
+
+		if field.type == "list-single" or field.type == "list-multi" then
+			local options = {};
+			for option in tag:childtags("option") do
+				options[#options+1] = { label = option.attr.label, value = option:get_child_text("value") };
+			end
+			field.options = options;
+		end
+
+		if field.type == "jid-multi" or field.type == "list-multi" or field.type == "text-multi" then
+			local values = {};
+			for value in tag:childtags("value") do
+				values[#values+1] = value:get_text();
+			end
+			if field.type == "text-multi" then
+				values = table.concat(values, "\n");
+			end
+			field.value = values;
+		end
+
+		if field.type == "boolean" then
+			field.value = field.value == "true" or field.value == "1";
+		end
+
+		layout[#layout+1] = field;
+
+	end
+	return layout;
+end
+
+print("dataforms.new " .. require "util.serialization".serialize(from_stanza(require "util.xml".parse(io.read("*a"))), { unquoted = true }))
diff --git a/tools/jabberd14sql2prosody.lua b/tools/jabberd14sql2prosody.lua
index e43dc296..d1d66dc1 100644
--- a/tools/jabberd14sql2prosody.lua
+++ b/tools/jabberd14sql2prosody.lua
@@ -468,7 +468,7 @@ function store_vcard(username, host, stanza)
 end
 
 function store_roster(username, host, roster_items)
-	-- fetch current roster-table for username@host if he already has one
+	-- fetch current roster-table for username@host if they already have one
 	local roster = dm.load(username, host, "roster") or {};
 	-- merge imported roster-items with loaded roster
 	for item_tag in roster_items:childtags() do
@@ -508,7 +508,7 @@ end
 function store_subscription_request(username, host, presence_stanza)
 	local from_bare = presence_stanza.attr.from;
 
-	-- fetch current roster-table for username@host if he already has one
+	-- fetch current roster-table for username@host if they already have one
 	local roster = dm.load(username, host, "roster") or {};
 
 	local item = roster[from_bare];
diff --git a/tools/linedebug.lua b/tools/linedebug.lua
new file mode 100644
index 00000000..96214cc8
--- /dev/null
+++ b/tools/linedebug.lua
@@ -0,0 +1,18 @@
+local data = {}
+local getinfo = debug.getinfo;
+local function linehook(ev, li)
+	local S = getinfo(2, "S");
+	if S and S.source and S.source:match"^@" then
+		local file = S.source:sub(2);
+		local lines = data[file];
+		if not lines then
+			lines = {};
+			data[file] = lines;
+			for line in io.lines(file) do
+				lines[#lines+1] = line;
+			end
+		end
+		io.stderr:write(ev, " ", file, " ", li, " ", lines[li], "\n");
+	end
+end
+debug.sethook(linehook, "l");
diff --git a/tools/make_repo.lua b/tools/make_repo.lua
new file mode 100644
index 00000000..5b5e6234
--- /dev/null
+++ b/tools/make_repo.lua
@@ -0,0 +1,44 @@
+print("Getting all the available modules")
+if os.execute '[ -e "./downloaded_modules" ]' then
+	os.execute("rm -rf downloaded_modules")
+end
+os.execute("hg clone https://hg.prosody.im/prosody-modules/ downloaded_modules")
+local i, popen = 0, io.popen
+local flag = "mod_"
+if os.execute '[ -e "./repository" ]' then
+	os.execute("mkdir repository")
+end
+local pfile = popen('ls -a "downloaded_modules"')
+for filename in pfile:lines() do
+	i = i + 1
+	if filename:sub(1, #flag) == flag then
+		local file = io.open("repository/"..filename.."-scm-1.rockspec", "w")
+		file:write('package = "'..filename..'"', '\n')
+		file:write('version = "scm-1"', '\n')
+		file:write('source = {', '\n')
+		file:write('\turl = "hg+https://hg.prosody.im/prosody-modules",', '\n')
+		file:write('\tdir = "prosody-modules"', '\n')
+		file:write('}', '\n')
+		file:write('description = {', '\n')
+		file:write('\thomepage = "https://prosody.im/",', '\n')
+		file:write('\tlicense = "MIT"', '\n')
+		file:write('}', '\n')
+		file:write('dependencies = {', '\n')
+		file:write('\t"lua >= 5.1"', '\n')
+		file:write('}', '\n')
+		file:write('build = {', '\n')
+		file:write('\ttype = "builtin",', '\n')
+		file:write('\tmodules = {', '\n')
+		file:write('\t\t["'..filename..'.'..filename..'"] = "'..filename..'/'..filename..'.lua"', '\n')
+		file:write('\t}', '\n')
+		file:write('}', '\n')
+		file:close()
+	end
+end
+pfile:close()
+os.execute("cd repository/ && luarocks-admin make_manifest ./ && chmod -R 644 ./*")
+print("")
+print("Done!. Modules' sources are locally available at ./downloaded_modules")
+print("Repository is available at ./repository")
+print("The repository contains all of prosody modules' respective rockspecs, as well as manifest files and an html Index")
+print("You can now either point your server to this folder, or copy its contents to another configured folder.")
diff --git a/tools/migration/Makefile b/tools/migration/Makefile
index 713831d2..fce0f7d9 100644
--- a/tools/migration/Makefile
+++ b/tools/migration/Makefile
@@ -12,16 +12,12 @@ INSTALLEDCONFIG = $(SYSCONFDIR)
 INSTALLEDMODULES = $(LIBDIR)/prosody/modules
 INSTALLEDDATA = $(DATADIR)
 
-SOURCE_FILES = migrator/*.lua
-
-all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua $(SOURCE_FILES)
+all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua
 
 install: prosody-migrator.install migrator.cfg.lua.install
-	install -d $(BIN) $(CONFIG) $(SOURCE) $(SOURCE)/migrator
+	install -d $(BIN) $(CONFIG) $(SOURCE)
 	install -d $(MAN)/man1
-	install -d $(SOURCE)/migrator
 	install -m755 ./prosody-migrator.install $(BIN)/prosody-migrator
-	install -m644 $(SOURCE_FILES) $(SOURCE)/migrator
 	test -e $(CONFIG)/migrator.cfg.lua || install -m644 migrator.cfg.lua.install $(CONFIG)/migrator.cfg.lua
 
 clean:
@@ -31,7 +27,9 @@ clean:
 prosody-migrator.install: prosody-migrator.lua
 	sed "1s/\blua\b/$(RUNWITH)/; \
 		s|^CFG_SOURCEDIR=.*;$$|CFG_SOURCEDIR='$(INSTALLEDSOURCE)';|; \
-		s|^CFG_CONFIGDIR=.*;$$|CFG_CONFIGDIR='$(INSTALLEDCONFIG)';|;" \
+		s|^CFG_CONFIGDIR=.*;$$|CFG_CONFIGDIR='$(INSTALLEDCONFIG)';|; \
+		s|^CFG_DATADIR=.*;$$|CFG_DATADIR='$(INSTALLEDDATA)';|; \
+		s|^CFG_PLUGINDIR=.*;$$|CFG_PLUGINDIR='$(INSTALLEDMODULES)/';|;" \
 			< prosody-migrator.lua > prosody-migrator.install
 
 migrator.cfg.lua.install: migrator.cfg.lua
diff --git a/tools/migration/migrator.cfg.lua b/tools/migration/migrator.cfg.lua
index fa37f2a3..c26fa869 100644
--- a/tools/migration/migrator.cfg.lua
+++ b/tools/migration/migrator.cfg.lua
@@ -1,12 +1,38 @@
 local data_path = "../../data";
 
+local vhost = {
+	"accounts",
+	"account_details",
+	"roster",
+	"vcard",
+	"private",
+	"blocklist",
+	"privacy",
+	"archive-archive",
+	"offline-archive",
+	"pubsub_nodes",
+	-- "pubsub_*-archive",
+	"pep",
+	-- "pep_*-archive",
+}
+local muc = {
+	"persistent",
+	"config",
+	"state",
+	"muc_log-archive",
+};
+
 input {
-	type = "prosody_files";
+	hosts = {
+		["example.com"] = vhost;
+		["conference.example.com"] = muc;
+	};
+	type = "internal";
 	path = data_path;
 }
 
 output {
-	type = "prosody_sql";
+	type = "sql";
 	driver = "SQLite3";
 	database = data_path.."/prosody.sqlite";
 }
@@ -14,11 +40,11 @@ output {
 --[[
 
 input {
-	type = "prosody_files";
+	type = "internal";
 	path = data_path;
 }
 output {
-	type = "prosody_sql";
+	type = "sql";
 	driver = "SQLite3";
 	database = data_path.."/prosody.sqlite";
 }
diff --git a/tools/migration/migrator/mtools.lua b/tools/migration/migrator/mtools.lua
deleted file mode 100644
index cfbfcce8..00000000
--- a/tools/migration/migrator/mtools.lua
+++ /dev/null
@@ -1,58 +0,0 @@
-
-
-local print = print;
-local t_insert = table.insert;
-local t_sort = table.sort;
-
-
-local function sorted(params)
-
-	local reader = params.reader; -- iterator to get items from
-	local sorter = params.sorter; -- sorting function
-	local filter = params.filter; -- filter function
-
-	local cache = {};
-	for item in reader do
-		if filter then item = filter(item); end
-		if item then t_insert(cache, item); end
-	end
-	if sorter then
-		t_sort(cache, sorter);
-	end
-	local i = 0;
-	return function()
-		i = i + 1;
-		return cache[i];
-	end;
-
-end
-
-local function merged(reader, merger)
-
-	local item1 = reader();
-	local merged = { item1 };
-	return function()
-		while true do
-			if not item1 then return nil; end
-			local item2 = reader();
-			if not item2 then item1 = nil; return merged; end
-			if merger(item1, item2) then
-			--print("merged")
-				item1 = item2;
-				t_insert(merged, item1);
-			else
-			--print("unmerged", merged)
-				item1 = item2;
-				local tmp = merged;
-				merged = { item1 };
-				return tmp;
-			end
-		end
-	end;
-
-end
-
-return {
-	sorted = sorted;
-	merged = merged;
-}
diff --git a/tools/migration/migrator/prosody_files.lua b/tools/migration/migrator/prosody_files.lua
deleted file mode 100644
index 4de09273..00000000
--- a/tools/migration/migrator/prosody_files.lua
+++ /dev/null
@@ -1,144 +0,0 @@
-
-local print = print;
-local assert = assert;
-local setmetatable = setmetatable;
-local tonumber = tonumber;
-local char = string.char;
-local coroutine = coroutine;
-local lfs = require "lfs";
-local loadfile = loadfile;
-local pcall = pcall;
-local mtools = require "migrator.mtools";
-local next = next;
-local pairs = pairs;
-local json = require "util.json";
-local os_getenv = os.getenv;
-local error = error;
-
-prosody = {};
-local dm = require "util.datamanager"
-
-
-local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end
-local function is_file(path) return lfs.attributes(path, "mode") == "file"; end
-local function clean_path(path)
-	return path:gsub("\\", "/"):gsub("//+", "/"):gsub("^~", os_getenv("HOME") or "~");
-end
-local encode, decode; do
-	local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end });
-	decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end
-	encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end
-end
-local function decode_dir(x)
-	if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
-		return decode(x);
-	end
-end
-local function decode_file(x)
-	if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
-		return decode(x:gsub("%.dat$", ""));
-	end
-end
-local function prosody_dir(path, ondir, onfile, ...)
-	for x in lfs.dir(path) do
-		local xpath = path.."/"..x;
-		if decode_dir(x) and is_dir(xpath) then
-			ondir(xpath, x, ...);
-		elseif decode_file(x) and is_file(xpath) then
-			onfile(xpath, x, ...);
-		end
-	end
-end
-
-local function handle_root_file(path, name)
-	--print("root file: ", decode_file(name))
-	coroutine.yield { user = nil, host = nil, store = decode_file(name) };
-end
-local function handle_host_file(path, name, host)
-	--print("host file: ", decode_dir(host).."/"..decode_file(name))
-	coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) };
-end
-local function handle_store_file(path, name, store, host)
-	--print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store))
-	coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) };
-end
-local function handle_host_store(path, name, host)
-	prosody_dir(path, function() end, handle_store_file, name, host);
-end
-local function handle_host_dir(path, name)
-	prosody_dir(path, handle_host_store, handle_host_file, name);
-end
-local function handle_root_dir(path)
-	prosody_dir(path, handle_host_dir, handle_root_file);
-end
-
-local function decode_user(item)
-	local userdata = {
-		user = item[1].user;
-		host = item[1].host;
-		stores = {};
-	};
-	for i=1,#item do -- loop over stores
-		local result = {};
-		local store = item[i];
-		userdata.stores[store.store] = store.data;
-		store.user = nil; store.host = nil; store.store = nil;
-	end
-	return userdata;
-end
-
-local function reader(input)
-	local path = clean_path(assert(input.path, "no input.path specified"));
-	assert(is_dir(path), "input.path is not a directory");
-	local iter = coroutine.wrap(function()handle_root_dir(path);end);
-	-- get per-user stores, sorted
-	local iter = mtools.sorted {
-		reader = function()
-			local x = iter();
-			while x do
-				dm.set_data_path(path);
-				local err;
-				x.data, err = dm.load(x.user, x.host, x.store);
-				if x.data == nil and err then
-					local p = dm.getpath(x.user, x.host, x.store);
-					print(("Error loading data at path %s for %s@%s (%s store): %s")
-						:format(p, x.user or "<nil>", x.host or "<nil>", x.store or "<nil>", err or "<nil>"));
-				else
-					return x;
-				end
-				x = iter();
-			end
-		end;
-		sorter = function(a, b)
-			local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";
-			local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";
-			return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);
-		end;
-	};
-	-- merge stores to get users
-	iter = mtools.merged(iter, function(a, b)
-		return (a.host == b.host and a.user == b.user);
-	end);
-
-	return function()
-		local x = iter();
-		return x and decode_user(x);
-	end
-end
-
-local function writer(output)
-	local path = clean_path(assert(output.path, "no output.path specified"));
-	assert(is_dir(path), "output.path is not a directory");
-	return function(item)
-		if not item then return; end -- end of input
-		dm.set_data_path(path);
-		for store, data in pairs(item.stores) do
-			assert(dm.store(item.user, item.host, store, data));
-		end
-	end
-end
-
-return {
-	reader = reader;
-	writer = writer;
-}
diff --git a/tools/migration/migrator/prosody_sql.lua b/tools/migration/migrator/prosody_sql.lua
deleted file mode 100644
index 6df2b025..00000000
--- a/tools/migration/migrator/prosody_sql.lua
+++ /dev/null
@@ -1,190 +0,0 @@
-
-local assert = assert;
-local have_DBI = pcall(require,"DBI");
-local print = print;
-local type = type;
-local next = next;
-local pairs = pairs;
-local t_sort = table.sort;
-local json = require "util.json";
-local mtools = require "migrator.mtools";
-local tostring = tostring;
-local tonumber = tonumber;
-
-if not have_DBI then
-	error("LuaDBI (required for SQL support) was not found, please see https://prosody.im/doc/depends#luadbi", 0);
-end
-
-local sql = require "util.sql";
-
-local function create_table(engine, name) -- luacheck: ignore 431/engine
-	local Table, Column, Index = sql.Table, sql.Column, sql.Index;
-
-	local ProsodyTable = Table {
-		name= name or "prosody";
-		Column { name="host", type="TEXT", nullable=false };
-		Column { name="user", type="TEXT", nullable=false };
-		Column { name="store", type="TEXT", nullable=false };
-		Column { name="key", type="TEXT", nullable=false };
-		Column { name="type", type="TEXT", nullable=false };
-		Column { name="value", type="MEDIUMTEXT", nullable=false };
-		Index { name="prosody_index", "host", "user", "store", "key" };
-	};
-	engine:transaction(function()
-		ProsodyTable:create(engine);
-	end);
-
-end
-
-local function serialize(value)
-	local t = type(value);
-	if t == "string" or t == "boolean" or t == "number" then
-		return t, tostring(value);
-	elseif t == "table" then
-		local value,err = json.encode(value);
-		if value then return "json", value; end
-		return nil, err;
-	end
-	return nil, "Unhandled value type: "..t;
-end
-local function deserialize(t, value)
-	if t == "string" then return value;
-	elseif t == "boolean" then
-		if value == "true" then return true;
-		elseif value == "false" then return false; end
-	elseif t == "number" then return tonumber(value);
-	elseif t == "json" then
-		return json.decode(value);
-	end
-end
-
-local function decode_user(item)
-	local userdata = {
-		user = item[1][1].user;
-		host = item[1][1].host;
-		stores = {};
-	};
-	for i=1,#item do -- loop over stores
-		local result = {};
-		local store = item[i];
-		for i=1,#store do -- loop over store data
-			local row = store[i];
-			local k = row.key;
-			local v = deserialize(row.type, row.value);
-			if k and v then
-				if k ~= "" then result[k] = v; elseif type(v) == "table" then
-					for a,b in pairs(v) do
-						result[a] = b;
-					end
-				end
-			end
-			userdata.stores[store[1].store] = result;
-		end
-	end
-	return userdata;
-end
-
-local function needs_upgrade(engine, params)
-	if params.driver == "MySQL" then
-		local success = engine:transaction(function()
-			local result = engine:execute("SHOW COLUMNS FROM prosody WHERE Field='value' and Type='text'");
-			assert(result:rowcount() == 0);
-
-			-- COMPAT w/pre-0.10: Upgrade table to UTF-8 if not already
-			local check_encoding_query = [[
-			SELECT "COLUMN_NAME","COLUMN_TYPE","TABLE_NAME"
-			FROM "information_schema"."columns"
-			WHERE "TABLE_NAME" LIKE 'prosody%%' AND ( "CHARACTER_SET_NAME"!='%s' OR "COLLATION_NAME"!='%s_bin' );
-			]];
-			check_encoding_query = check_encoding_query:format(engine.charset, engine.charset);
-			local result = engine:execute(check_encoding_query);
-			assert(result:rowcount() == 0)
-		end);
-		if not success then
-			-- Upgrade required
-			return true;
-		end
-	end
-	return false;
-end
-
-local function reader(input)
-	local engine = assert(sql:create_engine(input, function (engine) -- luacheck: ignore 431/engine
-		if needs_upgrade(engine, input) then
-			error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade");
-		end
-	end));
-	local keys = {"host", "user", "store", "key", "type", "value"};
-	assert(engine:connect());
-	local f,s,val = assert(engine:select("SELECT \"host\", \"user\", \"store\", \"key\", \"type\", \"value\" FROM \"prosody\";"));
-	-- get SQL rows, sorted
-	local iter = mtools.sorted {
-		reader = function() val = f(s, val); return val; end;
-		filter = function(x)
-			for i=1,#keys do
-				x[ keys[i] ] = x[i];
-			end
-			if x.host  == "" then x.host  = nil; end
-			if x.user  == "" then x.user  = nil; end
-			if x.store == "" then x.store = nil; end
-			return x;
-		end;
-		sorter = function(a, b)
-			local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";
-			local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";
-			return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);
-		end;
-	};
-	-- merge rows to get stores
-	iter = mtools.merged(iter, function(a, b)
-		return (a.host == b.host and a.user == b.user and a.store == b.store);
-	end);
-	-- merge stores to get users
-	iter = mtools.merged(iter, function(a, b)
-		return (a[1].host == b[1].host and a[1].user == b[1].user);
-	end);
-	return function()
-		local x = iter();
-		return x and decode_user(x);
-	end;
-end
-
-local function writer(output, iter)
-	local engine = assert(sql:create_engine(output, function (engine) -- luacheck: ignore 431/engine
-		if needs_upgrade(engine, output) then
-			error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade");
-		end
-		create_table(engine);
-	end));
-	assert(engine:connect());
-	assert(engine:delete("DELETE FROM \"prosody\""));
-	local insert_sql = "INSERT INTO \"prosody\" (\"host\",\"user\",\"store\",\"key\",\"type\",\"value\") VALUES (?,?,?,?,?,?)";
-
-	return function(item)
-		if not item then assert(engine.conn:commit()) return end -- end of input
-		local host = item.host or "";
-		local user = item.user or "";
-		for store, data in pairs(item.stores) do
-			-- TODO transactions
-			local extradata = {};
-			for key, value in pairs(data) do
-				if type(key) == "string" and key ~= "" then
-					local t, value = assert(serialize(value));
-					local ok, err = assert(engine:insert(insert_sql, host, user, store, key, t, value));
-				else
-					extradata[key] = value;
-				end
-			end
-			if next(extradata) ~= nil then
-				local t, extradata = assert(serialize(extradata));
-				local ok, err = assert(engine:insert(insert_sql, host, user, store, "", t, extradata));
-			end
-		end
-	end;
-end
-
-
-return {
-	reader = reader;
-	writer = writer;
-}
diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua
index 1219d891..26b9ef58 100644
--- a/tools/migration/prosody-migrator.lua
+++ b/tools/migration/prosody-migrator.lua
@@ -1,41 +1,67 @@
 #!/usr/bin/env lua
 
-CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR");
-CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR");
+CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR");
+CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR");
+CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR");
+CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR");
 
--- Substitute ~ with path to home directory in paths
-if CFG_CONFIGDIR then
-	CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME"));
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+local function is_relative(path)
+	local path_sep = package.config:sub(1,1);
+        return ((path_sep == "/" and path:sub(1,1) ~= "/")
+	or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\")))
 end
 
+-- Tell Lua where to find our libraries
 if CFG_SOURCEDIR then
-	CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME"));
+	local function filter_relative_paths(path)
+		if is_relative(path) then return ""; end
+	end
+	local function sanitise_paths(paths)
+		return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";"));
+	end
+	package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path);
+	package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath);
+end
+
+-- Substitute ~ with path to home directory in data path
+if CFG_DATADIR then
+	if os.getenv("HOME") then
+		CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME"));
+	end
 end
 
 local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua";
 
--- Command-line parsing
-local options = {};
-local i = 1;
-while arg[i] do
-	if arg[i]:sub(1,2) == "--" then
-		local opt, val = arg[i]:match("([%w-]+)=?(.*)");
-		if opt then
-			options[(opt:sub(3):gsub("%-", "_"))] = #val > 0 and val or true;
-		end
-		table.remove(arg, i);
-	else
-		i = i + 1;
+local startup = require "util.startup";
+do
+	startup.parse_args();
+	startup.init_global_state();
+	prosody.process_type = "migrator";
+	startup.force_console_logging();
+	startup.init_logging();
+	startup.init_gc();
+	startup.init_errors();
+	startup.setup_plugindir();
+	startup.setup_plugin_install_path();
+	startup.setup_datadir();
+	startup.chdir();
+	startup.read_version();
+	startup.switch_user();
+	startup.check_dependencies();
+	startup.log_startup_warnings();
+	prosody.config_loaded = true;
+	startup.load_libraries();
+	startup.init_http_client();
+	prosody.core_post_stanza = function ()
+		-- silence assert in core.moduleapi
+		error("Attempt to send stanzas from inside migrator.", 0);
 	end
 end
 
-if CFG_SOURCEDIR then
-	package.path = CFG_SOURCEDIR.."/?.lua;"..package.path;
-	package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath;
-else
-	package.path = "../../?.lua;"..package.path
-	package.cpath = "../../?.so;"..package.cpath
-end
+-- Command-line parsing
+local options = prosody.opts;
 
 local envloadfile = require "util.envload".envloadfile;
 
@@ -69,24 +95,14 @@ if not config[to_store] then
 	print("Error: Output store '"..to_store.."' not found in the config file.");
 end
 
-function load_store_handler(name)
-	local store_type = config[name].type;
-	if not store_type then
-		print("Error: "..name.." store type not specified in the config file");
-		return false;
-	else
-		local ok, err = pcall(require, "migrator."..store_type);
-		if not ok then
-			print(("Error: Failed to initialize '%s' store:\n\t%s")
-				:format(name, err));
-			return false;
-		end
+for store, conf in pairs(config) do -- COMPAT
+	if conf.type == "prosody_files" then
+		conf.type = "internal";
+	elseif conf.type == "prosody_sql" then
+		conf.type = "sql";
 	end
-	return true;
 end
 
-have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output"));
-
 if have_err then
 	print("");
 	print("Usage: "..arg[0].." FROM_STORE TO_STORE");
@@ -101,17 +117,88 @@ if have_err then
 	os.exit(1);
 end
 
-local itype = config[from_store].type;
-local otype = config[to_store].type;
-local reader = require("migrator."..itype).reader(config[from_store]);
-local writer = require("migrator."..otype).writer(config[to_store]);
+local async = require "util.async";
+local server = require "net.server";
+local watchers = {
+	error = function (_, err)
+		error(err);
+	end;
+	waiting = function ()
+		server.loop();
+	end;
+};
+
+local cm = require "core.configmanager";
+local hm = require "core.hostmanager";
+local sm = require "core.storagemanager";
+local um = require "core.usermanager";
+
+local function users(store, host)
+	if store.users then
+		return store:users();
+	else
+		return um.users(host);
+	end
+end
 
-local json = require "util.json";
+local function prepare_config(host, conf)
+	if conf.type == "internal" then
+		sm.olddm.set_data_path(conf.path or prosody.paths.data);
+	elseif conf.type == "sql" then
+		cm.set(host, "sql", conf);
+	end
+end
 
-io.stderr:write("Migrating...\n");
-for x in reader do
-	--print(json.encode(x))
-	writer(x);
+local function get_driver(host, conf)
+	prepare_config(host, conf);
+	return assert(sm.load_driver(host, conf.type));
 end
-writer(nil); -- close
+
+local migration_runner = async.runner(function (job)
+	for host, stores in pairs(job.input.hosts) do
+		prosody.hosts[host] = startup.make_host(host);
+		sm.initialize_host(host);
+		um.initialize_host(host);
+
+		local input_driver = get_driver(host, job.input);
+
+		local output_driver = get_driver(host, job.output);
+
+		for _, store in ipairs(stores) do
+			local p, typ = store:match("()%-(%w+)$");
+			if typ then store = store:sub(1, p-1); else typ = "keyval"; end
+			log("info", "Migrating host %s store %s (%s)", host, store, typ);
+
+			local origin = assert(input_driver:open(store, typ));
+			local destination = assert(output_driver:open(store, typ));
+
+			if typ == "keyval" then -- host data
+				local data, err = origin:get(nil);
+				assert(not err, err);
+				assert(destination:set(nil, data));
+			end
+
+			for user in users(origin, host) do
+				if typ == "keyval" then
+					local data, err = origin:get(user);
+					assert(not err, err);
+					assert(destination:set(user, data));
+				elseif typ == "archive" then
+					local iter, err = origin:find(user);
+					assert(iter, err);
+					for id, item, when, with in iter do
+						assert(destination:append(user, id, item, when, with));
+					end
+				else
+					error("Don't know how to migrate data of type '"..typ.."'.");
+				end
+			end
+		end
+	end
+end, watchers);
+
+io.stderr:write("Migrating...\n");
+
+migration_runner:run({ input = config[from_store], output = config[to_store] });
+
 io.stderr:write("Done!\n");
diff --git a/tools/modtrace.lua b/tools/modtrace.lua
new file mode 100644
index 00000000..45fa9f6a
--- /dev/null
+++ b/tools/modtrace.lua
@@ -0,0 +1,159 @@
+-- Trace module calls and method calls on created objects
+--
+-- Very rough and for debugging purposes only. It makes many
+-- assumptions and there are many ways it could fail.
+--
+-- Example use:
+--
+--   local dbuffer = require "tools.modtrace".trace("util.dbuffer");
+--
+
+local t_pack = require "util.table".pack;
+local serialize = require "util.serialization".serialize;
+local unpack = table.unpack or unpack; --luacheck: ignore 113
+local set = require "util.set";
+
+local serialize_cfg = {
+	preset = "oneline";
+	freeze = true;
+	fatal = false;
+	fallback = function (v) return "<"..tostring(v)..">" end;
+};
+
+local function stringify_value(v)
+	if type(v) == "string" and #v > 20 then
+		return ("<string(%d)>"):format(#v);
+	elseif type(v) == "function" then
+		return tostring(v);
+	end
+	return serialize(v, serialize_cfg);
+end
+
+local function stringify_params(...)
+	local n = select("#", ...);
+	local r = {};
+	for i = 1, n do
+		table.insert(r, stringify_value((select(i, ...))));
+	end
+	return table.concat(r, ", ");
+end
+
+local function stringify_result(ret)
+	local r = {};
+	for i = 1, ret.n do
+		table.insert(r, stringify_value(ret[i]));
+	end
+	return table.concat(r, ", ");
+end
+
+local function stringify_call(method_name, ...)
+	return ("%s(%s)"):format(method_name, stringify_params(...));
+end
+
+local function wrap_method(original_obj, original_method, method_name)
+	method_name = ("<%s>:%s"):format(getmetatable(original_obj).__name or "object", method_name);
+	return function (new_obj_self, ...)
+		local opts = new_obj_self._modtrace_opts;
+		local f = opts.output or io.stderr;
+		f:write(stringify_call(method_name, ...));
+		local ret = t_pack(original_method(original_obj, ...));
+		if ret.n > 0 then
+			f:write(" = ", stringify_result(ret), "\n");
+		else
+			f:write("\n");
+		end
+		return unpack(ret, 1, ret.n);
+	end;
+end
+
+local function wrap_function(original_function, function_name, opts)
+	local f = opts.output or io.stderr;
+	return function (...)
+		f:write(stringify_call(function_name, ...));
+		local ret = t_pack(original_function(...));
+		if ret.n > 0 then
+			f:write(" = ", stringify_result(ret), "\n");
+		else
+			f:write("\n");
+		end
+		return unpack(ret, 1, ret.n);
+	end;
+end
+
+local function wrap_metamethod(name, method)
+	if name == "__index" then
+		return function (new_obj, k)
+			local original_method;
+			if type(method) == "table" then
+				original_method = new_obj._modtrace_original_obj[k];
+			else
+				original_method = method(new_obj._modtrace_original_obj, k);
+			end
+			if original_method == nil then
+				return nil;
+			end
+			return wrap_method(new_obj._modtrace_original_obj, original_method, k);
+		end;
+	end
+	return function (new_obj, ...)
+		return method(new_obj._modtrace_original_obj, ...);
+	end;
+end
+
+local function wrap_mt(original_mt)
+	local new_mt = {};
+	for k, v in pairs(original_mt) do
+		new_mt[k] = wrap_metamethod(k, v);
+	end
+	return new_mt;
+end
+
+local function wrap_obj(original_obj, opts)
+	local new_mt = wrap_mt(getmetatable(original_obj));
+	return setmetatable({_modtrace_original_obj = original_obj, _modtrace_opts = opts}, new_mt);
+end
+
+local function wrap_new(original_new, function_name, opts)
+	local f = opts.output or io.stderr;
+	return function (...)
+		f:write(stringify_call(function_name, ...));
+		local ret = t_pack(original_new(...));
+		local obj = ret[1];
+
+		if ret.n == 1 and type(ret[1]) == "table" then
+			f:write(" = <", getmetatable(ret[1]).__name or "object", ">", "\n");
+		elseif ret.n > 0 then
+			f:write(" = ", stringify_result(ret), "\n");
+		else
+			f:write("\n");
+		end
+
+		if obj then
+			ret[1] = wrap_obj(obj, opts);
+		end
+		return unpack(ret, 1, ret.n);
+	end;
+end
+
+local function trace(module, opts)
+	if type(module) == "string" then
+		module = require(module);
+	end
+	opts = opts or {};
+	local new_methods = set.new(opts.new_methods or {"new"});
+	local fake_module = setmetatable({}, {
+		__index = function (_, k)
+			if new_methods:contains(k) then
+				return wrap_new(module[k], k, opts);
+			else
+				return wrap_function(module[k], k, opts);
+			end
+		end;
+	});
+	return fake_module;
+end
+
+return {
+	wrap = trace;
+	trace = trace;
+}
diff --git a/tools/tb2err b/tools/tb2err
new file mode 100755
index 00000000..7b676813
--- /dev/null
+++ b/tools/tb2err
@@ -0,0 +1,21 @@
+#!/usr/bin/env lua-any
+-- Lua-Versions: 5.3 5.2 5.1
+-- traceback to errors.err for vim -q
+local path_sep = package.config:sub(1,1);
+for line in io.lines() do
+	local src, err = line:match("%s*(%S+)(:%d+: .*)")
+	if src then
+		src = src:gsub("\\", path_sep);
+		local cut = src:match("/()core/")
+			or src:match("/()net/")
+			or src:match("/()util/")
+			or src:match("/()modules/")
+			or src:match("/()plugins/")
+			or src:match("/()prosody[ctl]*$") 
+		if cut then
+			src = src:sub(cut);
+		end
+		src = src:gsub("^modules/", "plugins/")
+		io.write(src, err, "\n");
+	end
+end
diff --git a/tools/xep227toprosody.lua b/tools/xep227toprosody.lua
index 521851e9..af8e3f1c 100755
--- a/tools/xep227toprosody.lua
+++ b/tools/xep227toprosody.lua
@@ -67,7 +67,7 @@ function store_password(username, host, password)
 end
 
 function store_roster(username, host, roster_items)
-	-- fetch current roster-table for username@host if he already has one
+	-- fetch current roster-table for username@host if they already have one
 	local roster = dm.load(username, host, "roster") or {};
 	-- merge imported roster-items with loaded roster
 	for item_tag in roster_items:childtags("item") do
@@ -129,7 +129,7 @@ end
 function store_subscription_request(username, host, presence_stanza)
 	local from_bare = presence_stanza.attr.from;
 
-	-- fetch current roster-table for username@host if he already has one
+	-- fetch current roster-table for username@host if they already have one
 	local roster = dm.load(username, host, "roster") or {};
 
 	local item = roster[from_bare];