migrator: Rewrite to use storage modules

This allows migrating to and from any storage module that supports the
right methods. Based on experimental mod_migrate work.

6 files changed, 142 insertions, 439 deletions

diff --git a/tools/migration/Makefile b/tools/migration/Makefile
index 713831d2..dc14e0da 100644
--- a/tools/migration/Makefile
+++ b/tools/migration/Makefile
@@ -12,16 +12,13 @@ INSTALLEDCONFIG = $(SYSCONFDIR)
 INSTALLEDMODULES = $(LIBDIR)/prosody/modules
 INSTALLEDDATA = $(DATADIR)
 
-SOURCE_FILES = migrator/*.lua
-
-all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua $(SOURCE_FILES)
+all: prosody-migrator.install migrator.cfg.lua.install prosody-migrator.lua
 
 install: prosody-migrator.install migrator.cfg.lua.install
-	install -d $(BIN) $(CONFIG) $(SOURCE) $(SOURCE)/migrator
+	install -d $(BIN) $(CONFIG) $(SOURCE)
 	install -d $(MAN)/man1
 	install -d $(SOURCE)/migrator
 	install -m755 ./prosody-migrator.install $(BIN)/prosody-migrator
-	install -m644 $(SOURCE_FILES) $(SOURCE)/migrator
 	test -e $(CONFIG)/migrator.cfg.lua || install -m644 migrator.cfg.lua.install $(CONFIG)/migrator.cfg.lua
 
 clean:
diff --git a/tools/migration/migrator.cfg.lua b/tools/migration/migrator.cfg.lua
index fa37f2a3..c26fa869 100644
--- a/tools/migration/migrator.cfg.lua
+++ b/tools/migration/migrator.cfg.lua
@@ -1,12 +1,38 @@
 local data_path = "../../data";
 
+local vhost = {
+	"accounts",
+	"account_details",
+	"roster",
+	"vcard",
+	"private",
+	"blocklist",
+	"privacy",
+	"archive-archive",
+	"offline-archive",
+	"pubsub_nodes",
+	-- "pubsub_*-archive",
+	"pep",
+	-- "pep_*-archive",
+}
+local muc = {
+	"persistent",
+	"config",
+	"state",
+	"muc_log-archive",
+};
+
 input {
-	type = "prosody_files";
+	hosts = {
+		["example.com"] = vhost;
+		["conference.example.com"] = muc;
+	};
+	type = "internal";
 	path = data_path;
 }
 
 output {
-	type = "prosody_sql";
+	type = "sql";
 	driver = "SQLite3";
 	database = data_path.."/prosody.sqlite";
 }
@@ -14,11 +40,11 @@ output {
 --[[
 
 input {
-	type = "prosody_files";
+	type = "internal";
 	path = data_path;
 }
 output {
-	type = "prosody_sql";
+	type = "sql";
 	driver = "SQLite3";
 	database = data_path.."/prosody.sqlite";
 }
diff --git a/tools/migration/migrator/mtools.lua b/tools/migration/migrator/mtools.lua
deleted file mode 100644
index cfbfcce8..00000000
--- a/tools/migration/migrator/mtools.lua
+++ /dev/null
@@ -1,58 +0,0 @@
-
-
-local print = print;
-local t_insert = table.insert;
-local t_sort = table.sort;
-
-
-local function sorted(params)
-
-	local reader = params.reader; -- iterator to get items from
-	local sorter = params.sorter; -- sorting function
-	local filter = params.filter; -- filter function
-
-	local cache = {};
-	for item in reader do
-		if filter then item = filter(item); end
-		if item then t_insert(cache, item); end
-	end
-	if sorter then
-		t_sort(cache, sorter);
-	end
-	local i = 0;
-	return function()
-		i = i + 1;
-		return cache[i];
-	end;
-
-end
-
-local function merged(reader, merger)
-
-	local item1 = reader();
-	local merged = { item1 };
-	return function()
-		while true do
-			if not item1 then return nil; end
-			local item2 = reader();
-			if not item2 then item1 = nil; return merged; end
-			if merger(item1, item2) then
-			--print("merged")
-				item1 = item2;
-				t_insert(merged, item1);
-			else
-			--print("unmerged", merged)
-				item1 = item2;
-				local tmp = merged;
-				merged = { item1 };
-				return tmp;
-			end
-		end
-	end;
-
-end
-
-return {
-	sorted = sorted;
-	merged = merged;
-}
diff --git a/tools/migration/migrator/prosody_files.lua b/tools/migration/migrator/prosody_files.lua
deleted file mode 100644
index 4de09273..00000000
--- a/tools/migration/migrator/prosody_files.lua
+++ /dev/null
@@ -1,144 +0,0 @@
-
-local print = print;
-local assert = assert;
-local setmetatable = setmetatable;
-local tonumber = tonumber;
-local char = string.char;
-local coroutine = coroutine;
-local lfs = require "lfs";
-local loadfile = loadfile;
-local pcall = pcall;
-local mtools = require "migrator.mtools";
-local next = next;
-local pairs = pairs;
-local json = require "util.json";
-local os_getenv = os.getenv;
-local error = error;
-
-prosody = {};
-local dm = require "util.datamanager"
-
-
-local function is_dir(path) return lfs.attributes(path, "mode") == "directory"; end
-local function is_file(path) return lfs.attributes(path, "mode") == "file"; end
-local function clean_path(path)
-	return path:gsub("\\", "/"):gsub("//+", "/"):gsub("^~", os_getenv("HOME") or "~");
-end
-local encode, decode; do
-	local urlcodes = setmetatable({}, { __index = function (t, k) t[k] = char(tonumber("0x"..k)); return t[k]; end });
-	decode = function (s) return s and (s:gsub("+", " "):gsub("%%([a-fA-F0-9][a-fA-F0-9])", urlcodes)); end
-	encode = function (s) return s and (s:gsub("%W", function (c) return format("%%%02x", c:byte()); end)); end
-end
-local function decode_dir(x)
-	if x:gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
-		return decode(x);
-	end
-end
-local function decode_file(x)
-	if x:match(".%.dat$") and x:gsub("%.dat$", ""):gsub("%%%x%x", ""):gsub("[a-zA-Z0-9]", "") == "" then
-		return decode(x:gsub("%.dat$", ""));
-	end
-end
-local function prosody_dir(path, ondir, onfile, ...)
-	for x in lfs.dir(path) do
-		local xpath = path.."/"..x;
-		if decode_dir(x) and is_dir(xpath) then
-			ondir(xpath, x, ...);
-		elseif decode_file(x) and is_file(xpath) then
-			onfile(xpath, x, ...);
-		end
-	end
-end
-
-local function handle_root_file(path, name)
-	--print("root file: ", decode_file(name))
-	coroutine.yield { user = nil, host = nil, store = decode_file(name) };
-end
-local function handle_host_file(path, name, host)
-	--print("host file: ", decode_dir(host).."/"..decode_file(name))
-	coroutine.yield { user = nil, host = decode_dir(host), store = decode_file(name) };
-end
-local function handle_store_file(path, name, store, host)
-	--print("store file: ", decode_file(name).."@"..decode_dir(host).."/"..decode_dir(store))
-	coroutine.yield { user = decode_file(name), host = decode_dir(host), store = decode_dir(store) };
-end
-local function handle_host_store(path, name, host)
-	prosody_dir(path, function() end, handle_store_file, name, host);
-end
-local function handle_host_dir(path, name)
-	prosody_dir(path, handle_host_store, handle_host_file, name);
-end
-local function handle_root_dir(path)
-	prosody_dir(path, handle_host_dir, handle_root_file);
-end
-
-local function decode_user(item)
-	local userdata = {
-		user = item[1].user;
-		host = item[1].host;
-		stores = {};
-	};
-	for i=1,#item do -- loop over stores
-		local result = {};
-		local store = item[i];
-		userdata.stores[store.store] = store.data;
-		store.user = nil; store.host = nil; store.store = nil;
-	end
-	return userdata;
-end
-
-local function reader(input)
-	local path = clean_path(assert(input.path, "no input.path specified"));
-	assert(is_dir(path), "input.path is not a directory");
-	local iter = coroutine.wrap(function()handle_root_dir(path);end);
-	-- get per-user stores, sorted
-	local iter = mtools.sorted {
-		reader = function()
-			local x = iter();
-			while x do
-				dm.set_data_path(path);
-				local err;
-				x.data, err = dm.load(x.user, x.host, x.store);
-				if x.data == nil and err then
-					local p = dm.getpath(x.user, x.host, x.store);
-					print(("Error loading data at path %s for %s@%s (%s store): %s")
-						:format(p, x.user or "<nil>", x.host or "<nil>", x.store or "<nil>", err or "<nil>"));
-				else
-					return x;
-				end
-				x = iter();
-			end
-		end;
-		sorter = function(a, b)
-			local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";
-			local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";
-			return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);
-		end;
-	};
-	-- merge stores to get users
-	iter = mtools.merged(iter, function(a, b)
-		return (a.host == b.host and a.user == b.user);
-	end);
-
-	return function()
-		local x = iter();
-		return x and decode_user(x);
-	end
-end
-
-local function writer(output)
-	local path = clean_path(assert(output.path, "no output.path specified"));
-	assert(is_dir(path), "output.path is not a directory");
-	return function(item)
-		if not item then return; end -- end of input
-		dm.set_data_path(path);
-		for store, data in pairs(item.stores) do
-			assert(dm.store(item.user, item.host, store, data));
-		end
-	end
-end
-
-return {
-	reader = reader;
-	writer = writer;
-}
diff --git a/tools/migration/migrator/prosody_sql.lua b/tools/migration/migrator/prosody_sql.lua
deleted file mode 100644
index 6df2b025..00000000
--- a/tools/migration/migrator/prosody_sql.lua
+++ /dev/null
@@ -1,190 +0,0 @@
-
-local assert = assert;
-local have_DBI = pcall(require,"DBI");
-local print = print;
-local type = type;
-local next = next;
-local pairs = pairs;
-local t_sort = table.sort;
-local json = require "util.json";
-local mtools = require "migrator.mtools";
-local tostring = tostring;
-local tonumber = tonumber;
-
-if not have_DBI then
-	error("LuaDBI (required for SQL support) was not found, please see https://prosody.im/doc/depends#luadbi", 0);
-end
-
-local sql = require "util.sql";
-
-local function create_table(engine, name) -- luacheck: ignore 431/engine
-	local Table, Column, Index = sql.Table, sql.Column, sql.Index;
-
-	local ProsodyTable = Table {
-		name= name or "prosody";
-		Column { name="host", type="TEXT", nullable=false };
-		Column { name="user", type="TEXT", nullable=false };
-		Column { name="store", type="TEXT", nullable=false };
-		Column { name="key", type="TEXT", nullable=false };
-		Column { name="type", type="TEXT", nullable=false };
-		Column { name="value", type="MEDIUMTEXT", nullable=false };
-		Index { name="prosody_index", "host", "user", "store", "key" };
-	};
-	engine:transaction(function()
-		ProsodyTable:create(engine);
-	end);
-
-end
-
-local function serialize(value)
-	local t = type(value);
-	if t == "string" or t == "boolean" or t == "number" then
-		return t, tostring(value);
-	elseif t == "table" then
-		local value,err = json.encode(value);
-		if value then return "json", value; end
-		return nil, err;
-	end
-	return nil, "Unhandled value type: "..t;
-end
-local function deserialize(t, value)
-	if t == "string" then return value;
-	elseif t == "boolean" then
-		if value == "true" then return true;
-		elseif value == "false" then return false; end
-	elseif t == "number" then return tonumber(value);
-	elseif t == "json" then
-		return json.decode(value);
-	end
-end
-
-local function decode_user(item)
-	local userdata = {
-		user = item[1][1].user;
-		host = item[1][1].host;
-		stores = {};
-	};
-	for i=1,#item do -- loop over stores
-		local result = {};
-		local store = item[i];
-		for i=1,#store do -- loop over store data
-			local row = store[i];
-			local k = row.key;
-			local v = deserialize(row.type, row.value);
-			if k and v then
-				if k ~= "" then result[k] = v; elseif type(v) == "table" then
-					for a,b in pairs(v) do
-						result[a] = b;
-					end
-				end
-			end
-			userdata.stores[store[1].store] = result;
-		end
-	end
-	return userdata;
-end
-
-local function needs_upgrade(engine, params)
-	if params.driver == "MySQL" then
-		local success = engine:transaction(function()
-			local result = engine:execute("SHOW COLUMNS FROM prosody WHERE Field='value' and Type='text'");
-			assert(result:rowcount() == 0);
-
-			-- COMPAT w/pre-0.10: Upgrade table to UTF-8 if not already
-			local check_encoding_query = [[
-			SELECT "COLUMN_NAME","COLUMN_TYPE","TABLE_NAME"
-			FROM "information_schema"."columns"
-			WHERE "TABLE_NAME" LIKE 'prosody%%' AND ( "CHARACTER_SET_NAME"!='%s' OR "COLLATION_NAME"!='%s_bin' );
-			]];
-			check_encoding_query = check_encoding_query:format(engine.charset, engine.charset);
-			local result = engine:execute(check_encoding_query);
-			assert(result:rowcount() == 0)
-		end);
-		if not success then
-			-- Upgrade required
-			return true;
-		end
-	end
-	return false;
-end
-
-local function reader(input)
-	local engine = assert(sql:create_engine(input, function (engine) -- luacheck: ignore 431/engine
-		if needs_upgrade(engine, input) then
-			error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade");
-		end
-	end));
-	local keys = {"host", "user", "store", "key", "type", "value"};
-	assert(engine:connect());
-	local f,s,val = assert(engine:select("SELECT \"host\", \"user\", \"store\", \"key\", \"type\", \"value\" FROM \"prosody\";"));
-	-- get SQL rows, sorted
-	local iter = mtools.sorted {
-		reader = function() val = f(s, val); return val; end;
-		filter = function(x)
-			for i=1,#keys do
-				x[ keys[i] ] = x[i];
-			end
-			if x.host  == "" then x.host  = nil; end
-			if x.user  == "" then x.user  = nil; end
-			if x.store == "" then x.store = nil; end
-			return x;
-		end;
-		sorter = function(a, b)
-			local a_host, a_user, a_store = a.host or "", a.user or "", a.store or "";
-			local b_host, b_user, b_store = b.host or "", b.user or "", b.store or "";
-			return a_host > b_host or (a_host==b_host and a_user > b_user) or (a_host==b_host and a_user==b_user and a_store > b_store);
-		end;
-	};
-	-- merge rows to get stores
-	iter = mtools.merged(iter, function(a, b)
-		return (a.host == b.host and a.user == b.user and a.store == b.store);
-	end);
-	-- merge stores to get users
-	iter = mtools.merged(iter, function(a, b)
-		return (a[1].host == b[1].host and a[1].user == b[1].user);
-	end);
-	return function()
-		local x = iter();
-		return x and decode_user(x);
-	end;
-end
-
-local function writer(output, iter)
-	local engine = assert(sql:create_engine(output, function (engine) -- luacheck: ignore 431/engine
-		if needs_upgrade(engine, output) then
-			error("Old database format detected. Please run: prosodyctl mod_storage_sql upgrade");
-		end
-		create_table(engine);
-	end));
-	assert(engine:connect());
-	assert(engine:delete("DELETE FROM \"prosody\""));
-	local insert_sql = "INSERT INTO \"prosody\" (\"host\",\"user\",\"store\",\"key\",\"type\",\"value\") VALUES (?,?,?,?,?,?)";
-
-	return function(item)
-		if not item then assert(engine.conn:commit()) return end -- end of input
-		local host = item.host or "";
-		local user = item.user or "";
-		for store, data in pairs(item.stores) do
-			-- TODO transactions
-			local extradata = {};
-			for key, value in pairs(data) do
-				if type(key) == "string" and key ~= "" then
-					local t, value = assert(serialize(value));
-					local ok, err = assert(engine:insert(insert_sql, host, user, store, key, t, value));
-				else
-					extradata[key] = value;
-				end
-			end
-			if next(extradata) ~= nil then
-				local t, extradata = assert(serialize(extradata));
-				local ok, err = assert(engine:insert(insert_sql, host, user, store, "", t, extradata));
-			end
-		end
-	end;
-end
-
-
-return {
-	reader = reader;
-	writer = writer;
-}
diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua
index 1219d891..3cc10ecf 100644
--- a/tools/migration/prosody-migrator.lua
+++ b/tools/migration/prosody-migrator.lua
@@ -1,19 +1,43 @@
 #!/usr/bin/env lua
 
-CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR");
-CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR");
+CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR");
+CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR");
+CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR");
+CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR");
 
--- Substitute ~ with path to home directory in paths
-if CFG_CONFIGDIR then
-	CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME"));
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+local function is_relative(path)
+	local path_sep = package.config:sub(1,1);
+        return ((path_sep == "/" and path:sub(1,1) ~= "/")
+	or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\")))
 end
 
+-- Tell Lua where to find our libraries
 if CFG_SOURCEDIR then
-	CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME"));
+	local function filter_relative_paths(path)
+		if is_relative(path) then return ""; end
+	end
+	local function sanitise_paths(paths)
+		return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";"));
+	end
+	package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path);
+	package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath);
+end
+
+-- Substitute ~ with path to home directory in data path
+if CFG_DATADIR then
+	if os.getenv("HOME") then
+		CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME"));
+	end
 end
 
 local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua";
 
+local startup = require "util.startup";
+startup.prosodyctl();
+-- TODO startup.migrator ?
+
 -- Command-line parsing
 local options = {};
 local i = 1;
@@ -29,13 +53,6 @@ while arg[i] do
 	end
 end
 
-if CFG_SOURCEDIR then
-	package.path = CFG_SOURCEDIR.."/?.lua;"..package.path;
-	package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath;
-else
-	package.path = "../../?.lua;"..package.path
-	package.cpath = "../../?.so;"..package.cpath
-end
 
 local envloadfile = require "util.envload".envloadfile;
 
@@ -69,24 +86,14 @@ if not config[to_store] then
 	print("Error: Output store '"..to_store.."' not found in the config file.");
 end
 
-function load_store_handler(name)
-	local store_type = config[name].type;
-	if not store_type then
-		print("Error: "..name.." store type not specified in the config file");
-		return false;
-	else
-		local ok, err = pcall(require, "migrator."..store_type);
-		if not ok then
-			print(("Error: Failed to initialize '%s' store:\n\t%s")
-				:format(name, err));
-			return false;
-		end
+for store, conf in pairs(config) do -- COMPAT
+	if conf.type == "prosody_files" then
+		conf.type = "internal";
+	elseif conf.type == "prosody_sql" then
+		conf.type = "sql";
 	end
-	return true;
 end
 
-have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output"));
-
 if have_err then
 	print("");
 	print("Usage: "..arg[0].." FROM_STORE TO_STORE");
@@ -101,17 +108,82 @@ if have_err then
 	os.exit(1);
 end
 
-local itype = config[from_store].type;
-local otype = config[to_store].type;
-local reader = require("migrator."..itype).reader(config[from_store]);
-local writer = require("migrator."..otype).writer(config[to_store]);
+local async = require "util.async";
+local server = require "net.server";
+local watchers = {
+	error = function (_, err)
+		error(err);
+	end;
+	waiting = function ()
+		server.loop();
+	end;
+};
+
+local cm = require "core.configmanager";
+local hm = require "core.hostmanager";
+local sm = require "core.storagemanager";
+local um = require "core.usermanager";
+
+local function users(store, host)
+	if store.users then
+		return store:users();
+	else
+		return um.users(host);
+	end
+end
 
-local json = require "util.json";
+local function prepare_config(host, conf)
+	if conf.type == "internal" then
+		sm.olddm.set_data_path(conf.path or prosody.paths.data);
+	elseif conf.type == "sql" then
+		cm.set(host, "sql", conf);
+	end
+end
 
-io.stderr:write("Migrating...\n");
-for x in reader do
-	--print(json.encode(x))
-	writer(x);
+local function get_driver(host, conf)
+	prepare_config(host, conf);
+	return assert(sm.load_driver(host, conf.type));
 end
-writer(nil); -- close
+
+local migration_runner = async.runner(function (job)
+	for host, stores in pairs(job.input.hosts) do
+		prosody.hosts[host] = startup.make_host(host);
+		sm.initialize_host(host);
+		um.initialize_host(host);
+
+		local input_driver = get_driver(host, job.input);
+
+		local output_driver = get_driver(host, job.output);
+
+		for _, store in ipairs(stores) do
+			local p, typ = store:match("()%-(%w+)$");
+			if typ then store = store:sub(1, p-1); else typ = "keyval"; end
+			log("info", "Migrating host %s store %s (%s)", host, store, typ);
+
+			local origin = assert(input_driver:open(store, typ));
+			local destination = assert(output_driver:open(store, typ));
+
+			if typ == "keyval" then -- host data
+				local data, err = origin:get(nil);
+				assert(not err, err);
+				assert(destination:set(nil, data));
+			end
+
+			for user in users(origin, host) do
+				if typ == "keyval" then
+					local data, err = origin:get(user);
+					assert(not err, err);
+					assert(destination:set(user, data));
+				else
+					error("Don't know how to migrate data of type '"..typ.."'.");
+				end
+			end
+		end
+	end
+end, watchers);
+
+io.stderr:write("Migrating...\n");
+
+migration_runner:run({ input = config[from_store], output = config[to_store] });
+
 io.stderr:write("Done!\n");