aboutsummaryrefslogtreecommitdiffstats
path: root/tools/migration/prosody-migrator.lua
diff options
context:
space:
mode:
Diffstat (limited to 'tools/migration/prosody-migrator.lua')
-rw-r--r--tools/migration/prosody-migrator.lua244
1 files changed, 192 insertions, 52 deletions
diff --git a/tools/migration/prosody-migrator.lua b/tools/migration/prosody-migrator.lua
index 1219d891..21eb32e7 100644
--- a/tools/migration/prosody-migrator.lua
+++ b/tools/migration/prosody-migrator.lua
@@ -1,42 +1,84 @@
#!/usr/bin/env lua
-CFG_SOURCEDIR=os.getenv("PROSODY_SRCDIR");
-CFG_CONFIGDIR=os.getenv("PROSODY_CFGDIR");
+CFG_SOURCEDIR=CFG_SOURCEDIR or os.getenv("PROSODY_SRCDIR");
+CFG_CONFIGDIR=CFG_CONFIGDIR or os.getenv("PROSODY_CFGDIR");
+CFG_PLUGINDIR=CFG_PLUGINDIR or os.getenv("PROSODY_PLUGINDIR");
+CFG_DATADIR=CFG_DATADIR or os.getenv("PROSODY_DATADIR");
--- Substitute ~ with path to home directory in paths
-if CFG_CONFIGDIR then
- CFG_CONFIGDIR = CFG_CONFIGDIR:gsub("^~", os.getenv("HOME"));
+-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
+
+local function is_relative(path)
+ local path_sep = package.config:sub(1,1);
+ return ((path_sep == "/" and path:sub(1,1) ~= "/")
+ or (path_sep == "\\" and (path:sub(1,1) ~= "/" and path:sub(2,3) ~= ":\\")))
end
+-- Tell Lua where to find our libraries
if CFG_SOURCEDIR then
- CFG_SOURCEDIR = CFG_SOURCEDIR:gsub("^~", os.getenv("HOME"));
+ local function filter_relative_paths(path)
+ if is_relative(path) then return ""; end
+ end
+ local function sanitise_paths(paths)
+ return (paths:gsub("[^;]+;?", filter_relative_paths):gsub(";;+", ";"));
+ end
+ package.path = sanitise_paths(CFG_SOURCEDIR.."/?.lua;"..package.path);
+ package.cpath = sanitise_paths(CFG_SOURCEDIR.."/?.so;"..package.cpath);
+end
+
+-- Substitute ~ with path to home directory in data path
+if CFG_DATADIR then
+ if os.getenv("HOME") then
+ CFG_DATADIR = CFG_DATADIR:gsub("^~", os.getenv("HOME"));
+ end
end
local default_config = (CFG_CONFIGDIR or ".").."/migrator.cfg.lua";
--- Command-line parsing
-local options = {};
-local i = 1;
-while arg[i] do
- if arg[i]:sub(1,2) == "--" then
- local opt, val = arg[i]:match("([%w-]+)=?(.*)");
- if opt then
- options[(opt:sub(3):gsub("%-", "_"))] = #val > 0 and val or true;
- end
- table.remove(arg, i);
- else
- i = i + 1;
- end
+local function usage()
+ print("Usage: " .. arg[0] .. " [OPTIONS] FROM_STORE TO_STORE");
+ print(" --config FILE Specify config file")
+ print(" --keep-going Keep going in case of errors");
+ print(" -v, --verbose Increase log-level");
+ print("");
+ print("If no stores are specified, 'input' and 'output' are used.");
end
-if CFG_SOURCEDIR then
- package.path = CFG_SOURCEDIR.."/?.lua;"..package.path;
- package.cpath = CFG_SOURCEDIR.."/?.so;"..package.cpath;
-else
- package.path = "../../?.lua;"..package.path
- package.cpath = "../../?.so;"..package.cpath
+local startup = require "util.startup";
+do
+ startup.parse_args({
+ short_params = { v = "verbose", h = "help", ["?"] = "help" };
+ value_params = { config = true };
+ });
+ startup.init_global_state();
+ prosody.process_type = "migrator";
+ if prosody.opts.help then
+ usage();
+ os.exit(0);
+ end
+ startup.force_console_logging();
+ startup.init_logging();
+ startup.init_gc();
+ startup.init_errors();
+ startup.setup_plugindir();
+ startup.setup_plugin_install_path();
+ startup.setup_datadir();
+ startup.chdir();
+ startup.read_version();
+ startup.switch_user();
+ startup.check_dependencies();
+ startup.log_startup_warnings();
+ prosody.config_loaded = true;
+ startup.load_libraries();
+ startup.init_http_client();
+ prosody.core_post_stanza = function ()
+ -- silence assert in core.moduleapi
+ error("Attempt to send stanzas from inside migrator.", 0);
+ end
end
+-- Command-line parsing
+local options = prosody.opts;
+
local envloadfile = require "util.envload".envloadfile;
local config_file = options.config or default_config;
@@ -69,28 +111,17 @@ if not config[to_store] then
print("Error: Output store '"..to_store.."' not found in the config file.");
end
-function load_store_handler(name)
- local store_type = config[name].type;
- if not store_type then
- print("Error: "..name.." store type not specified in the config file");
- return false;
- else
- local ok, err = pcall(require, "migrator."..store_type);
- if not ok then
- print(("Error: Failed to initialize '%s' store:\n\t%s")
- :format(name, err));
- return false;
- end
+for store, conf in pairs(config) do -- COMPAT
+ if conf.type == "prosody_files" then
+ conf.type = "internal";
+ elseif conf.type == "prosody_sql" then
+ conf.type = "sql";
end
- return true;
end
-have_err = have_err or not(load_store_handler(from_store, "input") and load_store_handler(to_store, "output"));
-
if have_err then
print("");
- print("Usage: "..arg[0].." FROM_STORE TO_STORE");
- print("If no stores are specified, 'input' and 'output' are used.");
+ usage();
print("");
print("The available stores in your migrator config are:");
print("");
@@ -101,17 +132,126 @@ if have_err then
os.exit(1);
end
-local itype = config[from_store].type;
-local otype = config[to_store].type;
-local reader = require("migrator."..itype).reader(config[from_store]);
-local writer = require("migrator."..otype).writer(config[to_store]);
+local async = require "util.async";
+local server = require "net.server";
+local watchers = {
+ error = function (_, err)
+ error(err);
+ end;
+ waiting = function ()
+ server.loop();
+ end;
+};
-local json = require "util.json";
+local cm = require "core.configmanager";
+local hm = require "core.hostmanager";
+local sm = require "core.storagemanager";
+local um = require "core.usermanager";
-io.stderr:write("Migrating...\n");
-for x in reader do
- --print(json.encode(x))
- writer(x);
+local function users(store, host)
+ if store.users then
+ log("debug", "Using store user iterator")
+ return store:users();
+ else
+ log("debug", "Using usermanager user iterator")
+ return um.users(host);
+ end
+end
+
+local function prepare_config(host, conf)
+ if conf.type == "internal" then
+ sm.olddm.set_data_path(conf.path or prosody.paths.data);
+ elseif conf.type == "sql" then
+ cm.set(host, "sql", conf);
+ end
+end
+
+local function get_driver(host, conf)
+ prepare_config(host, conf);
+ return assert(sm.load_driver(host, conf.type));
+end
+
+local migrate_once = {
+ keyval = function(origin, destination, user)
+ local data, err = origin:get(user);
+ assert(not err, err);
+ assert(destination:set(user, data));
+ end;
+ archive = function(origin, destination, user)
+ local iter, err = origin:find(user);
+ assert(iter, err);
+ for id, item, when, with in iter do
+ assert(destination:append(user, id, item, when, with));
+ end
+ end;
+}
+migrate_once.pubsub = function(origin, destination, user, prefix, input_driver, output_driver)
+ if not user and prefix == "pubsub_" then return end
+ local data, err = origin:get(user);
+ assert(not err, err);
+ if not data then return end
+ assert(destination:set(user, data));
+ if prefix == "pubsub_" then user = nil end
+ for node in pairs(data) do
+ local pep_origin = assert(input_driver:open(prefix .. node, "archive"));
+ local pep_destination = assert(output_driver:open(prefix .. node, "archive"));
+ migrate_once.archive(pep_origin, pep_destination, user);
+ end
end
-writer(nil); -- close
+
+if options["keep-going"] then
+ local xpcall = require "util.xpcall".xpcall;
+ for t, f in pairs(migrate_once) do
+ migrate_once[t] = function (origin, destination, user, ...)
+ local function log_err(err)
+ if user then
+ log("error", "Error migrating data for user %q: %s", user, err);
+ else
+ log("error", "Error migrating data for host: %s", err);
+ end
+ log("debug", "%s", debug.traceback(nil, 2));
+ end
+ xpcall(f, log_err, origin, destination, user, ...);
+ end
+ end
+end
+
+local migration_runner = async.runner(function (job)
+ for host, stores in pairs(job.input.hosts) do
+ prosody.hosts[host] = startup.make_host(host);
+ sm.initialize_host(host);
+ um.initialize_host(host);
+
+ local input_driver = get_driver(host, job.input);
+
+ local output_driver = get_driver(host, job.output);
+
+ for _, store in ipairs(stores) do
+ local p, typ = store:match("()%-(%w+)$");
+ if typ then store = store:sub(1, p-1); else typ = "keyval"; end
+ log("info", "Migrating host %s store %s (%s)", host, store, typ);
+
+ local migrate = assert(migrate_once[typ], "Unknown store type: "..typ);
+
+ local prefix = store .. "_";
+ if typ == "pubsub" then typ = "keyval"; end
+ if store == "pubsub_nodes" then prefix = "pubsub_"; end
+
+ local origin = assert(input_driver:open(store, typ));
+ local destination = assert(output_driver:open(store, typ));
+
+ migrate(origin, destination, nil, prefix, input_driver, output_driver); -- host data
+
+ for user in users(origin, host) do
+ log("info", "Migrating user %s@%s store %s (%s)", user, host, store, typ);
+ migrate(origin, destination, user, prefix, input_driver, output_driver);
+ end
+ end
+ end
+end, watchers);
+
+io.stderr:write("Migrating...\n");
+
+migration_runner:run({ input = config[from_store], output = config[to_store] });
+
io.stderr:write("Done!\n");