From e6b0ecf2435c94405c08f5ffd7d595f0b75cc8a4 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 29 Jun 2015 13:50:31 +0100 Subject: luacheckrc: Allow plugins to implement functions under module.* (module.load, etc.) --- .luacheckrc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.luacheckrc b/.luacheckrc index 3f147806..7ebc7af4 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -6,3 +6,7 @@ module = true unused_secondaries = false codes = true ignore = { "411/err", "421/err", "411/ok", "421/ok" } + +files["plugins/"] = { + ignore = { "122/module" }; +} -- cgit v1.2.3 From 5292b8232a0cb961cc2d301556a179505ba98dee Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Sun, 5 Jul 2015 22:20:29 +0200 Subject: prosodyctl: check: Add some more known global config options --- prosodyctl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prosodyctl b/prosodyctl index 6c4b148a..a67f3dbd 100755 --- a/prosodyctl +++ b/prosodyctl @@ -841,7 +841,8 @@ function commands.check(arg) }); local known_global_options = set.new({ "pidfile", "log", "plugin_paths", "prosody_user", "prosody_group", "daemonize", - "umask", "prosodyctl_timeout", "use_ipv6", "use_libevent", "network_settings" + "umask", "prosodyctl_timeout", "use_ipv6", "use_libevent", "network_settings", + "network_backend", "http_default_host", }); local config = config.getconfig(); -- Check that we have any global options (caused by putting a host at the top) -- cgit v1.2.3 From ff7b482c6d6005acde8da8a6d586f95fd477a555 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Mon, 6 Jul 2015 19:11:39 +0100 Subject: net.dns: Remove resolver:new(), as it needlessly and incorrectly duplicates dns.resolver() --- net/dns.lua | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/net/dns.lua b/net/dns.lua index 037f3ec6..992f3986 100644 --- a/net/dns.lua +++ b/net/dns.lua @@ -213,15 +213,6 @@ function cache_metatable.__tostring(cache) end -function resolver:new() -- - - - - - - - - - - - - - - - - - - - - resolver - local r = { active = {}, cache = {}, unsorted = {} }; - setmetatable(r, resolver); - setmetatable(r.cache, cache_metatable); - setmetatable(r.unsorted, { __mode = 'kv' }); - return r; -end - - -- packet layer -------------------------------------------------- packet layer @@ -1054,8 +1045,6 @@ end function dns.resolver () -- - - - - - - - - - - - - - - - - - - - - resolver - -- this function seems to be redundant with resolver.new () - local r = { active = {}, cache = {}, unsorted = {}, wanted = {}, best_server = 1 }; setmetatable (r, resolver); setmetatable (r.cache, cache_metatable); -- cgit v1.2.3 From 434b71bc6f4420b5af32fd085d5d13ac64d3e6f8 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 7 Jul 2015 17:39:56 +0100 Subject: mod_storage_sql2: Rename variable to avoid name clash --- plugins/mod_storage_sql2.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/mod_storage_sql2.lua b/plugins/mod_storage_sql2.lua index 10a16743..bba27748 100644 --- a/plugins/mod_storage_sql2.lua +++ b/plugins/mod_storage_sql2.lua @@ -13,8 +13,8 @@ local function is_stanza(x) return getmetatable(x) == stanza_mt; end local noop = function() end local unpack = unpack local function iterator(result) - return function(result) - local row = result(); + return function(result_) + local row = result_(); if row ~= nil then return unpack(row); end -- cgit v1.2.3 From 86822ee5e4dadd72ed000669dcbecebdf57f87cb Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 7 Jul 2015 17:40:23 +0100 Subject: mod_storage_sql2: Add missing return to upgrade_table(), so we know if an upgrade is required --- plugins/mod_storage_sql2.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/mod_storage_sql2.lua b/plugins/mod_storage_sql2.lua index bba27748..59c50b5a 100644 --- a/plugins/mod_storage_sql2.lua +++ b/plugins/mod_storage_sql2.lua @@ -376,6 +376,7 @@ local function upgrade_table(params, apply_changes) return false; end end + return changes; end local function normalize_params(params) -- cgit v1.2.3 From 68fa7831dada4d821ddff964723c376f287d4ed5 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 7 Jul 2015 17:41:09 +0100 Subject: mod_storage_sql2: Add prosodyctl command to upgrade tables from the command-line --- plugins/mod_storage_sql2.lua | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/plugins/mod_storage_sql2.lua b/plugins/mod_storage_sql2.lua index 59c50b5a..b040cb05 100644 --- a/plugins/mod_storage_sql2.lua +++ b/plugins/mod_storage_sql2.lua @@ -405,3 +405,36 @@ function module.load() module:provides("storage", driver); end + +function module.command(arg) + local config = require "core.configmanager"; + local prosodyctl = require "util.prosodyctl"; + local command = table.remove(arg, 1); + if command == "upgrade" then + -- We need to find every unique dburi in the config + local uris = {}; + for host in pairs(prosody.hosts) do + local params = config.get(host, "sql") or default_params; + uris[sql.db2uri(params)] = params; + end + print("We will check and upgrade the following databases:\n"); + for _, params in pairs(uris) do + print("", "["..params.driver.."] "..params.database..(params.host and " on "..params.host or "")); + end + print(""); + print("Ensure you have working backups of the above databases before continuing! "); + if not prosodyctl.show_yesno("Continue with the database upgrade? [yN]") then + print("Ok, no upgrade. But you do have backups, don't you? ...don't you?? :-)"); + return; + end + -- Upgrade each one + for _, params in pairs(uris) do + print("Checking "..params.database.."..."); + engine = sql:create_engine(params); + upgrade_table(params, true); + end + print("All done!"); + else + print("Unknown command: "..command); + end +end -- cgit v1.2.3 From 7573762c919f99025acd0e1cac7ec9d3df82b852 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 7 Jul 2015 17:42:29 +0100 Subject: mod_storage_sql2: Don't upgrade tables without admin intervention --- plugins/mod_storage_sql2.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/mod_storage_sql2.lua b/plugins/mod_storage_sql2.lua index b040cb05..a9d82214 100644 --- a/plugins/mod_storage_sql2.lua +++ b/plugins/mod_storage_sql2.lua @@ -396,8 +396,8 @@ function module.load() -- FIXME: we should check in information_schema, etc. create_table(); -- Check whether the table needs upgrading - if not upgrade_table(params, true) then - module:log("error", "Old database format detected, and upgrade failed"); + if upgrade_table(params, false) then + module:log("error", "Old database format detected. Please run: prosodyctl mod_%s upgrade", module.name); return false, "database upgrade needed"; end end -- cgit v1.2.3 From 137f76396600f44532054db2829d1e7ae2fa2cb6 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Tue, 7 Jul 2015 17:43:14 +0100 Subject: util.sql: Allow onconnect callback to fail connection to the DB by returning false, err --- util/sql.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/util/sql.lua b/util/sql.lua index 92032f43..dedef435 100644 --- a/util/sql.lua +++ b/util/sql.lua @@ -111,7 +111,10 @@ function engine:connect() self.conn = dbh; self.prepared = {}; self:set_encoding(); - self:onconnect(); + local ok, err = self:onconnect(); + if ok == false then + return ok, err; + end return true; end function engine:onconnect() -- cgit v1.2.3 From 7a4ec3c57c8e38c49f6527ea00abea9a5dd3ada6 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 8 Jul 2015 15:04:23 +0100 Subject: util.sql: Create table with same charset as the charset we selected for our connection, also use corresponding _bin collation --- util/sql.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/sql.lua b/util/sql.lua index dedef435..562ac709 100644 --- a/util/sql.lua +++ b/util/sql.lua @@ -245,7 +245,7 @@ function engine:_create_table(table) if self.params.driver == "PostgreSQL" then sql = sql:gsub("`", "\""); elseif self.params.driver == "MySQL" then - sql = sql:gsub(";$", " CHARACTER SET 'utf8' COLLATE 'utf8_bin';"); + sql = sql:gsub(";$", (" CHARACTER SET '%s' COLLATE '%s_bin';"):format(self.charset, self.charset)); end local success,err = self:execute(sql); if not success then return success,err; end -- cgit v1.2.3 From eeb1c1be540af288105580944eb3664b20abfd21 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 8 Jul 2015 15:06:20 +0100 Subject: util.sql: Use appropriate collation for the chosen character set - fixes MySQL silently ignoring our SET NAMES command when we use utf8mb4 --- util/sql.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/sql.lua b/util/sql.lua index 562ac709..fc1191f9 100644 --- a/util/sql.lua +++ b/util/sql.lua @@ -268,12 +268,12 @@ function engine:set_encoding() -- to UTF-8 local set_names_query = "SET NAMES '%s';" local charset = "utf8"; if driver == "MySQL" then - set_names_query = set_names_query:gsub(";$", " COLLATE 'utf8_bin';"); local ok, charsets = self:transaction(function() return self:select"SELECT `CHARACTER_SET_NAME` FROM `information_schema`.`CHARACTER_SETS` WHERE `CHARACTER_SET_NAME` LIKE 'utf8%' ORDER BY MAXLEN DESC LIMIT 1;"; end); local row = ok and charsets(); charset = row and row[1] or charset; + set_names_query = set_names_query:gsub(";$", (" COLLATE '%s';"):format(charset.."_bin")); end self.charset = charset; return self:transaction(function() return self:execute(set_names_query:format(charset)); end); -- cgit v1.2.3 From 0fc9766334f6fdc84b02e256492931f16a76b80d Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 8 Jul 2015 15:07:23 +0100 Subject: mod_storage_sql2: Upgrade table if its charset does not match our connection's charset (thanks Zash) --- plugins/mod_storage_sql2.lua | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/plugins/mod_storage_sql2.lua b/plugins/mod_storage_sql2.lua index a9d82214..7695e15d 100644 --- a/plugins/mod_storage_sql2.lua +++ b/plugins/mod_storage_sql2.lua @@ -351,7 +351,8 @@ local function upgrade_table(params, apply_changes) end -- COMPAT w/pre-0.10: Upgrade table to UTF-8 if not already - local check_encoding_query = "SELECT `COLUMN_NAME`,`COLUMN_TYPE` FROM `information_schema`.`columns` WHERE `TABLE_NAME`='prosody' AND ( `CHARACTER_SET_NAME`!='utf8' OR `COLLATION_NAME`!='utf8_bin' );"; + local check_encoding_query = "SELECT `COLUMN_NAME`,`COLUMN_TYPE`,`TABLE_NAME` FROM `information_schema`.`columns` WHERE `TABLE_NAME` LIKE 'prosody%%' AND ( `CHARACTER_SET_NAME`!='%s' OR `COLLATION_NAME`!='%s_bin' );"; + check_encoding_query = check_encoding_query:format(engine.charset, engine.charset); success,err = engine:transaction(function() local result = engine:execute(check_encoding_query); local n_bad_columns = result:rowcount(); @@ -359,12 +360,13 @@ local function upgrade_table(params, apply_changes) changes = true; if apply_changes then module:log("warn", "Found %d columns in prosody table requiring encoding change, updating now...", n_bad_columns); - local fix_column_query1 = "ALTER TABLE `prosody` CHANGE `%s` `%s` BLOB;"; - local fix_column_query2 = "ALTER TABLE `prosody` CHANGE `%s` `%s` %s CHARACTER SET 'utf8' COLLATE 'utf8_bin';"; + local fix_column_query1 = "ALTER TABLE `%s` CHANGE `%s` `%s` BLOB;"; + local fix_column_query2 = "ALTER TABLE `%s` CHANGE `%s` `%s` %s CHARACTER SET '%s' COLLATE '%s_bin';"; for row in result:rows() do - local column_name, column_type = unpack(row); - engine:execute(fix_column_query1:format(column_name, column_name)); - engine:execute(fix_column_query2:format(column_name, column_name, column_type)); + local column_name, column_type, table_name = unpack(row); + module:log("debug", "Fixing column %s in table %s", column_name, table_name); + engine:execute(fix_column_query1:format(table_name, column_name, column_name)); + engine:execute(fix_column_query2:format(table_name, column_name, column_name, column_type, engine.charset, engine.charset)); end module:log("info", "Database encoding upgrade complete!"); end -- cgit v1.2.3 From 9f5bd334c59da59813016d3f98cb31a5c144fcda Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 8 Jul 2015 15:10:25 +0100 Subject: util.sql: Return failure if set_encoding() fails --- util/sql.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/util/sql.lua b/util/sql.lua index fc1191f9..d0da9302 100644 --- a/util/sql.lua +++ b/util/sql.lua @@ -110,7 +110,10 @@ function engine:connect() dbh:autocommit(false); -- don't commit automatically self.conn = dbh; self.prepared = {}; - self:set_encoding(); + local ok, err = self:set_encoding(); + if not ok then + return ok, err; + end local ok, err = self:onconnect(); if ok == false then return ok, err; -- cgit v1.2.3 From eed81c2f941254646a0e77e6ca3f9a7b95710772 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 8 Jul 2015 15:14:24 +0100 Subject: util.sql: Make set_encoding() return failure of SET NAMES --- util/sql.lua | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/util/sql.lua b/util/sql.lua index d0da9302..6622ad56 100644 --- a/util/sql.lua +++ b/util/sql.lua @@ -279,7 +279,13 @@ function engine:set_encoding() -- to UTF-8 set_names_query = set_names_query:gsub(";$", (" COLLATE '%s';"):format(charset.."_bin")); end self.charset = charset; - return self:transaction(function() return self:execute(set_names_query:format(charset)); end); + log("debug", "Using encoding '%s' for database connection", charset); + local ok, err = self:transaction(function() return self:execute(set_names_query:format(charset)); end); + if not ok then + return ok, err; + end + + return true; end local engine_mt = { __index = engine }; -- cgit v1.2.3 From 5a90847cd7b5df458a34000b187dcd11522940ea Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 8 Jul 2015 15:15:32 +0100 Subject: util.sql: Log debug message when connecting to database --- util/sql.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/util/sql.lua b/util/sql.lua index 6622ad56..037dbc76 100644 --- a/util/sql.lua +++ b/util/sql.lua @@ -101,6 +101,7 @@ function engine:connect() local params = self.params; assert(params.driver, "no driver") + log("error", "Connecting to [%s] %s...", params.driver, params.database); local dbh, err = DBI.Connect( params.driver, params.database, params.username, params.password, -- cgit v1.2.3 From c23293f38250ac5f85a813452a6d4973ea94b5fe Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 8 Jul 2015 15:16:17 +0100 Subject: util.sql: Fix log level of debug message --- util/sql.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/sql.lua b/util/sql.lua index 037dbc76..693b67d9 100644 --- a/util/sql.lua +++ b/util/sql.lua @@ -101,7 +101,7 @@ function engine:connect() local params = self.params; assert(params.driver, "no driver") - log("error", "Connecting to [%s] %s...", params.driver, params.database); + log("debug", "Connecting to [%s] %s...", params.driver, params.database); local dbh, err = DBI.Connect( params.driver, params.database, params.username, params.password, -- cgit v1.2.3 From 376dd433962a1a897a239d06dbc91d3e03290fe3 Mon Sep 17 00:00:00 2001 From: Matthew Wild Date: Wed, 8 Jul 2015 15:25:42 +0100 Subject: util.sql: Add safety check to ensure our chosen connection charset is actually being used (MySQL) --- util/sql.lua | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/util/sql.lua b/util/sql.lua index 693b67d9..f934bbdf 100644 --- a/util/sql.lua +++ b/util/sql.lua @@ -286,6 +286,18 @@ function engine:set_encoding() -- to UTF-8 return ok, err; end + if driver == "MySQL" then + local ok, actual_charset = self:transaction(function () + return self:select"SHOW SESSION VARIABLES LIKE 'character_set_client'"; + end); + for row in actual_charset do + if row[2] ~= charset then + log("error", "MySQL %s is actually %q (expected %q)", row[1], row[2], charset); + return false, "Failed to set connection encoding"; + end + end + end + return true; end local engine_mt = { __index = engine }; -- cgit v1.2.3