From 6e638c213f36e30bb165ab232856bede8f8f0a19 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Wed, 1 Mar 2017 01:33:00 +0100 Subject: util.pposix: Add function for atomically appending data to a file --- plugins/mod_posix.lua | 2 +- prosodyctl | 2 +- util-src/pposix.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/plugins/mod_posix.lua b/plugins/mod_posix.lua index 2103071a..412a7779 100644 --- a/plugins/mod_posix.lua +++ b/plugins/mod_posix.lua @@ -7,7 +7,7 @@ -- -local want_pposix_version = "0.3.6"; +local want_pposix_version = "0.3.7"; local pposix = assert(require "util.pposix"); if pposix._VERSION ~= want_pposix_version then diff --git a/prosodyctl b/prosodyctl index cc440c59..4fb67a95 100755 --- a/prosodyctl +++ b/prosodyctl @@ -135,7 +135,7 @@ dependencies.log_warnings(); -- Switch away from root and into the prosody user -- local switched_user, current_uid; -local want_pposix_version = "0.3.6"; +local want_pposix_version = "0.3.7"; local ok, pposix = pcall(require, "util.pposix"); if ok and pposix then diff --git a/util-src/pposix.c b/util-src/pposix.c index 4b4552f5..a68aa89d 100644 --- a/util-src/pposix.c +++ b/util-src/pposix.c @@ -13,7 +13,7 @@ * POSIX support functions for Lua */ -#define MODULE_VERSION "0.3.6" +#define MODULE_VERSION "0.3.7" #if defined(__linux__) @@ -788,6 +788,68 @@ int lc_fallocate(lua_State *L) { } } +/* + * Append some data to a file handle + * Attempt to allocate space first + * Truncate to original size on failure + */ +int lc_atomic_append(lua_State *L) { + int err; + size_t len; + + FILE *f = *(FILE **) luaL_checkudata(L, 1, LUA_FILEHANDLE); + const char *data = luaL_checklstring(L, 2, &len); + + off_t offset = ftell(f); + +#if defined(__linux__) + /* Try to allocate space without changing the file size. */ + if((err = fallocate(fileno(f), FALLOC_FL_KEEP_SIZE, offset, len))) { + if(errno != 0) { + /* Some old versions of Linux apparently use the return value instead of errno */ + err = errno; + } + switch(err) { + case ENOSYS: /* Kernel doesn't implement fallocate */ + case EOPNOTSUPP: /* Filesystem doesn't support it */ + /* Ignore and proceed to try to write */ + break; + + case ENOSPC: /* No space left */ + default: /* Other issues */ + lua_pushnil(L); + lua_pushstring(L, strerror(err)); + lua_pushinteger(L, err); + return 3; + } + } +#endif + + if(fwrite(data, sizeof(char), len, f) == len) { + if(fflush(f) == 0) { + lua_pushboolean(L, 1); /* Great success! */ + return 1; + } else { + err = errno; + } + } else { + err = ferror(f); + } + + fseek(f, offset, SEEK_SET); + + /* Cut partially written data */ + if(ftruncate(fileno(f), offset)) { + /* The file is now most likely corrupted, throw hard error */ + return luaL_error(L, "atomic_append() failed in ftruncate(): %s", strerror(errno)); + } + + lua_pushnil(L); + lua_pushstring(L, strerror(err)); + lua_pushinteger(L, err); + return 3; +} + /* Register functions */ int luaopen_util_pposix(lua_State *L) { @@ -828,6 +890,7 @@ int luaopen_util_pposix(lua_State *L) { #endif { "fallocate", lc_fallocate }, + { "atomic_append", lc_atomic_append }, { NULL, NULL } }; -- cgit v1.2.3 From 088818ab0f09530e125bfe0c7904ff9050fc75aa Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 28 Feb 2017 11:33:43 +0100 Subject: util.datamanager: Use pposix.atomic_append --- util/datamanager.lua | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/util/datamanager.lua b/util/datamanager.lua index b825cb33..b2a5fa0e 100644 --- a/util/datamanager.lua +++ b/util/datamanager.lua @@ -17,7 +17,6 @@ local io_open = io.open; local os_remove = os.remove; local os_rename = os.rename; local tonumber = tonumber; -local tostring = tostring; local next = next; local type = type; local t_insert = table.insert; @@ -31,21 +30,12 @@ local path_separator = assert ( package.config:match ( "^([^\n]+)" ) , "package. local prosody = prosody; local raw_mkdir = lfs.mkdir; -local function fallocate(f, offset, len) - -- This assumes that current position == offset - local fake_data = (" "):rep(len); - local ok, msg = f:write(fake_data); - if not ok then - return ok, msg; - end - f:seek("set", offset); - return true; -end; +local atomic_append; local ENOENT = 2; pcall(function() local pposix = require "util.pposix"; raw_mkdir = pposix.mkdir or raw_mkdir; -- Doesn't trample on umask - fallocate = pposix.fallocate or fallocate; + atomic_append = pposix.atomic_append; ENOENT = pposix.ENOENT or ENOENT; end); @@ -65,6 +55,19 @@ do end end +if not atomic_append then + function atomic_append(f, data) + local pos = f:seek(); + if not f:write(data) or not f:flush() then + f:seek("set", pos); + f:write((" "):rep(#data)); + f:flush(); + return nil, "write-failed"; + end + return true; + end +end + local _mkdir = {}; local function mkdir(path) path = path:gsub("/", path_separator); -- TODO as an optimization, do this during path creation rather than here @@ -221,25 +224,16 @@ local function append(username, host, datastore, ext, data) local filename = getpath(username, host, datastore, ext, true); local ok; - local f, msg = io_open(filename, "r+"); + local f, msg, errno = io_open(filename, "r+"); if not f then return atomic_store(filename, data); -- File did probably not exist, let's create it end local pos = f:seek("end"); - ok, msg = fallocate(f, pos, #data); - if not ok then - log("warn", "fallocate() failed: %s", tostring(msg)); - -- This doesn't work on every file system - end - if f:seek() ~= pos then - log("debug", "fallocate() changed file position"); - f:seek("set", pos); - end + ok, msg = atomic_append(f, data); - ok, msg = f:write(data); if not ok then f:close(); return ok, msg, "write"; -- cgit v1.2.3 From e7a2ecb01e6c11b3348fc92d2c30d23b4e6b1053 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 28 Feb 2017 13:26:05 +0100 Subject: util.pposix: Remove fallocate --- plugins/mod_posix.lua | 2 +- prosodyctl | 2 +- util-src/pposix.c | 61 +-------------------------------------------------- 3 files changed, 3 insertions(+), 62 deletions(-) diff --git a/plugins/mod_posix.lua b/plugins/mod_posix.lua index 412a7779..5a6cd0c2 100644 --- a/plugins/mod_posix.lua +++ b/plugins/mod_posix.lua @@ -7,7 +7,7 @@ -- -local want_pposix_version = "0.3.7"; +local want_pposix_version = "0.4.0"; local pposix = assert(require "util.pposix"); if pposix._VERSION ~= want_pposix_version then diff --git a/prosodyctl b/prosodyctl index 4fb67a95..be68f0f4 100755 --- a/prosodyctl +++ b/prosodyctl @@ -135,7 +135,7 @@ dependencies.log_warnings(); -- Switch away from root and into the prosody user -- local switched_user, current_uid; -local want_pposix_version = "0.3.7"; +local want_pposix_version = "0.4.0"; local ok, pposix = pcall(require, "util.pposix"); if ok and pposix then diff --git a/util-src/pposix.c b/util-src/pposix.c index a68aa89d..e70a9d7f 100644 --- a/util-src/pposix.c +++ b/util-src/pposix.c @@ -13,7 +13,7 @@ * POSIX support functions for Lua */ -#define MODULE_VERSION "0.3.7" +#define MODULE_VERSION "0.4.0" #if defined(__linux__) @@ -730,64 +730,6 @@ int lc_meminfo(lua_State *L) { } #endif -/* File handle extraction blatantly stolen from - * https://github.com/rrthomas/luaposix/blob/master/lposix.c#L631 - * */ - -int lc_fallocate(lua_State *L) { - int ret; - off_t offset, len; - FILE *f = *(FILE **) luaL_checkudata(L, 1, LUA_FILEHANDLE); - - if(f == NULL) { - return luaL_error(L, "attempt to use a closed file"); - } - - offset = luaL_checkinteger(L, 2); - len = luaL_checkinteger(L, 3); - -#if defined(__linux__) - errno = 0; - ret = fallocate(fileno(f), FALLOC_FL_KEEP_SIZE, offset, len); - - if(ret == 0) { - lua_pushboolean(L, 1); - return 1; - } - - /* Some old versions of Linux apparently use the return value instead of errno */ - if(errno == 0) { - errno = ret; - } - - if(errno != ENOSYS && errno != EOPNOTSUPP) { - lua_pushnil(L); - lua_pushstring(L, strerror(errno)); - return 2; - } - -#endif - - ret = posix_fallocate(fileno(f), offset, len); - - if(ret == 0) { - lua_pushboolean(L, 1); - return 1; - } else { - lua_pushnil(L); - lua_pushstring(L, strerror(ret)); - - /* posix_fallocate() can leave a bunch of NULs at the end, so we cut that - * this assumes that offset == length of the file */ - if(ftruncate(fileno(f), offset) != 0) { - lua_pushstring(L, strerror(errno)); - return 3; - } - - return 2; - } -} - /* * Append some data to a file handle * Attempt to allocate space first @@ -889,7 +831,6 @@ int luaopen_util_pposix(lua_State *L) { { "meminfo", lc_meminfo }, #endif - { "fallocate", lc_fallocate }, { "atomic_append", lc_atomic_append }, { NULL, NULL } -- cgit v1.2.3 From ba5133a97723367212b65a9c27cc91c74928f394 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 28 Mar 2017 17:31:24 +0200 Subject: util.datamanager: Rearrange locals --- util/datamanager.lua | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/util/datamanager.lua b/util/datamanager.lua index b2a5fa0e..2ed672e5 100644 --- a/util/datamanager.lua +++ b/util/datamanager.lua @@ -223,8 +223,7 @@ local function append(username, host, datastore, ext, data) if type(data) ~= "string" then return; end local filename = getpath(username, host, datastore, ext, true); - local ok; - local f, msg, errno = io_open(filename, "r+"); + local f = io_open(filename, "r+"); if not f then return atomic_store(filename, data); -- File did probably not exist, let's create it @@ -232,7 +231,7 @@ local function append(username, host, datastore, ext, data) local pos = f:seek("end"); - ok, msg = atomic_append(f, data); + local ok, msg = atomic_append(f, data); if not ok then f:close(); -- cgit v1.2.3 From bcc0a4f11d07b3ff01011790c33abf1b4298c244 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 28 Mar 2017 17:31:55 +0200 Subject: util.datamanager: Log where the error came from --- util/datamanager.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/util/datamanager.lua b/util/datamanager.lua index 2ed672e5..6edf7ac0 100644 --- a/util/datamanager.lua +++ b/util/datamanager.lua @@ -240,7 +240,7 @@ local function append(username, host, datastore, ext, data) ok, msg = f:close(); if not ok then - return ok, msg; + return ok, msg, "close"; end return true, pos; @@ -252,9 +252,10 @@ local function list_append(username, host, datastore, data) -- save the datastore data = "item(" .. serialize(data) .. ");\n"; - local ok, msg = append(username, host, datastore, "list", data); + local ok, msg, where = append(username, host, datastore, "list", data); if not ok then - log("error", "Unable to write to %s storage ('%s') for user: %s@%s", datastore, msg, username or "nil", host or "nil"); + log("error", "Unable to write to %s storage ('%s' in %s) for user: %s@%s", + datastore, msg, where, username or "nil", host or "nil"); return ok, msg; end return true; -- cgit v1.2.3