gluster/gluster-block
1
0
Fork 0
mirror of https://github.com/gluster/gluster-block.git synced 2026-02-05 12:45:33 +01:00
Code Issues Releases Activity

gbConf: Use shared memory instead of from the data segment

Browse Source 
Problems:
-------
Since we have make the cli and server routines into 2 different
processes in gluster-blockd daemon, the gbConf data will also
be have different copies. If the gbConf data is changed by the
dyn-config thread in the cli(parent) process, the new changes won't
be visible for the server(child) process.

Resolution:
----------
For the gbConf data, use the shared memory instead of the in the
data segment.

Signed-off-by: Xiubo Li <xiubli@redhat.com>
Reviewed-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
This commit is contained in:
Xiubo Li
2019-04-26 16:32:25 +08:00
committed by Prasanna Kumar Kalever
parent f1b066aca4
commit d3b1bb7ed5
7 changed files with 135 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
cli/gluster-block.c
View File

@@ -1029,6 +1029,7 @@ main(int argc, char *argv[])
int count = 1;
int args = argc;
int json = GB_JSON_NONE;
int ret;
if (argc <= 1) {
@@ -1036,6 +1037,10 @@ main(int argc, char *argv[])
goto fail;
}
if (initGbConfig()) {
goto fail;
}
if(initLogging()) {
goto fail;
}
@@ -1075,8 +1080,13 @@ main(int argc, char *argv[])
break;
}
return glusterBlockParseArgs(args - 1, &argv[count], opt, json);
ret = glusterBlockParseArgs(args - 1, &argv[count], opt, json);
finiGbConfig();
return ret;
fail:
finiGbConfig();
exit(EXIT_FAILURE);
}

18
daemon/gluster-blockd.c
View File

@@ -337,7 +337,7 @@ glusterBlockDParseArgs(int count, char **options)
break;
case GB_DAEMON_NO_REMOTE_RPC:
gbConf.noRemoteRpc = true;
gbConf->noRemoteRpc = true;
break;
}
@@ -552,7 +552,7 @@ blockNodeSanityCheck(void)
/* Check if dependencies meet minimum recommended versions */
gbDependenciesVersionCheck();
if (GB_ASPRINTF(&global_opts, GB_TGCLI_GLOBALS, gbConf.configShellLogFile) == -1) {
if (GB_ASPRINTF(&global_opts, GB_TGCLI_GLOBALS, gbConf->configShellLogFile) == -1) {
return ENOMEM;
}
/* Set targetcli globals */
@@ -588,16 +588,20 @@ main (int argc, char **argv)
int wstatus;
if(initLogging()) {
if (initGbConfig()) {
exit(EXIT_FAILURE);
}
if(initLogging()) {
goto out;
}
fetchGlfsVolServerFromEnv();
gbCfg = glusterBlockSetupConfig();
if (!gbCfg) {
LOG("mgmt", GB_LOG_ERROR, "glusterBlockSetupConfig() failed");
exit(EXIT_FAILURE);
goto out;
}
if (glusterBlockDParseArgs(argc, argv)) {
@@ -619,7 +623,7 @@ main (int argc, char **argv)
goto out;
}
if (!gbConf.noRemoteRpc) {
if (!gbConf->noRemoteRpc) {
if (blockNodeSanityCheck()) {
goto out;
}
@@ -638,7 +642,7 @@ main (int argc, char **argv)
signal(SIGPIPE, SIG_IGN);
pmap_unset(GLUSTER_BLOCK_CLI, GLUSTER_BLOCK_CLI_VERS);
if (!gbConf.noRemoteRpc) {
if (!gbConf->noRemoteRpc) {
pmap_unset(GLUSTER_BLOCK, GLUSTER_BLOCK_VERS);
}
@@ -686,6 +690,8 @@ main (int argc, char **argv)
}
out:
finiGbConfig();
glusterBlockCleanGlobals();
exit (EXIT_FAILURE);
/* NOTREACHED */

12
rpc/block_svc_routines.c
View File

@@ -1937,8 +1937,8 @@ glusterBlockReplaceNodeRemoteAsync(struct glfs *glfs, blockReplaceCli *blk,
goto out;
}
cobj->xdata.xdata_len = strlen(gbConf.volServer);
cobj->xdata.xdata_val = (char *) gbConf.volServer;
cobj->xdata.xdata_len = strlen(gbConf->volServer);
cobj->xdata.xdata_val = (char *) gbConf->volServer;
GB_STRCPYSTATIC(cobj->ipaddr, blk->new_node);
GB_STRCPYSTATIC(cobj->volume, info->volume);
GB_STRCPYSTATIC(cobj->gbid, info->gbid);
@@ -2709,10 +2709,10 @@ getSoObj(char *block, MetaInfo *info, blockGenConfigCli *blk)
json_object_object_add(so_obj, "attributes", so_obj_attr);
// }
if (!strcmp(gbConf.volServer, "localhost")) {
if (!strcmp(gbConf->volServer, "localhost")) {
snprintf(cfgstr, 1024, "glfs/%s@%s/block-store/%s", info->volume, blk->addr, info->gbid);
} else {
snprintf(cfgstr, 1024, "glfs/%s@%s/block-store/%s", info->volume, gbConf.volServer, info->gbid);
snprintf(cfgstr, 1024, "glfs/%s@%s/block-store/%s", info->volume, gbConf->volServer, info->gbid);
}
json_object_object_add(so_obj, "config", GB_JSON_OBJ_TO_STR(cfgstr[0]?cfgstr:NULL));
if (info->rb_size) {
@@ -3981,8 +3981,8 @@ block_create_cli_1_svc_st(blockCreateCli *blk, struct svc_req *rqstp)
cobj.rb_size = blk->rb_size;
GB_STRCPYSTATIC(cobj.gbid, gbid);
GB_STRDUP(cobj.block_hosts, blk->block_hosts);
cobj.xdata.xdata_len = strlen(gbConf.volServer);
cobj.xdata.xdata_val = (char *) gbConf.volServer;
cobj.xdata.xdata_len = strlen(gbConf->volServer);
cobj.xdata.xdata_val = (char *) gbConf->volServer;
if (blk->auth_mode) {
uuid_generate(uuid);

10
rpc/glfs-operations.c
View File

@@ -34,27 +34,27 @@ glusterBlockVolumeInit(char *volume, int *errCode, char **errMsg)
GB_ASPRINTF (errMsg, "Not able to Initialize volume %s [%s]", volume,
strerror(*errCode));
LOG("gfapi", GB_LOG_ERROR, "glfs_new(%s) from %s failed[%s]", volume,
gbConf.volServer, strerror(*errCode));
gbConf->volServer, strerror(*errCode));
return NULL;
}
ret = glfs_set_volfile_server(glfs, "tcp", gbConf.volServer, 24007);
ret = glfs_set_volfile_server(glfs, "tcp", gbConf->volServer, 24007);
if (ret) {
*errCode = errno;
GB_ASPRINTF (errMsg, "Not able to add Volfile server for volume %s[%s]",
volume, strerror(*errCode));
LOG("gfapi", GB_LOG_ERROR, "glfs_set_volfile_server(%s) of %s "
"failed[%s]", gbConf.volServer, volume, strerror(*errCode));
"failed[%s]", gbConf->volServer, volume, strerror(*errCode));
goto out;
}
ret = glfs_set_logging(glfs, gbConf.gfapiLogFile, GFAPI_LOG_LEVEL);
ret = glfs_set_logging(glfs, gbConf->gfapiLogFile, GFAPI_LOG_LEVEL);
if (ret) {
*errCode = errno;
GB_ASPRINTF (errMsg, "Not able to add logging for volume %s[%s]", volume,
strerror(*errCode));
LOG("gfapi", GB_LOG_ERROR, "glfs_set_logging(%s, %d) on %s failed[%s]",
gbConf.gfapiLogFile, GFAPI_LOG_LEVEL, volume, strerror(*errCode));
gbConf->gfapiLogFile, GFAPI_LOG_LEVEL, volume, strerror(*errCode));
goto out;
}

14
utils/lru.c
View File

@@ -64,9 +64,9 @@ glusterBlockSetLruCount(const size_t lruCount)
return -1;
}
LOCK(gbConf.lock);
gbConf.glfsLruCount = lruCount;
UNLOCK(gbConf.lock);
LOCK(gbConf->lock);
gbConf->glfsLruCount = lruCount;
UNLOCK(gbConf->lock);
LOG("mgmt", GB_LOG_CRIT,
"glfsLruCount now is %lu", lruCount);
@@ -103,13 +103,13 @@ appendNewEntry(const char *volname, glfs_t *fs)
Entry *tmp;
LOCK(gbConf.lock);
if (lruCount == gbConf.glfsLruCount) {
LOCK(gbConf->lock);
if (lruCount == gbConf->glfsLruCount) {
releaseColdEntry();
}
if (GB_ALLOC(tmp) < 0) {
UNLOCK(gbConf.lock);
UNLOCK(gbConf->lock);
return -1;
}
GB_STRCPYSTATIC(tmp->volume, volname);
@@ -120,7 +120,7 @@ appendNewEntry(const char *volname, glfs_t *fs)
UNLOCK(lru_lock);
lruCount++;
UNLOCK(gbConf.lock);
UNLOCK(gbConf->lock);
return 0;
}

115
utils/utils.c
View File

@@ -14,19 +14,12 @@
# include <dirent.h>
# include <sys/stat.h>
# include <pthread.h>
# include <sys/mman.h>
# include "utils.h"
# include "lru.h"
# include "config.h"
struct gbConf gbConf = {
.lock = PTHREAD_MUTEX_INITIALIZER,
.glfsLruCount = LRU_COUNT_DEF,
.logLevel = GB_LOG_INFO,
.logDir = GB_LOGDIR_DEF,
.cliTimeout = CLI_TIMEOUT_DEF
};
const char *argp_program_version = "" \
PACKAGE_NAME" ("PACKAGE_VERSION")" \
"\nRepository rev: https://github.com/gluster/gluster-block.git\n" \
@@ -37,6 +30,64 @@ const char *argp_program_version = "" \
"or later), or the GNU General Public License, version 2 (GPLv2),\n"\
"in all cases as published by the Free Software Foundation.";
struct gbConf *gbConf = NULL;
int
initGbConfig(void)
{
pthread_mutexattr_t attr;
int ret;
if (gbConf)
return 0;
if (gbCtx == GB_CLI_MODE) {
if (GB_ALLOC_N(gbConf, sizeof(gbConf)) < 0) {
ret = errno;
MSG(stderr, "Failed to allocate memory for gbConf %s", strerror (errno));
return ret;
}
pthread_mutex_init(&gbConf->lock, NULL);
} else {
gbConf = mmap(NULL, sizeof(*gbConf), PROT_READ|PROT_WRITE,
MAP_SHARED|MAP_ANON, -1, 0);
if (gbConf == MAP_FAILED) {
ret = errno;
MSG(stderr, "mmap the gbConf failed %s", strerror (errno));
return ret;
}
pthread_mutexattr_init(&attr);
pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED);
pthread_mutex_init(&gbConf->lock, &attr);
}
gbConf->glfsLruCount = LRU_COUNT_DEF;
gbConf->logLevel = GB_LOG_INFO;
snprintf(gbConf->logDir, PATH_MAX, "%s", GB_LOGDIR_DEF);
gbConf->cliTimeout = CLI_TIMEOUT_DEF;
return 0;
}
void
finiGbConfig(void)
{
if (!gbConf)
return;
pthread_mutex_destroy(&gbConf->lock);
if (gbCtx == GB_CLI_MODE) {
GB_FREE(gbConf);
} else {
munmap(gbConf, sizeof(*gbConf));
}
}
int
glusterBlockSetLogLevel(unsigned int logLevel)
@@ -45,9 +96,9 @@ glusterBlockSetLogLevel(unsigned int logLevel)
MSG(stderr, "unknown LOG-LEVEL: '%d'", logLevel);
return -1;
}
LOCK(gbConf.lock);
gbConf.logLevel = logLevel;
UNLOCK(gbConf.lock);
LOCK(gbConf->lock);
gbConf->logLevel = logLevel;
UNLOCK(gbConf->lock);
if (gbCtx == GB_CLI_MODE) {
LOG("cli", GB_LOG_DEBUG,
@@ -69,9 +120,9 @@ glusterBlockSetCliTimeout(size_t timeout)
MSG(stderr, "unknown GB_CLI_TIMEOUT: '%zu'", timeout);
return -1;
}
LOCK(gbConf.lock);
gbConf.cliTimeout = timeout;
UNLOCK(gbConf.lock);
LOCK(gbConf->lock);
gbConf->cliTimeout = timeout;
UNLOCK(gbConf->lock);
return 0;
}
@@ -247,20 +298,20 @@ out:
static bool
glusterBlockLogdirCreate(void)
{
DIR* dir = opendir(gbConf.logDir);
DIR* dir = opendir(gbConf->logDir);
char *buf = NULL;
if (dir) {
closedir(dir);
} else if (errno == ENOENT) {
GB_ASPRINTF(&buf, "mkdir -p %s -m 0755 > /dev/null", gbConf.logDir);
GB_ASPRINTF(&buf, "mkdir -p %s -m 0755 > /dev/null", gbConf->logDir);
if (gbRunner(buf) == -1) {
MSG(stderr, "mkdir(%s) failed (%s)", gbConf.logDir, strerror (errno));
MSG(stderr, "mkdir(%s) failed (%s)", gbConf->logDir, strerror (errno));
return 0; /* False */
}
} else {
MSG(stderr, "opendir(%s) failed (%s)", gbConf.logDir, strerror (errno));
MSG(stderr, "opendir(%s) failed (%s)", gbConf->logDir, strerror (errno));
return 0; /* False */
}
@@ -279,9 +330,9 @@ fetchGlfsVolServerFromEnv()
if (!volServer) {
volServer = "localhost";
}
snprintf(gbConf.volServer, HOST_NAME_MAX, "%s", volServer);
snprintf(gbConf->volServer, HOST_NAME_MAX, "%s", volServer);
LOG("mgmt", GB_LOG_INFO, "Block Hosting Volfile Server Set to: %s", gbConf.volServer);
LOG("mgmt", GB_LOG_INFO, "Block Hosting Volfile Server Set to: %s", gbConf->volServer);
}
@@ -423,7 +474,7 @@ initLogDirAndFiles(char *newLogDir)
}
LOG(dom, logLevel,
"trying to change logDir from %s to %s", gbConf.logDir, logDir);
"trying to change logDir from %s to %s", gbConf->logDir, logDir);
if (strlen(logDir) > PATH_MAX - GB_MAX_LOGFILENAME) {
LOG(dom, GB_LOG_ERROR, "strlen of logDir Path > PATH_MAX: %s", logDir);
@@ -432,35 +483,35 @@ initLogDirAndFiles(char *newLogDir)
}
/* set logfile paths */
LOCK(gbConf.lock);
snprintf(gbConf.logDir, PATH_MAX,
LOCK(gbConf->lock);
snprintf(gbConf->logDir, PATH_MAX,
"%s", logDir);
snprintf(gbConf.daemonLogFile, PATH_MAX,
snprintf(gbConf->daemonLogFile, PATH_MAX,
"%s/gluster-blockd.log", logDir);
snprintf(gbConf.cliLogFile, PATH_MAX,
snprintf(gbConf->cliLogFile, PATH_MAX,
"%s/gluster-block-cli.log", logDir);
snprintf(gbConf.gfapiLogFile, PATH_MAX,
snprintf(gbConf->gfapiLogFile, PATH_MAX,
"%s/gluster-block-gfapi.log", logDir);
snprintf(gbConf.configShellLogFile, PATH_MAX,
snprintf(gbConf->configShellLogFile, PATH_MAX,
"%s/gluster-block-configshell.log", logDir);
snprintf(gbConf.cmdhistoryLogFile, PATH_MAX,
snprintf(gbConf->cmdhistoryLogFile, PATH_MAX,
"%s/cmd_history.log", logDir);
if(!glusterBlockLogdirCreate()) {
ret = -1;
}
UNLOCK(gbConf.lock);
UNLOCK(gbConf->lock);
if (ret == -1) {
goto out;
}
LOG(dom, logLevel, "logDir now is %s", gbConf.logDir);
LOG(dom, logLevel, "logDir now is %s", gbConf->logDir);
glusterBlockUpdateLruLogdir(gbConf.gfapiLogFile);
glusterBlockUpdateLruLogdir(gbConf->gfapiLogFile);
if (!def) {
glusterLogrotateConfigSet(gbConf.logDir);
glusterLogrotateConfigSet(gbConf->logDir);
}
out:

19
utils/utils.h
View File

@@ -156,7 +156,7 @@ struct gbConf {
char volServer[HOST_NAME_MAX];
};
extern struct gbConf gbConf;
extern struct gbConf *gbConf;
# define LOG(str, level, fmt, ...) \
do { \
@@ -165,16 +165,16 @@ extern struct gbConf gbConf;
char *tmp; \
if (GB_STRDUP(tmp, str) < 0) \
fprintf(stderr, "No memory: %s\n", strerror(errno)); \
LOCK(gbConf.lock); \
if (level <= gbConf.logLevel) { \
LOCK(gbConf->lock); \
if (level <= gbConf->logLevel) { \
if (!strcmp(tmp, "mgmt")) \
fd = fopen (gbConf.daemonLogFile, "a"); \
fd = fopen (gbConf->daemonLogFile, "a"); \
else if (!strcmp(tmp, "cli")) \
fd = fopen (gbConf.cliLogFile, "a"); \
fd = fopen (gbConf->cliLogFile, "a"); \
else if (!strcmp(tmp, "gfapi")) \
fd = fopen (gbConf.gfapiLogFile, "a"); \
fd = fopen (gbConf->gfapiLogFile, "a"); \
else if (!strcmp(tmp, "cmdlog")) \
fd = fopen (gbConf.cmdhistoryLogFile, "a"); \
fd = fopen (gbConf->cmdhistoryLogFile, "a"); \
else \
fd = stderr; \
if (fd == NULL) { \
@@ -190,7 +190,7 @@ extern struct gbConf gbConf;
if (fd != stderr) \
fclose(fd); \
} \
UNLOCK(gbConf.lock); \
UNLOCK(gbConf->lock); \
GB_FREE(tmp); \
} while (0)
@@ -618,6 +618,9 @@ typedef enum gbDependencies {
TARGETCLI = 2,
} gbDependencies;
int initGbConfig(void);
void finiGbConfig(void);
int glusterBlockSetLogLevel(unsigned int logLevel);
//int glusterBlockSetCliTimeout(size_t timeout);