Skip to content

Commit

Permalink
[CBRD-24721] Improve the sql logging in HA (#4243)
Browse files Browse the repository at this point in the history 
http://jira.cubrid.org/browse/CBRD-24721
  • Loading branch information
@hornetmj
hornetmj authored Apr 7, 2023
1 parent 3917aab commit 538fa05
Show file tree
Hide file tree
Showing 3 changed files with 144 additions and 22 deletions.
37 changes: 37 additions & 0 deletions src/base/system_parameter.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,10 @@ static const char sysprm_ha_conf_file_name[] = "cubrid_ha.conf";

#define PRM_NAME_HA_SQL_LOG_MAX_SIZE_IN_MB "ha_sql_log_max_size_in_mbytes"

#define PRM_NAME_HA_SQL_LOG_PATH "ha_sql_log_path"

#define PRM_NAME_HA_SQL_LOG_MAX_COUNT "ha_sql_log_max_count"

#define PRM_NAME_HA_COPY_LOG_MAX_ARCHIVES "ha_copy_log_max_archives"

#define PRM_NAME_HA_COPY_LOG_TIMEOUT "ha_copy_log_timeout"
Expand Down Expand Up @@ -1483,6 +1487,16 @@ bool PRM_HA_SQL_LOGGING = false;
static bool prm_ha_sql_logging_default = false;
static unsigned int prm_ha_sql_logging_flag = 0;

const char *PRM_HA_SQL_LOG_PATH = "";
static char *prm_ha_sql_log_path_default = NULL;
static unsigned int prm_ha_sql_log_path_flag = 0;

int PRM_HA_SQL_LOG_MAX_COUNT = 2;
static int prm_ha_sql_log_max_count_default = 2;
static int prm_ha_sql_log_max_count_upper = 5;
static int prm_ha_sql_log_max_count_lower = 2;
static unsigned int prm_ha_sql_log_max_count_flag = 0;

int PRM_HA_SQL_LOG_MAX_SIZE_IN_MB = INT_MIN;
static int prm_ha_sql_log_max_size_in_mb_default = 50;
static int prm_ha_sql_log_max_size_in_mb_upper = 2048;
Expand Down Expand Up @@ -6228,6 +6242,29 @@ SYSPRM_PARAM prm_Def[] = {
(void *) NULL, (void *) NULL,
(char *) NULL,
(DUP_PRM_FUNC) NULL,
(DUP_PRM_FUNC) NULL},
{PRM_ID_HA_SQL_LOG_PATH,
PRM_NAME_HA_SQL_LOG_PATH,
(PRM_FOR_CLIENT | PRM_FOR_HA),
PRM_STRING,
&prm_ha_sql_log_path_flag,
(void *) &prm_ha_sql_log_path_default,
(void *) &PRM_HA_SQL_LOG_PATH,
(void *) NULL, (void *) NULL,
(char *) NULL,
(DUP_PRM_FUNC) NULL,
(DUP_PRM_FUNC) NULL},
{PRM_ID_HA_SQL_LOG_MAX_COUNT,
PRM_NAME_HA_SQL_LOG_MAX_COUNT,
(PRM_FOR_CLIENT | PRM_FOR_HA),
PRM_INTEGER,
&prm_ha_sql_log_max_count_flag,
(void *) &prm_ha_sql_log_max_count_default,
(void *) &PRM_HA_SQL_LOG_MAX_COUNT,
(void *) &prm_ha_sql_log_max_count_upper,
(void *) &prm_ha_sql_log_max_count_lower,
(char *) NULL,
(DUP_PRM_FUNC) NULL,
(DUP_PRM_FUNC) NULL}
};

Expand Down
4 changes: 3 additions & 1 deletion src/base/system_parameter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -469,8 +469,10 @@ enum param_id
PRM_ID_HA_TCP_PING_HOSTS,
PRM_ID_HA_PING_TIMEOUT,
PRM_ID_STATDUMP_FORCE_ADD_INT_MAX, /* Hidden parameter for QA only */
PRM_ID_HA_SQL_LOG_PATH,
PRM_ID_HA_SQL_LOG_MAX_COUNT,
/* change PRM_LAST_ID when adding new system parameters */
PRM_LAST_ID = PRM_ID_STATDUMP_FORCE_ADD_INT_MAX
PRM_LAST_ID = PRM_ID_HA_SQL_LOG_MAX_COUNT
};
typedef enum param_id PARAM_ID;

Expand Down
125 changes: 104 additions & 21 deletions src/transaction/log_applier_sql_log.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
#include "set_object.h"
#include "schema_manager.h"
#include "dbtype.h"
#include "file_io.h"

#include "db_value_printer.hpp"
#include "mem_block.hpp"
Expand All @@ -63,6 +64,7 @@ SL_INFO sl_Info;

static FILE *log_fp;
static FILE *catalog_fp;
static int sql_log_max_cnt = 0;
static char sql_log_base_path[PATH_MAX];
static char sql_catalog_path[PATH_MAX];

Expand All @@ -78,9 +80,10 @@ static DB_VALUE *sl_find_att_value (const char *att_name, OBJ_TEMPASSIGN ** assi

static FILE *sl_open_next_file (FILE * old_fp);
static FILE *sl_log_open (void);
static int sl_remove_oldest_file (void);
static int sl_read_catalog (void);
static int sl_write_catalog (void);
static int create_dir (const char *new_dir);
static int sl_create_sql_log_dir (const char *repl_log_path, char *path_buf, int path_buf_size);

static char *
trim_single_quote (char *str, size_t len)
Expand Down Expand Up @@ -169,17 +172,17 @@ sl_read_catalog (void)
int
sl_init (const char *db_name, const char *repl_log_path)
{
char tmp_log_path[PATH_MAX];
char basename_buf[PATH_MAX];
char sql_log_path[PATH_MAX];

memset (&sl_Info, 0, sizeof (sl_Info));
if (sl_create_sql_log_dir (repl_log_path, sql_log_path, sizeof (sql_log_path)) != NO_ERROR)
{
return ER_FAILED;
}

snprintf (tmp_log_path, PATH_MAX, "%s/sql_log/", repl_log_path);
create_dir (tmp_log_path);
snprintf (sql_log_base_path, PATH_MAX, "%s/%s.sql.log", sql_log_path, basename ((char *) repl_log_path));
snprintf (sql_catalog_path, PATH_MAX, "%s/%s_applylogdb.sql.info", dirname (sql_log_path), db_name);

strcpy (basename_buf, repl_log_path);
snprintf (sql_log_base_path, PATH_MAX, "%s/sql_log/%s.sql.log", repl_log_path, basename (basename_buf));
snprintf (sql_catalog_path, PATH_MAX, "%s/%s_applylogdb.sql.info", repl_log_path, db_name);
memset (&sl_Info, 0, sizeof (sl_Info));

sl_Info.curr_file_id = 0;
sl_Info.last_inserted_sql_id = 0;
Expand All @@ -201,6 +204,8 @@ sl_init (const char *db_name, const char *repl_log_path)
return ER_FAILED;
}

sql_log_max_cnt = prm_get_integer_value (PRM_ID_HA_SQL_LOG_MAX_COUNT);

return NO_ERROR;
}

Expand Down Expand Up @@ -546,6 +551,14 @@ sl_write_sql (string_buffer & query, string_buffer * select)
if (ftell (log_fp) >= SL_LOG_FILE_MAX_SIZE)
{
log_fp = sl_open_next_file (log_fp);

if (sl_Info.curr_file_id >= sql_log_max_cnt)
{
if (sl_remove_oldest_file () != NO_ERROR)
{
return ER_FAILED;
}
}
}

return NO_ERROR;
Expand Down Expand Up @@ -612,44 +625,114 @@ sl_open_next_file (FILE * old_fp)
return new_fp;
}

/*
* sl_remove_oldest_file() - remove oldest sql log file
* return: error code
*
* Note:
* This function is related to the ha_sql_log_max_count system parameter.
* This system parameter can be set from 2 to 5 and only that number of sql log files are kept.
*/
static int
sl_remove_oldest_file (void)
{
int oldest_file_id;
char oldest_file_path[PATH_MAX];

oldest_file_id = sl_Info.curr_file_id - sql_log_max_cnt;

snprintf (oldest_file_path, PATH_MAX - 1, "%s.%d", sql_log_base_path, oldest_file_id);

unlink (oldest_file_path);

return NO_ERROR;
}

/*
* sl_create_sql_log_dir() - verify and create the SQL log path
* return: NO_ERROR or ER_FAILED
* repl_log_path(in): log volume path for apply (default path)
* path_buf(out): SQL log path buffer
* path_buf_size(in): buffer size
*
* Note:
* This function is related to the ha_sql_log_path system parameter. The SQL log path can be changed by setting this.
*/
static int
create_dir (const char *new_dir)
sl_create_sql_log_dir (const char *repl_log_path, char *path_buf, int path_buf_size)
{
char *p, path[PATH_MAX];
const char *log_path = NULL, *path_base_name = "sql_log";
char *p = NULL;
char tmp_log_path[PATH_MAX], er_msg[PATH_MAX];

assert (repl_log_path != NULL && path_buf != NULL && path_buf_size >= PATH_MAX);

log_path = prm_get_string_value (PRM_ID_HA_SQL_LOG_PATH);
if (log_path != NULL && *log_path != '\0')
{
if (!IS_ABS_PATH (log_path))
{
snprintf (tmp_log_path, sizeof (tmp_log_path), "%s%s%s", repl_log_path, FILEIO_PATH_SEPARATOR (repl_log_path),
log_path);

log_path = tmp_log_path;
}
}
else
{
log_path = repl_log_path;
}

if (new_dir == NULL)
if (strlen (log_path) + 1 + strlen (path_base_name) >= path_buf_size)
{
snprintf (er_msg, sizeof (er_msg), "Too long the SQL log path \'%s\'", path_buf);

er_stack_push ();
er_set (ER_ERROR_SEVERITY, ARG_FILE_LINE, ER_HA_GENERIC_ERROR, 1, er_msg);
er_stack_pop ();

return ER_FAILED;
}

strcpy (path, new_dir);
snprintf (path_buf, path_buf_size, "%s%s%s", log_path, FILEIO_PATH_SEPARATOR (log_path), path_base_name);

p = path;
if (path[0] == '/')
p = path_buf;
if (*p == PATH_SEPARATOR)
{
p = path + 1;
p++;
}

while (p != NULL)
{
p = strchr (p, '/');
p = strchr (p, PATH_SEPARATOR);
if (p != NULL)
{
*p = '\0';
}

if (access (path, F_OK) < 0)
if (strcmp (basename (path_buf), ".") && strcmp (basename (path_buf), ".."))
{
if (mkdir (path, 0777) < 0)
if (access (path_buf, F_OK) < 0)
{
return ER_FAILED;
if (mkdir (path_buf, 0777) < 0)
{
snprintf (er_msg, sizeof (er_msg), "Failed to create SQL log directory \'%s\'", path_buf);

er_stack_push ();
er_set_with_oserror (ER_ERROR_SEVERITY, ARG_FILE_LINE, ER_HA_GENERIC_ERROR, 1, er_msg);
er_stack_pop ();

return ER_FAILED;
}
}
}

if (p != NULL)
{
*p = '/';
*p = PATH_SEPARATOR;
p++;
}
}

return NO_ERROR;
}

0 comments on commit 538fa05

Please sign in to comment.