Skip to content

Commit

Permalink
Set SPDK log level on startup and do not lock OMAP for SPDK log comma…
Browse files Browse the repository at this point in the history 
…nds.

Fixes ceph#771
Fixes ceph#772

Signed-off-by: Gil Bregman <[email protected]>
  • Loading branch information
@gbregman
gbregman committed Jul 30, 2024
1 parent 248c11b commit 135cf3d
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 73 deletions.
4 changes: 3 additions & 1 deletion ceph-nvmeof.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,9 @@ tgt_path = /usr/local/bin/nvmf_tgt
#rpc_socket_name = spdk.sock
#tgt_cmd_extra_args = --env-context="--no-huge -m1024" --iova-mode=va
timeout = 60.0
log_level = WARNING
log_level =
print_level =
protocol_log_level = WARNING

# Example value: -m 0x3 -L all
# tgt_cmd_extra_args =
Expand Down
130 changes: 61 additions & 69 deletions control/grpc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2479,29 +2479,27 @@ def get_spdk_nvmf_log_flags_and_level_safe(self, request, context):
peer_msg = self.get_peer_message(context)
self.logger.info(f"Received request to get SPDK nvmf log flags and level{peer_msg}")
log_flags = []
omap_lock = self.omap_lock.get_omap_lock_to_use(context)
with omap_lock:
try:
nvmf_log_flags = {key: value for key, value in rpc_log.log_get_flags(
self.spdk_rpc_client).items() if key.startswith('nvmf')}
for flag, flagvalue in nvmf_log_flags.items():
pb2_log_flag = pb2.spdk_log_flag_info(name = flag, enabled = flagvalue)
log_flags.append(pb2_log_flag)
spdk_log_level = rpc_log.log_get_level(self.spdk_rpc_client)
spdk_log_print_level = rpc_log.log_get_print_level(self.spdk_rpc_client)
self.logger.debug(f"spdk log flags: {nvmf_log_flags}, "
f"spdk log level: {spdk_log_level}, "
f"spdk log print level: {spdk_log_print_level}")
except Exception as ex:
errmsg = f"Failure getting SPDK log levels and nvmf log flags"
self.logger.exception(errmsg)
errmsg = f"{errmsg}:\n{ex}"
resp = self.parse_json_exeption(ex)
status = errno.ENOKEY
if resp:
status = resp["code"]
errmsg = f"Failure getting SPDK log levels and nvmf log flags: {resp['message']}"
return pb2.spdk_nvmf_log_flags_and_level_info(status = status, error_message = errmsg)
try:
nvmf_log_flags = {key: value for key, value in rpc_log.log_get_flags(
self.spdk_rpc_client).items() if key.startswith('nvmf')}
for flag, flagvalue in nvmf_log_flags.items():
pb2_log_flag = pb2.spdk_log_flag_info(name = flag, enabled = flagvalue)
log_flags.append(pb2_log_flag)
spdk_log_level = rpc_log.log_get_level(self.spdk_rpc_client)
spdk_log_print_level = rpc_log.log_get_print_level(self.spdk_rpc_client)
self.logger.debug(f"spdk log flags: {nvmf_log_flags}, "
f"spdk log level: {spdk_log_level}, "
f"spdk log print level: {spdk_log_print_level}")
except Exception as ex:
errmsg = f"Failure getting SPDK log levels and nvmf log flags"
self.logger.exception(errmsg)
errmsg = f"{errmsg}:\n{ex}"
resp = self.parse_json_exeption(ex)
status = errno.ENOKEY
if resp:
status = resp["code"]
errmsg = f"Failure getting SPDK log levels and nvmf log flags: {resp['message']}"
return pb2.spdk_nvmf_log_flags_and_level_info(status = status, error_message = errmsg)

return pb2.spdk_nvmf_log_flags_and_level_info(
nvmf_log_flags=log_flags,
Expand Down Expand Up @@ -2537,33 +2535,30 @@ def set_spdk_nvmf_logs_safe(self, request, context):

self.logger.info(f"Received request to set SPDK nvmf logs: log_level: {log_level}, print_level: {print_level}{peer_msg}")

omap_lock = self.omap_lock.get_omap_lock_to_use(context)
with omap_lock:
try:
nvmf_log_flags = [key for key in rpc_log.log_get_flags(self.spdk_rpc_client).keys() \
if key.startswith('nvmf')]
ret = [rpc_log.log_set_flag(
self.spdk_rpc_client, flag=flag) for flag in nvmf_log_flags]
self.logger.debug(f"Set SPDK nvmf log flags {nvmf_log_flags} to TRUE: {ret}")
if log_level != None:
ret_log = rpc_log.log_set_level(self.spdk_rpc_client, level=log_level)
self.logger.debug(f"Set log level to {log_level}: {ret_log}")
if print_level != None:
ret_print = rpc_log.log_set_print_level(
self.spdk_rpc_client, level=print_level)
self.logger.debug(f"Set log print level to {print_level}: {ret_print}")
except Exception as ex:
errmsg="Failure setting SPDK log levels"
self.logger.exception(errmsg)
errmsg="{errmsg}:\n{ex}"
for flag in nvmf_log_flags:
rpc_log.log_clear_flag(self.spdk_rpc_client, flag=flag)
resp = self.parse_json_exeption(ex)
status = errno.EINVAL
if resp:
status = resp["code"]
errmsg = f"Failure setting SPDK log levels: {resp['message']}"
return pb2.req_status(status=status, error_message=errmsg)
try:
nvmf_log_flags = [key for key in rpc_log.log_get_flags(self.spdk_rpc_client).keys() if key.startswith('nvmf')]
ret = [rpc_log.log_set_flag(
self.spdk_rpc_client, flag=flag) for flag in nvmf_log_flags]
self.logger.debug(f"Set SPDK nvmf log flags {nvmf_log_flags} to TRUE: {ret}")
if log_level != None:
ret_log = rpc_log.log_set_level(self.spdk_rpc_client, level=log_level)
self.logger.debug(f"Set log level to {log_level}: {ret_log}")
if print_level != None:
ret_print = rpc_log.log_set_print_level(
self.spdk_rpc_client, level=print_level)
self.logger.debug(f"Set log print level to {print_level}: {ret_print}")
except Exception as ex:
errmsg="Failure setting SPDK log levels"
self.logger.exception(errmsg)
errmsg="{errmsg}:\n{ex}"
for flag in nvmf_log_flags:
rpc_log.log_clear_flag(self.spdk_rpc_client, flag=flag)
resp = self.parse_json_exeption(ex)
status = errno.EINVAL
if resp:
status = resp["code"]
errmsg = f"Failure setting SPDK log levels: {resp['message']}"
return pb2.req_status(status=status, error_message=errmsg)

status = 0
errmsg = os.strerror(0)
Expand All @@ -2586,25 +2581,22 @@ def disable_spdk_nvmf_logs_safe(self, request, context):
peer_msg = self.get_peer_message(context)
self.logger.info(f"Received request to disable SPDK nvmf logs{peer_msg}")

omap_lock = self.omap_lock.get_omap_lock_to_use(context)
with omap_lock:
try:
nvmf_log_flags = [key for key in rpc_log.log_get_flags(self.spdk_rpc_client).keys() \
if key.startswith('nvmf')]
ret = [rpc_log.log_clear_flag(self.spdk_rpc_client, flag=flag) for flag in nvmf_log_flags]
logs_level = [rpc_log.log_set_level(self.spdk_rpc_client, level='NOTICE'),
rpc_log.log_set_print_level(self.spdk_rpc_client, level='INFO')]
ret.extend(logs_level)
except Exception as ex:
errmsg = f"Failure in disable SPDK nvmf log flags"
self.logger.exception(errmsg)
errmsg = f"{errmsg}:\n{ex}"
resp = self.parse_json_exeption(ex)
status = errno.EINVAL
if resp:
status = resp["code"]
errmsg = f"Failure in disable SPDK nvmf log flags: {resp['message']}"
return pb2.req_status(status=status, error_message=errmsg)
try:
nvmf_log_flags = [key for key in rpc_log.log_get_flags(self.spdk_rpc_client).keys() if key.startswith('nvmf')]
ret = [rpc_log.log_clear_flag(self.spdk_rpc_client, flag=flag) for flag in nvmf_log_flags]
logs_level = [rpc_log.log_set_level(self.spdk_rpc_client, level='NOTICE'),
rpc_log.log_set_print_level(self.spdk_rpc_client, level='INFO')]
ret.extend(logs_level)
except Exception as ex:
errmsg = f"Failure in disable SPDK nvmf log flags"
self.logger.exception(errmsg)
errmsg = f"{errmsg}:\n{ex}"
resp = self.parse_json_exeption(ex)
status = errno.EINVAL
if resp:
status = resp["code"]
errmsg = f"Failure in disable SPDK nvmf log flags: {resp['message']}"
return pb2.req_status(status=status, error_message=errmsg)

status = 0
errmsg = os.strerror(0)
Expand Down
18 changes: 15 additions & 3 deletions control/server.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,18 @@ def serve(self):
else:
self.logger.info(f"Prometheus endpoint is disabled. To enable, set the config option 'enable_prometheus_exporter = True'")

# Set SPDK log level
log_level_args = {}
log_level = self.config.get_with_default("spdk", "log_level", None)
print_level = self.config.get_with_default("spdk", "print_level", None)
if log_level:
log_level_args["log_level"] = log_level.upper()
if print_level:
log_level_args["print_level"] = print_level.upper()
if log_level_args:
log_req = pb2.set_spdk_nvmf_logs_req(**log_level_args)
self.gateway_rpc.set_spdk_nvmf_logs(log_req)

def _monitor_client_version(self) -> str:
"""Return monitor client version string."""
# Get the current SIGCHLD handler
Expand Down Expand Up @@ -381,7 +393,7 @@ def _start_spdk(self, omap_state):

# Initialization
timeout = self.config.getfloat_with_default("spdk", "timeout", 60.0)
log_level = self.config.get_with_default("spdk", "log_level", "WARNING")
protocol_log_level = self.config.get_with_default("spdk", "protocol_log_level", "WARNING")
# connect timeout: spdk client retries 5 times per sec
conn_retries = int(timeout * 5)
self.logger.info(f"SPDK process id: {self.spdk_process.pid}")
Expand All @@ -394,14 +406,14 @@ def _start_spdk(self, omap_state):
self.spdk_rpc_socket_path,
None,
timeout,
log_level=log_level,
log_level=protocol_log_level,
conn_retries=conn_retries,
)
self.spdk_rpc_ping_client = rpc_client.JSONRPCClient(
self.spdk_rpc_socket_path,
None,
timeout,
log_level=log_level,
log_level=protocol_log_level,
conn_retries=conn_retries,
)
except Exception:
Expand Down

0 comments on commit 135cf3d

Please sign in to comment.