Skip to content

Commit

Permalink
Merge pull request #948 from sandialabs/bugfix
Browse files Browse the repository at this point in the history 
Bugfix
  • Loading branch information
@Spurs20
Spurs20 authored May 6, 2020
2 parents 1b0fe32 + c14434f commit d460079
Show file tree
Hide file tree
Showing 8 changed files with 153 additions and 117 deletions.
6 changes: 2 additions & 4 deletions agent/parse_frames.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,11 @@ def cmdscale(D):
def create_job_logger(file_name):
"""
returns a logging function with the jid.log as the file name
changed to print for compatibility for hpc
:param jid: job id
:return:
"""
log = logging.getLogger()
log.setLevel(logging.INFO)
log.addHandler(logging.FileHandler(str(file_name)))
return lambda msg: log.log(logging.INFO, msg)
return lambda msg: print(msg)


# start of main script
Expand Down
12 changes: 10 additions & 2 deletions agent/slycat-agent-compute-timeseries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import json
import numpy
import os
import tarfile
import scipy.cluster.hierarchy
import scipy.spatial.distance
import slycat.hdf5
Expand Down Expand Up @@ -68,7 +69,13 @@
pool = ipyparallel.Client(profile=arguments.profile)[:]
except:
raise Exception("A running IPython parallel cluster is required to run this script.")

def tardir(path):
# ziph is zipfile handle
with tarfile.open(os.path.join(path, 'slycat-timeseries.tar.gz'), 'w:gz') as tarh:
for root, dirs, files in os.walk(path):
for file in files:
if file != 'slycat-timeseries.tar.gz':
tarh.add(os.path.join(root, file), arcname=file)
# Compute the model.
try:
print("Examining and verifying data.")
Expand Down Expand Up @@ -385,9 +392,10 @@ def uniform_paa(directory, min_time, max_time, bin_count, timeseries_index, attr
with open(os.path.join(dirname, "waveform_%s_values.pickle" % name), "wb") as values_file:
pickle.dump(waveform_values_array, values_file)


except:
import traceback

print((traceback.format_exc()))

tardir(dirname)
print("done.")
9 changes: 7 additions & 2 deletions agent/slycat-slurm-agent.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,19 @@ def run_remote_command(self, command):
background_thread.start()
except Exception as e:
output[0] = traceback.format_exc()
log_file_path = os.path.abspath(str(jid) + ".log")
if command["hpc"]["is_hpc_job"]:
split_output = str(output[0], 'utf-8').replace('\r', '').replace('\n', '').split(' ')
jid = split_output[len(split_output)-1]
log_file_path = os.path.join(os.getcwd(),'slurm-' + str(jid) + '.out')
results = {
"message": "ran the remote command",
"ok": True,
"jid": jid,
"command": command,
"output": output[0],
"errors": output[1],
"log_file_path": os.path.abspath(str(jid) + ".log"),
"log_file_path": log_file_path,
"available_scripts": [
{
"name": script["name"],
Expand Down Expand Up @@ -133,7 +138,7 @@ def run_shell_command(self, command, jid=0, log_to_file=False):
# execute script
value1, value2 = p.communicate()
if log_to_file:
log(str(value1))
log(str(value1,'utf-8'))
log("[RAN SCRIPT]")
return value1, value2
else:
Expand Down
121 changes: 43 additions & 78 deletions packages/slycat/web/server/remote.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -575,9 +575,46 @@ def browse(self, path, file_reject, file_allow, directory_reject, directory_allo
"mime-types": mime_types}
return response
except Exception as e:
cherrypy.response.headers["x-slycat-message"] = str(e)
cherrypy.log.error("slycat.web.server.remote.py browse", "cherrypy.HTTPError 400 %s" % str(e))
raise cherrypy.HTTPError(400)
cherrypy.log.error("Exception reading remote file %s: %s %s" % (path, type(e), str(e)))

if str(e) == "Garbage packet received":
cherrypy.response.headers["x-slycat-message"] = "Remote access failed: %s" % str(e)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 500 remote access failed: %s" % str(e))
raise cherrypy.HTTPError("500 Remote access failed.")

if str(e) == "No such file":
# Ideally this would be a 404, but we already use
# 404 to handle an unknown sessions, and clients need to make the distinction.
cherrypy.response.headers["x-slycat-message"] = "The remote file %s:%s does not exist." % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 the remote file %s:%s does not exist." % (
self.hostname, path))
raise cherrypy.HTTPError("400 File not found.")

if str(e) == "Permission denied":
# The file exists, but is not available due to access controls
cherrypy.response.headers["x-slycat-message"] = "You do not have permission to retrieve %s:%s" % (
self.hostname, path)
cherrypy.response.headers[
"x-slycat-hint"] = "Check the filesystem on %s to verify that your user has access " \
"to %s, and don't forget to set appropriate permissions on all " \
"the parent directories!" % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 you do not have permission to "
"retrieve %s:%s. Check the filesystem on %s to verify that your "
"user has access to %s, and don't forget to set appropriate permissions"
" on all the parent directories." % (
self.hostname, path, self.hostname, path))
raise cherrypy.HTTPError("400 Access denied.")

# Catchall
cherrypy.response.headers["x-slycat-message"] = "Remote access failed: %s" % str(e)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 remote access failed: %s" % str(e))
raise cherrypy.HTTPError("400 Remote access failed.")

def write_file(self, path, data, **kwargs):
'''
Expand All @@ -602,7 +639,7 @@ def write_file(self, path, data, **kwargs):
"cherrypy.HTTPError 400 must specify cache key.")
raise cherrypy.HTTPError("400 Must specify cache key.")

# Use the agent to retrieve a file.
# Use the agent to write a file.
if self._agent is not None:
stdin, stdout, stderr = self._agent
try:
Expand Down Expand Up @@ -688,78 +725,6 @@ def get_file(self, path, **kwargs):
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 must specify cache key.")
raise cherrypy.HTTPError("400 Must specify cache key.")

# Use the agent to retrieve a file.
if self._agent is not None:
stdin, stdout, stderr = self._agent
try:
cherrypy.log.error(json.dumps({"action": "get-file", "path": path}))
stdin.write("%s\n" % json.dumps({"action": "get-file", "path": path}))
stdin.flush()
except socket.error as e:
delete_session(self._sid)
raise socket.error('Socket is closed')
value = stdout.readline()
metadata = json.loads(value)
if metadata["message"] == "Path must be absolute.":
cherrypy.response.headers["x-slycat-message"] = "Remote path %s:%s is not absolute." % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 remote path %s:%s is not absolute." % (
self.hostname, path))
raise cherrypy.HTTPError("400 Path not absolute.")
elif metadata["message"] == "No read permission.":
cherrypy.response.headers["x-slycat-message"] = "You do not have permission to retrieve %s:%s" % (
self.hostname, path)
cherrypy.response.headers[
"x-slycat-hint"] = "Check the filesystem on %s to verify that your user has" \
" access to %s, and don't forget to set appropriate permissions" \
" on all the parent directories!" % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 you do not have permission to "
"retrieve %s:%s. Check the filesystem on %s to verify that"
" your user has access to %s, and don't forget to set appropriate "
"permissions on all the parent directories." % (
self.hostname, path, self.hostname, path))
raise cherrypy.HTTPError("400 Access denied.")
elif metadata["message"] == "Path not found.":
cherrypy.response.headers["x-slycat-message"] = "The remote file %s:%s does not exist." % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 the remote file %s:%s does not exist." % (
self.hostname, path))
raise cherrypy.HTTPError("400 File not found.")
elif metadata["message"] == "Directory unreadable.":
cherrypy.response.headers["x-slycat-message"] = "Remote path %s:%s is a directory." % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 can't read directory %s:%s." % (self.hostname, path))
raise cherrypy.HTTPError("400 Can't read directory.")
elif metadata["message"] == "Access denied.":
cherrypy.response.headers["x-slycat-message"] = "You do not have permission to retrieve %s:%s" % (
self.hostname, path)
cherrypy.response.headers[
"x-slycat-hint"] = "Check the filesystem on %s to verify that your user has access" \
" to %s, and don't forget to set appropriate permissions on all" \
" the parent directories!" % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 you do not have permission to"
" retrieve %s:%s. Check the filesystem on %s to verify "
"that your user has access to %s, and don't forget to set"
" appropriate permissions on all the parent directories." % (
self.hostname, path, self.hostname, path))
raise cherrypy.HTTPError("400 Access denied.")
content_type = metadata["content-type"]
content = base64.b64decode(metadata["content"])

if cache == "project":
cache_object(project, key, content_type, content)

cherrypy.response.headers["content-type"] = content_type
return content

# Use sftp to retrieve a file.
try:
if stat.S_ISDIR(self._sftp.stat(path).st_mode):
Expand Down Expand Up @@ -788,7 +753,7 @@ def get_file(self, path, **kwargs):
"cherrypy.HTTPError 500 remote access failed: %s" % str(e))
raise cherrypy.HTTPError("500 Remote access failed.")

if e.strerror == "No such file":
if str(e) == "No such file":
# Ideally this would be a 404, but we already use
# 404 to handle an unknown sessions, and clients need to make the distinction.
cherrypy.response.headers["x-slycat-message"] = "The remote file %s:%s does not exist." % (
Expand All @@ -798,7 +763,7 @@ def get_file(self, path, **kwargs):
self.hostname, path))
raise cherrypy.HTTPError("400 File not found.")

if e.strerror == "Permission denied":
if str(e) == "Permission denied":
# The file exists, but is not available due to access controls
cherrypy.response.headers["x-slycat-message"] = "You do not have permission to retrieve %s:%s" % (
self.hostname, path)
Expand Down
5 changes: 1 addition & 4 deletions web-server/js/slycat-web-client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -698,10 +698,7 @@ module.get_model_command_fetch = function(params, errorFunction)
if (!response.ok) {
throw `bad response with: ${response.status} :: ${response.statusText}`;
}
if (Object.keys(response).indexOf("json") > -1){
return response.json();
}
return {};
return response.json();
}).catch((error) => {
if (errorFunction) {
errorFunction(error)
Expand Down
5 changes: 2 additions & 3 deletions web-server/plugins/slycat-parameter-image/slycat-parameter-image.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ def update_table(database, model, verb, type, command, **kwargs):
Returns:
[json] -- {"success": "success changed linked models"}
"""

linked_models = kwargs["linked_models"]
if len(linked_models) <= 0:
response = {"success": "success nothing to change"}
Expand All @@ -111,6 +110,7 @@ def update_table(database, model, verb, type, command, **kwargs):
project_data = database.get("project_data", did)
attachment = database.get_attachment(project_data, "content")
file_attachment = attachment.read()
file_attachment = file_attachment.decode('utf-8')
models = [model for model in
database.scan("slycat/project-models", startkey=model["project"], endkey=model["project"])]
for model in models:
Expand All @@ -123,8 +123,7 @@ def update_table(database, model, verb, type, command, **kwargs):
project_data["mid"].append(model["_id"])
database.save(project_data)
database.save(model)
slycat.web.server.parse_existing_file(database, "slycat-csv-parser", True, [file_attachment], model,
"data-table")
slycat.web.server.parse_existing_file(database, "slycat-csv-parser", True, [file_attachment], model, "data-table")
response = {"success": "success changed linked models"}
return json.dumps(response)

Expand Down
Loading

0 comments on commit d460079

Please sign in to comment.