Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bugfix #948

Merged
merged 8 commits into from
May 6, 2020
Merged

Bugfix #948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
546849d
removing agent for getting files
Mletter1 Apr 24, 2020
dd843de
implemented tar timeseries pulls
Mletter1 Apr 27, 2020
9a7e5bd
Update slycat-timeseries-model.py
Mletter1 Apr 29, 2020
753ef74
Update slycat-web-client.js
Mletter1 Apr 30, 2020
a5b93e0
Update parse_frames.py
Mletter1 May 1, 2020
d3043b8
changing where the log file lives for VS
Mletter1 May 1, 2020
95cca3e
Fixing python 3 encoding error in update table
May 5, 2020
c14434f
Merge branch 'master' into bugfix
May 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions agent/parse_frames.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,11 @@ def cmdscale(D):
def create_job_logger(file_name):
"""
returns a logging function with the jid.log as the file name
changed to print for compatibility for hpc
:param jid: job id
:return:
"""
log = logging.getLogger()
log.setLevel(logging.INFO)
log.addHandler(logging.FileHandler(str(file_name)))
return lambda msg: log.log(logging.INFO, msg)
return lambda msg: print(msg)


# start of main script
Expand Down
12 changes: 10 additions & 2 deletions agent/slycat-agent-compute-timeseries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import json
import numpy
import os
import tarfile
import scipy.cluster.hierarchy
import scipy.spatial.distance
import slycat.hdf5
Expand Down Expand Up @@ -68,7 +69,13 @@
pool = ipyparallel.Client(profile=arguments.profile)[:]
except:
raise Exception("A running IPython parallel cluster is required to run this script.")

def tardir(path):
# ziph is zipfile handle
with tarfile.open(os.path.join(path, 'slycat-timeseries.tar.gz'), 'w:gz') as tarh:
for root, dirs, files in os.walk(path):
for file in files:
if file != 'slycat-timeseries.tar.gz':
tarh.add(os.path.join(root, file), arcname=file)
# Compute the model.
try:
print("Examining and verifying data.")
Expand Down Expand Up @@ -385,9 +392,10 @@ def uniform_paa(directory, min_time, max_time, bin_count, timeseries_index, attr
with open(os.path.join(dirname, "waveform_%s_values.pickle" % name), "wb") as values_file:
pickle.dump(waveform_values_array, values_file)


except:
import traceback

print((traceback.format_exc()))

tardir(dirname)
print("done.")
9 changes: 7 additions & 2 deletions agent/slycat-slurm-agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,19 @@ def run_remote_command(self, command):
background_thread.start()
except Exception as e:
output[0] = traceback.format_exc()
log_file_path = os.path.abspath(str(jid) + ".log")
if command["hpc"]["is_hpc_job"]:
split_output = str(output[0], 'utf-8').replace('\r', '').replace('\n', '').split(' ')
jid = split_output[len(split_output)-1]
log_file_path = os.path.join(os.getcwd(),'slurm-' + str(jid) + '.out')
results = {
"message": "ran the remote command",
"ok": True,
"jid": jid,
"command": command,
"output": output[0],
"errors": output[1],
"log_file_path": os.path.abspath(str(jid) + ".log"),
"log_file_path": log_file_path,
"available_scripts": [
{
"name": script["name"],
Expand Down Expand Up @@ -133,7 +138,7 @@ def run_shell_command(self, command, jid=0, log_to_file=False):
# execute script
value1, value2 = p.communicate()
if log_to_file:
log(str(value1))
log(str(value1,'utf-8'))
log("[RAN SCRIPT]")
return value1, value2
else:
Expand Down
121 changes: 43 additions & 78 deletions packages/slycat/web/server/remote.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,9 +575,46 @@ def browse(self, path, file_reject, file_allow, directory_reject, directory_allo
"mime-types": mime_types}
return response
except Exception as e:
cherrypy.response.headers["x-slycat-message"] = str(e)
cherrypy.log.error("slycat.web.server.remote.py browse", "cherrypy.HTTPError 400 %s" % str(e))
raise cherrypy.HTTPError(400)
cherrypy.log.error("Exception reading remote file %s: %s %s" % (path, type(e), str(e)))

if str(e) == "Garbage packet received":
cherrypy.response.headers["x-slycat-message"] = "Remote access failed: %s" % str(e)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 500 remote access failed: %s" % str(e))
raise cherrypy.HTTPError("500 Remote access failed.")

if str(e) == "No such file":
# Ideally this would be a 404, but we already use
# 404 to handle an unknown sessions, and clients need to make the distinction.
cherrypy.response.headers["x-slycat-message"] = "The remote file %s:%s does not exist." % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 the remote file %s:%s does not exist." % (
self.hostname, path))
raise cherrypy.HTTPError("400 File not found.")

if str(e) == "Permission denied":
# The file exists, but is not available due to access controls
cherrypy.response.headers["x-slycat-message"] = "You do not have permission to retrieve %s:%s" % (
self.hostname, path)
cherrypy.response.headers[
"x-slycat-hint"] = "Check the filesystem on %s to verify that your user has access " \
"to %s, and don't forget to set appropriate permissions on all " \
"the parent directories!" % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 you do not have permission to "
"retrieve %s:%s. Check the filesystem on %s to verify that your "
"user has access to %s, and don't forget to set appropriate permissions"
" on all the parent directories." % (
self.hostname, path, self.hostname, path))
raise cherrypy.HTTPError("400 Access denied.")

# Catchall
cherrypy.response.headers["x-slycat-message"] = "Remote access failed: %s" % str(e)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 remote access failed: %s" % str(e))
raise cherrypy.HTTPError("400 Remote access failed.")

def write_file(self, path, data, **kwargs):
'''
Expand All @@ -602,7 +639,7 @@ def write_file(self, path, data, **kwargs):
"cherrypy.HTTPError 400 must specify cache key.")
raise cherrypy.HTTPError("400 Must specify cache key.")

# Use the agent to retrieve a file.
# Use the agent to write a file.
if self._agent is not None:
stdin, stdout, stderr = self._agent
try:
Expand Down Expand Up @@ -688,78 +725,6 @@ def get_file(self, path, **kwargs):
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 must specify cache key.")
raise cherrypy.HTTPError("400 Must specify cache key.")

# Use the agent to retrieve a file.
if self._agent is not None:
stdin, stdout, stderr = self._agent
try:
cherrypy.log.error(json.dumps({"action": "get-file", "path": path}))
stdin.write("%s\n" % json.dumps({"action": "get-file", "path": path}))
stdin.flush()
except socket.error as e:
delete_session(self._sid)
raise socket.error('Socket is closed')
value = stdout.readline()
metadata = json.loads(value)
if metadata["message"] == "Path must be absolute.":
cherrypy.response.headers["x-slycat-message"] = "Remote path %s:%s is not absolute." % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 remote path %s:%s is not absolute." % (
self.hostname, path))
raise cherrypy.HTTPError("400 Path not absolute.")
elif metadata["message"] == "No read permission.":
cherrypy.response.headers["x-slycat-message"] = "You do not have permission to retrieve %s:%s" % (
self.hostname, path)
cherrypy.response.headers[
"x-slycat-hint"] = "Check the filesystem on %s to verify that your user has" \
" access to %s, and don't forget to set appropriate permissions" \
" on all the parent directories!" % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 you do not have permission to "
"retrieve %s:%s. Check the filesystem on %s to verify that"
" your user has access to %s, and don't forget to set appropriate "
"permissions on all the parent directories." % (
self.hostname, path, self.hostname, path))
raise cherrypy.HTTPError("400 Access denied.")
elif metadata["message"] == "Path not found.":
cherrypy.response.headers["x-slycat-message"] = "The remote file %s:%s does not exist." % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 the remote file %s:%s does not exist." % (
self.hostname, path))
raise cherrypy.HTTPError("400 File not found.")
elif metadata["message"] == "Directory unreadable.":
cherrypy.response.headers["x-slycat-message"] = "Remote path %s:%s is a directory." % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 can't read directory %s:%s." % (self.hostname, path))
raise cherrypy.HTTPError("400 Can't read directory.")
elif metadata["message"] == "Access denied.":
cherrypy.response.headers["x-slycat-message"] = "You do not have permission to retrieve %s:%s" % (
self.hostname, path)
cherrypy.response.headers[
"x-slycat-hint"] = "Check the filesystem on %s to verify that your user has access" \
" to %s, and don't forget to set appropriate permissions on all" \
" the parent directories!" % (
self.hostname, path)
cherrypy.log.error("slycat.web.server.remote.py get_file",
"cherrypy.HTTPError 400 you do not have permission to"
" retrieve %s:%s. Check the filesystem on %s to verify "
"that your user has access to %s, and don't forget to set"
" appropriate permissions on all the parent directories." % (
self.hostname, path, self.hostname, path))
raise cherrypy.HTTPError("400 Access denied.")
content_type = metadata["content-type"]
content = base64.b64decode(metadata["content"])

if cache == "project":
cache_object(project, key, content_type, content)

cherrypy.response.headers["content-type"] = content_type
return content

# Use sftp to retrieve a file.
try:
if stat.S_ISDIR(self._sftp.stat(path).st_mode):
Expand Down Expand Up @@ -788,7 +753,7 @@ def get_file(self, path, **kwargs):
"cherrypy.HTTPError 500 remote access failed: %s" % str(e))
raise cherrypy.HTTPError("500 Remote access failed.")

if e.strerror == "No such file":
if str(e) == "No such file":
# Ideally this would be a 404, but we already use
# 404 to handle an unknown sessions, and clients need to make the distinction.
cherrypy.response.headers["x-slycat-message"] = "The remote file %s:%s does not exist." % (
Expand All @@ -798,7 +763,7 @@ def get_file(self, path, **kwargs):
self.hostname, path))
raise cherrypy.HTTPError("400 File not found.")

if e.strerror == "Permission denied":
if str(e) == "Permission denied":
# The file exists, but is not available due to access controls
cherrypy.response.headers["x-slycat-message"] = "You do not have permission to retrieve %s:%s" % (
self.hostname, path)
Expand Down
5 changes: 1 addition & 4 deletions web-server/js/slycat-web-client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -698,10 +698,7 @@ module.get_model_command_fetch = function(params, errorFunction)
if (!response.ok) {
throw `bad response with: ${response.status} :: ${response.statusText}`;
}
if (Object.keys(response).indexOf("json") > -1){
return response.json();
}
return {};
return response.json();
}).catch((error) => {
if (errorFunction) {
errorFunction(error)
Expand Down
5 changes: 2 additions & 3 deletions web-server/plugins/slycat-parameter-image/slycat-parameter-image.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ def update_table(database, model, verb, type, command, **kwargs):
Returns:
[json] -- {"success": "success changed linked models"}
"""

linked_models = kwargs["linked_models"]
if len(linked_models) <= 0:
response = {"success": "success nothing to change"}
Expand All @@ -111,6 +110,7 @@ def update_table(database, model, verb, type, command, **kwargs):
project_data = database.get("project_data", did)
attachment = database.get_attachment(project_data, "content")
file_attachment = attachment.read()
file_attachment = file_attachment.decode('utf-8')
models = [model for model in
database.scan("slycat/project-models", startkey=model["project"], endkey=model["project"])]
for model in models:
Expand All @@ -123,8 +123,7 @@ def update_table(database, model, verb, type, command, **kwargs):
project_data["mid"].append(model["_id"])
database.save(project_data)
database.save(model)
slycat.web.server.parse_existing_file(database, "slycat-csv-parser", True, [file_attachment], model,
"data-table")
slycat.web.server.parse_existing_file(database, "slycat-csv-parser", True, [file_attachment], model, "data-table")
response = {"success": "success changed linked models"}
return json.dumps(response)

Expand Down
Loading