Skip to content

Commit

Permalink
Fixing issues:
Browse files Browse the repository at this point in the history 
https://github.com/rijksuniversiteit-groningen/iBridges/issues/60
https://github.com/rijksuniversiteit-groningen/iBridges/issues/59
https://github.com/rijksuniversiteit-groningen/iBridges/issues/57
https://github.com/rijksuniversiteit-groningen/iBridges/issues/58
  • Loading branch information
@chStaiger
chStaiger committed Jan 17, 2023
1 parent 621fe54 commit caf6b48
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 91 deletions.
172 changes: 97 additions & 75 deletions irods-iBridgesCli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

from utils.elabConnector import elabConnector
from utils.IrodsConnector import IrodsConnector
from utils.IrodsConnectorIcommands import IrodsConnectorIcommands
from irods.exception import ResourceDoesNotExist

import configparser
Expand All @@ -37,8 +38,6 @@ def getConfig(path):
args = config._sections
if not 'iRODS' in args:
raise AttributeError("iRODS environment not defined.")
if len(args) == 1:
print(BLUE+"INFO: No metadata store configured. Only upload data to iRODS."+DEFAULT)

return args

Expand All @@ -50,20 +49,24 @@ def connectIRODS(config):
if os.path.exists(standardEnv) and \
(config['iRODS']['irodsenv'] == '' or config['iRODS']['irodsenv'] == standardEnv):
try:
ic = IrodsConnector()
if ic.icommands:
print(BLUE+"INFO: iCommands and standard environment file are present.")
print("INFO: Using iCommands for data up and download."+DEFAULT)
else:
raise Exception('No iCommands found!')
ic = IrodsConnectorIcommands()
print(BLUE+"INFO: Icommands and standard environment file are present.")
print("INFO: Using icommands for data up and download."+DEFAULT)
except ConnectionRefusedError:
raise
except FileNotFoundError:
raise
except EnvironmentError:
print("INFO: Connect with python API")
passwd = getpass.getpass(
'Password for '+os.environ['HOME']+'/.irods/irods_environment.json'+': ')
ic = IrodsConnector(standardEnv, passwd)

except Exception as e:
raise

elif os.path.exists(config['iRODS']['irodsenv']):
print("INFO: Connect with python API")
passwd = getpass.getpass(
'Password for '+os.environ['HOME']+'/.irods/irods_environment.json'+': ')
ic = IrodsConnector(config['iRODS']['irodsenv'], passwd)
Expand All @@ -88,10 +91,8 @@ def setupIRODS(config, operation):
coll = ic.ensure_coll(config['iRODS']['irodscoll'])
print(YEL+'Uploading to '+config['iRODS']['irodscoll']+DEFAULT)
except:
print(RED+"Collection path not set or invalid: "+ config['iRODS']['irodscoll']+DEFAULT)
print(RED+"Collection path not set in config or invalid: "+ config['iRODS']['irodscoll']+DEFAULT)
success = False
print(''.join([coll.path+'\n' for coll
in ic.session.collections.get("/"+ic.session.zone+"/home").subcollections]))
while not success:
iPath = input('Choose iRODS collection: ')
try:
Expand All @@ -102,27 +103,33 @@ def setupIRODS(config, operation):
except:
print(RED+"Collection path not valid: "+ config['iRODS']['irodscoll']+DEFAULT)

# set iRODS resource
#set iRODS resource, can be located in ibridges config or in irods_environment, ibridges gets priority
try:
resc_name = config['iRODS']['irodsresc']
free_space = f'{ic.resources[resc_name]["free_space"]}GiB'
print(f'{resc_name} upload capacity, free space: {free_space}')
except ResourceDoesNotExist:
resc_name = config['iRODS']['irodsresc']
print(f'{RED}iRODS resource does not exist: {resc_name}{DEFAULT}')
resc_names, free_spaces = ic.list_resources()
largest_resc = resc_names[free_spaces.index(max(free_spaces))]
max_free_space = f'{max(free_spaces)}GiB'
menu = 'y'
menu = input(f'Choose {largest_resc} ({max_free_space} free)? (Yes/No) ')
if menu in ['Yes', 'yes', 'Y', 'y']:
config['iRODS']['irodsresc'] = largest_resc
resource = ic.get_resource(config['iRODS']['irodsresc'])
if ic.resourceSize(resource.name) is None:
print(config['iRODS']['irodsresc']+ " upload capacity, free space: No inofrmation")
else:
print("Aborted: no iRODS resource set.")
print(config['iRODS']['irodsresc']+ " upload capacity, free space: "+ \
str(round(int(ic.resourceSize(resource.name))/1000**3))+'GB')

except ResourceDoesNotExist:
print(RED+'iRODS resource does not exist: '+config['iRODS']['irodsresc']+DEFAULT)
try:
resource = ic.get_resource(ic.default_resc)
config['iRODS']['irodsresc'] = ic.default_resc
except:
print(RED+"No resource set in environment file either ('default_resource_name')"+DEFAULT)
print(RED+"ERROR: No resource set"+DEFAULT)
ic.session.cleanup()
sys.exit(2)
try:
print(ic.default_resc+" upload capacity, free space: " \
+str(ic.resource_space(ic.default_resc)))
except:
print(ic.default_resc+ " upload capacity, free space: not set")

return ic


def setupELN(config):
md = elabConnector(config['ELN']['token'])
Expand Down Expand Up @@ -162,23 +169,28 @@ def prepareUpload(dataPath, ic, config):
print('Aborted: Data path not given')
return False
else:
pass
pass
#store verified dataPath
config["iRODS"]["uploadItem"] = dataPath

size = get_local_size([dataPath])
freeSpace = ic.get_resource(config['iRODS']['irodsresc']).free_space

print('Checking storage capacity for '+dataPath+', '+str(float(size)/(1024**3))+'GB')

if int(freeSpace)-1024**3 < size:
freeSpace = int(ic.get_free_space(config['iRODS']['irodsresc']))
print('Checking storage capacity for '+dataPath+', '+str(float(size)/(1000**3))+'GB')

if freeSpace != None and int(freeSpace)-1000**3 < size:
print(RED+'Not enough space left on iRODS resource.'+DEFAULT)
print('Aborted: Not enough space left.')
return False
res = input('Do you want to force the upload (Y/N): ')
if res != 'Y':
print('Aborted: Not enough space left.')
return False
else:
return True
else:
return True


def prepareDownload(irodsItemPath, ic, config):
if not ic.session.data_objects.exists(irodsItemPath) or \
if not ic.session.data_objects.exists(irodsItemPath) and \
not ic.session.collections.exists(irodsItemPath):
print(RED+'iRODS path does not exist'+DEFAULT)
menu = input('Do you want to specify a new iRODS path? (Y/N)')
Expand All @@ -193,50 +205,58 @@ def prepareDownload(irodsItemPath, ic, config):
print('Aborted: iRODS path not given')
return False
else:
pass
config["iRODS"]["downloadItem"] = irodsItemPath

if not 'DOWNLOAD' in config.keys():
config['DOWNLOAD'] = {'path': ''}

if config['DOWNLOAD']['path'] == '' or os.path.isfile(config['DOWNLOAD']['path']):
print(RED+'No download directory given'+DEFAULT)
menu = input('Do you want to specify a new iRODS path? (Y/N)')
if menu in ['YES', 'Yes', 'Y', 'y', '']:
success = False
while not success:
dataPath = input('Download directory: ')
success = ensure_dir(dataPath)
config["DOWNLOAD"]["path"] = dataPath
else:
print('Aborted: download directory not given')
return False
success = False
while not success:
dataPath = input('Download directory: ')
success = ensure_dir(dataPath)
if not success:
abort = input('Abort download? (Y/N): ')
if abort == "Y":
ic.session.cleanup()
sys.exit(2)
else:
config["DOWNLOAD"]["path"] = dataPath
return ensure_dir(config['DOWNLOAD']['path'])
else:
return ensure_dir(config['DOWNLOAD']['path'])

return True

def printHelp():
print('Data upload client')
print('Uploads local data to iRODS, and, if specified, links dat to an entry in a metadata store (ELN).')
print('Usage: ./iUpload.py -c, --config= \t config file')
print('\t\t -d, --data= \t datapath')
print('\t\t -i, --irods= \t irodspath (download)')
print('Examples:')
print('Downloading: ./irods-iBridgesCli.py -c <yourConfigFile> --irods=/npecZone/home')
print('Uploading: ./irods-iBridgesCli.py -c <yourConfigFile> --data=/my/data/path')

def main(argv):

irodsEnvPath = os.path.expanduser('~')+ os.sep +".irods"
setup_logger(irodsEnvPath, "iBridgesCli")
#setup_logger(irodsEnvPath, "iBridgesCli")

try:
opts, args = getopt.getopt(argv,"hc:d:i:",["config=", "data=", "irods="])
except getopt.GetoptError:
print('iUpload -h')
print(RED+"ERROR: incorrect usage."+DEFAULT)
printHelp()
sys.exit(2)

config = None
operation = None

for opt, arg in opts:
if opt == '-h':
print('Data upload client')
print('Uploads local data to iRODS, and, if specified, links dat to an entry in a metadata store (ELN).')
print('Usage: ./iUpload.py -c, --config= \t config file')
print('\t\t -d, --data= \t datapath (upload)')
print('\t\t -i, --irods= \t irodspath (download)')
print('Examples:')
print('Downloading: ./irods-iBridgesCli.py -c <youConfigFile> --irods=/npecZone/home')
print('Uploading: ./irods-iBridgesCli.py -c <youConfigFile> --data=/my/data/path')
printHelp()
sys.exit(2)
elif opt in ['-c', '--config']:
try:
Expand All @@ -260,18 +280,15 @@ def main(argv):
else:
dataPath = arg
else:
print('Data upload client')
print('Uploads local data to iRODS, and, if specified, links dat to an entry in a metadata store (ELN).')
print('Usage: ./iUpload.py -c, --config= \t config file')
print('\t\t -d, --data= \t datapath')
print('\t\t -i, --irods= \t irodspath (download)')
print('Examples:')
print('Downloading: ./irods-iBridgesCli.py -c <youConfigFile> --irods=/npecZone/home')
print('Uploading: ./irods-iBridgesCli.py -c <youConfigFile> --data=/my/data/path')

printHelp()
sys.exit(2)

#initialise iRODS
if operation == None:
print(RED+"ERROR: missing parameter."+DEFAULT)
printHelp()
sys.exit(2)

ic = setupIRODS(config, operation)
#initialise medata store connetcions
if 'ELN' in config and operation == 'upload':
Expand All @@ -281,19 +298,21 @@ def main(argv):

#check files for upload
if operation == 'upload':
if len(config) == 1:
print(BLUE+"INFO: No metadata store configured. Only upload data to iRODS."+DEFAULT)
if prepareUpload(dataPath, ic, config):
if md != None:
iPath = config['iRODS']['irodscoll']+'/'+md.__name__+'/'+ \
str(config['ELN']['group'])+'/'+str(config['ELN']['experiment'])
elif os.path.isdir(dataPath):
iPath = config['iRODS']['irodscoll']+'/'+os.path.basename(dataPath)
#elif os.path.isdir(dataPath):
# iPath = config['iRODS']['irodscoll']+'/'+os.path.basename(dataPath)
else:
iPath = config['iRODS']['irodscoll']
ic.ensure_coll(iPath)
print('DEBUG: Created/Ensured iRODS collection '+iPath)
iColl = ic.session.collections.get(iPath)
ic.upload_data(dataPath, iColl, config['iRODS']['irodsresc'], get_local_size([dataPath]))
dataPath = config["iRODS"]["uploadItem"]
ic.upload_data(dataPath, iColl, config['iRODS']['irodsresc'], get_local_size([dataPath]), force=True)
else:
ic.session.cleanup()
sys.exit(2)
#tag data in iRODS and metadata store
if md != None:
Expand All @@ -310,24 +329,27 @@ def main(argv):
print(BLUE+'Upload complete with the following parameters:')
print(json.dumps(config, indent=4))
print(DEFAULT)
ic.session.cleanup()
elif operation == 'download':
print(json.dumps(config, indent=4))
#print(json.dumps(config, indent=4))
if prepareDownload(irodsPath, ic, config):
downloadDir = config['DOWNLOAD']['path']
irodsDataPath = config["iRODS"]["downloadItem"]
print(YEL,
'Downloading: '+irodsDataPath+', '+str(ic.get_irods_size([irodsDataPath])/1024**3)+'GiB',
'Downloading: '+irodsDataPath+', '+str(ic.get_irods_size([irodsDataPath])/1000**3)+'GB',
DEFAULT)
try:
item = ic.session.collections.get(irodsDataPath)
except:
item = ic.session.data_objects.get(irodsDataPath)
ic.download_data(item, downloadDir, ic.get_irods_size([irodsDataPath]), force=False)
ic.download_data(item, downloadDir, ic.get_irods_size([irodsDataPath]), force = False)
print()
print(BLUE+'Download complete with the following parameters:')
print(json.dumps(config, indent=4))
print(DEFAULT)
ic.session.cleanup()
else:
ic.session.cleanup()
sys.exit(2)
else:
print('Not an implemented operation.')
Expand Down
Loading

0 comments on commit caf6b48

Please sign in to comment.