Skip to content

Commit

Permalink
reduce check_res_file complexity
Browse files Browse the repository at this point in the history
  • Loading branch information
grantbuster committed Jan 9, 2025
1 parent f10d580 commit 86a944c
Showing 1 changed file with 55 additions and 20 deletions.
75 changes: 55 additions & 20 deletions rex/utilities/utilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -317,49 +317,83 @@ def check_res_file(res_file):
Boolean flag to use h5pyd to handle .h5 'files' hosted on AWS
behind HSDS
"""

multi_h5_res = False
hsds = False
bad = True

if os.path.isfile(res_file):
bad = False
pass

elif res_file.startswith('s3://'):
try:
import fsspec
import fsspec # pylint: disable=unused-import
bad = False
except Exception as e:
msg = (f'Tried to open s3 file path: "{res_file}" with '
'fsspec but could not import, try '
'`pip install NREL-rex[s3]`')
raise ImportError(msg) from e

elif '*' in res_file:
bad = False
multi_h5_res = True

elif os.path.isdir(res_file):
bad = False
msg = ('Cannot parse directory, need to add wildcard * suffix: {}'
.format(res_file))
raise FileInputError(msg)

else:
try:
import h5pyd
hsds_dir = os.path.dirname(res_file)
with h5pyd.Folder(hsds_dir + '/') as f:
hsds = True
fps = [f'{hsds_dir}/{fn}' for fn in f
if fnmatch(f'{hsds_dir}/{fn}', res_file)]
if not any(fps):
msg = ('{} is not a valid HSDS file path!'
.format(res_file))
raise FileNotFoundError(msg)
elif len(fps) > 1:
multi_h5_res = True

except Exception as ex:
msg = ("{} is not a valid file path, and HSDS "
"cannot be check for a file at this path:{}!"
.format(res_file, ex))
raise FileNotFoundError(msg) from ex
multi_h5_res, hsds = check_hsds_file(res_file)
bad = not hsds

if bad:
msg = ("{} is not a valid file path, and HSDS "
"cannot be check for a file at this path!"
.format(res_file))
raise FileNotFoundError(msg)

return multi_h5_res, hsds


def check_hsds_file(res_file):
"""
Check resource to see if the given path
- It belongs to a multi-file handler
- Is a hsds path
Parameters
----------
res_file : str
Filepath to single resource file, unix style multi-file path like
/h5_dir/prefix*suffix.h5, an hsds filepath
(filename of hsds path can also contain wildcards *), or
an s3 filepath starting with "s3://"
Returns
-------
multi_h5_res : bool
Boolean flag to use a MultiFileResource handler
hsds : bool
Boolean flag to use h5pyd to handle .h5 'files' hosted on AWS
behind HSDS
"""
import h5pyd
hsds_dir = os.path.dirname(res_file)

with h5pyd.Folder(hsds_dir + '/') as f:
hsds = True
fps = [f'{hsds_dir}/{fn}' for fn in f
if fnmatch(f'{hsds_dir}/{fn}', res_file)]
if not any(fps):
msg = ('{} is not a valid HSDS file path!'
.format(res_file))
raise FileNotFoundError(msg)
elif len(fps) > 1:
multi_h5_res = True

return multi_h5_res, hsds

Expand Down Expand Up @@ -572,6 +606,7 @@ class Retry:
"""
Retry Decorator to run a function multiple times
"""

def __init__(self, tries=3, n_sec=1):
"""
Parameters
Expand Down

0 comments on commit 86a944c

Please # to comment.