diff --git a/chalicelib_fourfront/checks/helpers/es_utils.py b/chalicelib_fourfront/checks/helpers/es_utils.py index 8b825cd6..9b978012 100644 --- a/chalicelib_fourfront/checks/helpers/es_utils.py +++ b/chalicelib_fourfront/checks/helpers/es_utils.py @@ -3,6 +3,7 @@ from dcicutils.es_utils import create_es_client from dcicutils import ff_utils + def get_es_metadata(*args, **kwargs): if (kwargs.get("es_client", None) is None) and ((es_host_local := _get_es_host_local()) is not None): es_client = create_es_client(es_host_local, use_aws_auth=True) @@ -10,5 +11,12 @@ def get_es_metadata(*args, **kwargs): return ff_utils.get_es_metadata(*args, **kwargs) +def expand_es_metadata(*args, **kwargs): + if (kwargs.get("es_client", None) is None) and ((es_host_local := _get_es_host_local()) is not None): + es_client = create_es_client(es_host_local, use_aws_auth=True) + return ff_utils.expand_es_metadata(*args, **kwargs, es_client=es_client) + return ff_utils.expand_es_metadata(*args, **kwargs) + + def _get_es_host_local() -> Optional[str]: return os.environ.get("ES_HOST_LOCAL", None) diff --git a/chalicelib_fourfront/checks/helpers/wfr_utils.py b/chalicelib_fourfront/checks/helpers/wfr_utils.py index 59aef2e9..7d670617 100644 --- a/chalicelib_fourfront/checks/helpers/wfr_utils.py +++ b/chalicelib_fourfront/checks/helpers/wfr_utils.py @@ -10,6 +10,8 @@ from operator import itemgetter from tibanna_4dn.core import API from . import wfrset_utils +from chalicelib_fourfront.checks.helpers.es_utils import expand_es_metadata + lambda_limit = wfrset_utils.lambda_limit random_wait = wfrset_utils.random_wait @@ -824,13 +826,13 @@ def check_hic(res, my_auth, exp_type, check, start, lambda_limit, nore=False, no """Check run status for each set in res, and report missing runs and completed process""" for a_set in res: # get all related items - all_items, _ = ff_utils.expand_es_metadata([a_set['uuid']], my_auth, + all_items, _ = expand_es_metadata([a_set['uuid']], my_auth, store_frame='embedded', add_pc_wfr=True, ignore_field=['experiment_relation', 'biosample_relation', 'references', - 'reference_pubs']) + 'reference_pubs'],) all_wfrs = all_items.get('workflow_run_awsem', []) + all_items.get('workflow_run_sbg', []) now = datetime.utcnow() print(a_set['accession'], (now-start).seconds) @@ -1041,13 +1043,13 @@ def check_margi(res, my_auth, exp_type, check, start, lambda_limit, nore=False, """Check run status for each set in res, and report missing runs and completed process""" for a_set in res: # get all related items - all_items, all_uuids = ff_utils.expand_es_metadata([a_set['uuid']], my_auth, - store_frame='embedded', - add_pc_wfr=True, - ignore_field=['experiment_relation', - 'biosample_relation', - 'references', - 'reference_pubs']) + all_items, all_uuids = expand_es_metadata([a_set['uuid']], my_auth, + store_frame='embedded', + add_pc_wfr=True, + ignore_field=['experiment_relation', + 'biosample_relation', + 'references', + 'reference_pubs'],) all_wfrs = all_items.get('workflow_run_awsem', []) + all_items.get('workflow_run_sbg', []) now = datetime.utcnow() print(a_set['accession'], (now-start).seconds, len(all_uuids)) @@ -1515,13 +1517,13 @@ def check_repli(res, my_auth, exp_type, check, start, lambda_limit, winsize=None """Check run status for each set in res, and report missing runs and completed process""" for a_set in res: # get all related items - all_items, _ = ff_utils.expand_es_metadata([a_set['uuid']], my_auth, - store_frame='embedded', - add_pc_wfr=True, - ignore_field=['experiment_relation', - 'biosample_relation', - 'references', - 'reference_pubs']) + all_items, _ = expand_es_metadata([a_set['uuid']], my_auth, + store_frame='embedded', + add_pc_wfr=True, + ignore_field=['experiment_relation', + 'biosample_relation', + 'references', + 'reference_pubs'],) all_wfrs = all_items.get('workflow_run_awsem', []) + all_items.get('workflow_run_sbg', []) now = datetime.utcnow() print(a_set['accession'], (now-start).seconds) @@ -1668,13 +1670,13 @@ def check_rna(res, my_auth, exp_type, check, start, lambda_limit, **kwargs): """Check run status for each set in res, and report missing runs and completed process""" for a_set in res: # get all related items - all_items, all_uuids = ff_utils.expand_es_metadata([a_set['uuid']], my_auth, - store_frame='embedded', - add_pc_wfr=True, - ignore_field=['experiment_relation', - 'biosample_relation', - 'references', - 'reference_pubs']) + all_items, _ = expand_es_metadata([a_set['uuid']], my_auth, + store_frame='embedded', + add_pc_wfr=True, + ignore_field=['experiment_relation', + 'biosample_relation', + 'references', + 'reference_pubs'],) all_wfrs = all_items.get('workflow_run_awsem', []) + all_items.get('workflow_run_sbg', []) now = datetime.utcnow() # print(a_set['accession'], (now-start).seconds) diff --git a/chalicelib_fourfront/checks/wfr_encode_checks.py b/chalicelib_fourfront/checks/wfr_encode_checks.py index 3581c9de..1343269c 100644 --- a/chalicelib_fourfront/checks/wfr_encode_checks.py +++ b/chalicelib_fourfront/checks/wfr_encode_checks.py @@ -1,7 +1,7 @@ from datetime import datetime from dcicutils import ff_utils from .helpers import wfr_utils - +from chalicelib_fourfront.checks.helpers.es_utils import expand_es_metadata # Use confchecks to import decorators object and its methods for each check module # rather than importing check_function, action_function, CheckResult, ActionResult # individually - they're now part of class Decorators in foursight-core::decorators @@ -52,13 +52,13 @@ def chipseq_status(connection, **kwargs): for a_set in res: set_acc = a_set['accession'] - all_items, all_uuids = ff_utils.expand_es_metadata([a_set['uuid']], my_auth, - store_frame='embedded', - add_pc_wfr=True, - ignore_field=[ # 'experiment_relation', - 'biosample_relation', - 'references', - 'reference_pubs']) + all_items, all_uuids = expand_es_metadata([a_set['uuid']], my_auth, + store_frame='embedded', + add_pc_wfr=True, + ignore_field=[ # 'experiment_relation', + 'biosample_relation', + 'references', + 'reference_pubs']) now = datetime.utcnow() print(a_set['accession'], (now-start).seconds, len(all_uuids)) if (now-start).seconds > lambda_limit: @@ -568,13 +568,13 @@ def atacseq_status(connection, **kwargs): for a_set in res: set_acc = a_set['accession'] - all_items, all_uuids = ff_utils.expand_es_metadata([a_set['uuid']], my_auth, - store_frame='embedded', - add_pc_wfr=True, - ignore_field=['experiment_relation', - 'biosample_relation', - 'references', - 'reference_pubs']) + all_items, all_uuids = expand_es_metadata([a_set['uuid']], my_auth, + store_frame='embedded', + add_pc_wfr=True, + ignore_field=['experiment_relation', + 'biosample_relation', + 'references', + 'reference_pubs']) now = datetime.utcnow() print(a_set['accession'], (now-start).seconds, len(all_uuids)) if (now-start).seconds > lambda_limit: diff --git a/pyproject.toml b/pyproject.toml index 4531af3b..d2461848 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "foursight" -version = "4.9.29" +version = "4.9.30" description = "Serverless Chalice Application for Monitoring" authors = ["4DN-DCIC Team "] license = "MIT" @@ -30,7 +30,7 @@ tibanna-ff = "^3.5.0" # use below for deployment foursight-core = "^5.7.0" # use below for tests but not for deployment -# foursight-core = { git = "https://github.com/4dn-dcic/foursight-core.git", branch="master" } +# foursight-core = { git = "https://github.com/4dn-dcic/foursight-core.git", branch="ajs_fiz_lce_script_250919" } # Need pytest-redis 3.0.2 or higher for pytest 7.4.2 (or higher). pytest = "^7.4.2" pytest-redis = "^3.0.2"