Skip to content

Add a helper function for expand_es_metadata #606

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Add a helper function for expand_es_metadata #606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bffc45e
add helper es_util call for expand_es_metadata
aschroed Sep 24, 2025
abe8375
remove commented out lines
aschroed Sep 25, 2025
74b0724
use the helper function
aschroed Sep 25, 2025
680693b
fix indent typo
aschroed Sep 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions chalicelib_fourfront/checks/helpers/es_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,20 @@
from dcicutils.es_utils import create_es_client
from dcicutils import ff_utils


def get_es_metadata(*args, **kwargs):
if (kwargs.get("es_client", None) is None) and ((es_host_local := _get_es_host_local()) is not None):
es_client = create_es_client(es_host_local, use_aws_auth=True)
return ff_utils.get_es_metadata(*args, **kwargs, es_client=es_client)
return ff_utils.get_es_metadata(*args, **kwargs)


def expand_es_metadata(*args, **kwargs):
if (kwargs.get("es_client", None) is None) and ((es_host_local := _get_es_host_local()) is not None):
es_client = create_es_client(es_host_local, use_aws_auth=True)
return ff_utils.expand_es_metadata(*args, **kwargs, es_client=es_client)
return ff_utils.expand_es_metadata(*args, **kwargs)


def _get_es_host_local() -> Optional[str]:
return os.environ.get("ES_HOST_LOCAL", None)
48 changes: 25 additions & 23 deletions chalicelib_fourfront/checks/helpers/wfr_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
from operator import itemgetter
from tibanna_4dn.core import API
from . import wfrset_utils
from chalicelib_fourfront.checks.helpers.es_utils import expand_es_metadata


lambda_limit = wfrset_utils.lambda_limit
random_wait = wfrset_utils.random_wait
Expand Down Expand Up @@ -824,13 +826,13 @@ def check_hic(res, my_auth, exp_type, check, start, lambda_limit, nore=False, no
"""Check run status for each set in res, and report missing runs and completed process"""
for a_set in res:
# get all related items
all_items, _ = ff_utils.expand_es_metadata([a_set['uuid']], my_auth,
all_items, _ = expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=['experiment_relation',
'biosample_relation',
'references',
'reference_pubs'])
'reference_pubs'],)
all_wfrs = all_items.get('workflow_run_awsem', []) + all_items.get('workflow_run_sbg', [])
now = datetime.utcnow()
print(a_set['accession'], (now-start).seconds)
Expand Down Expand Up @@ -1041,13 +1043,13 @@ def check_margi(res, my_auth, exp_type, check, start, lambda_limit, nore=False,
"""Check run status for each set in res, and report missing runs and completed process"""
for a_set in res:
# get all related items
all_items, all_uuids = ff_utils.expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=['experiment_relation',
'biosample_relation',
'references',
'reference_pubs'])
all_items, all_uuids = expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=['experiment_relation',
'biosample_relation',
'references',
'reference_pubs'],)
all_wfrs = all_items.get('workflow_run_awsem', []) + all_items.get('workflow_run_sbg', [])
now = datetime.utcnow()
print(a_set['accession'], (now-start).seconds, len(all_uuids))
Expand Down Expand Up @@ -1515,13 +1517,13 @@ def check_repli(res, my_auth, exp_type, check, start, lambda_limit, winsize=None
"""Check run status for each set in res, and report missing runs and completed process"""
for a_set in res:
# get all related items
all_items, _ = ff_utils.expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=['experiment_relation',
'biosample_relation',
'references',
'reference_pubs'])
all_items, _ = expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=['experiment_relation',
'biosample_relation',
'references',
'reference_pubs'],)
all_wfrs = all_items.get('workflow_run_awsem', []) + all_items.get('workflow_run_sbg', [])
now = datetime.utcnow()
print(a_set['accession'], (now-start).seconds)
Expand Down Expand Up @@ -1668,13 +1670,13 @@ def check_rna(res, my_auth, exp_type, check, start, lambda_limit, **kwargs):
"""Check run status for each set in res, and report missing runs and completed process"""
for a_set in res:
# get all related items
all_items, all_uuids = ff_utils.expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=['experiment_relation',
'biosample_relation',
'references',
'reference_pubs'])
all_items, _ = expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=['experiment_relation',
'biosample_relation',
'references',
'reference_pubs'],)
all_wfrs = all_items.get('workflow_run_awsem', []) + all_items.get('workflow_run_sbg', [])
now = datetime.utcnow()
# print(a_set['accession'], (now-start).seconds)
Expand Down
30 changes: 15 additions & 15 deletions chalicelib_fourfront/checks/wfr_encode_checks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from datetime import datetime
from dcicutils import ff_utils
from .helpers import wfr_utils

from chalicelib_fourfront.checks.helpers.es_utils import expand_es_metadata
# Use confchecks to import decorators object and its methods for each check module
# rather than importing check_function, action_function, CheckResult, ActionResult
# individually - they're now part of class Decorators in foursight-core::decorators
Expand Down Expand Up @@ -52,13 +52,13 @@ def chipseq_status(connection, **kwargs):

for a_set in res:
set_acc = a_set['accession']
all_items, all_uuids = ff_utils.expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=[ # 'experiment_relation',
'biosample_relation',
'references',
'reference_pubs'])
all_items, all_uuids = expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=[ # 'experiment_relation',
'biosample_relation',
'references',
'reference_pubs'])
now = datetime.utcnow()
print(a_set['accession'], (now-start).seconds, len(all_uuids))
if (now-start).seconds > lambda_limit:
Expand Down Expand Up @@ -568,13 +568,13 @@ def atacseq_status(connection, **kwargs):

for a_set in res:
set_acc = a_set['accession']
all_items, all_uuids = ff_utils.expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=['experiment_relation',
'biosample_relation',
'references',
'reference_pubs'])
all_items, all_uuids = expand_es_metadata([a_set['uuid']], my_auth,
store_frame='embedded',
add_pc_wfr=True,
ignore_field=['experiment_relation',
'biosample_relation',
'references',
'reference_pubs'])
now = datetime.utcnow()
print(a_set['accession'], (now-start).seconds, len(all_uuids))
if (now-start).seconds > lambda_limit:
Expand Down
4 changes: 2 additions & 2 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "foursight"
version = "4.9.29"
version = "4.9.30"
description = "Serverless Chalice Application for Monitoring"
authors = ["4DN-DCIC Team <[email protected]>"]
license = "MIT"
Expand Down Expand Up @@ -30,7 +30,7 @@ tibanna-ff = "^3.5.0"
# use below for deployment
foursight-core = "^5.7.0"
# use below for tests but not for deployment
# foursight-core = { git = "https://github.com/4dn-dcic/foursight-core.git", branch="master" }
# foursight-core = { git = "https://github.com/4dn-dcic/foursight-core.git", branch="ajs_fiz_lce_script_250919" }
# Need pytest-redis 3.0.2 or higher for pytest 7.4.2 (or higher).
pytest = "^7.4.2"
pytest-redis = "^3.0.2"
Expand Down
Loading