From 642751ac9aaff57e3ec214db57faa5dd1ed352fc Mon Sep 17 00:00:00 2001 From: SooLee Date: Fri, 20 Sep 2019 16:03:58 +0000 Subject: [PATCH 1/4] unicorninputjson before checking dependency --- tibanna/core.py | 5 ++++- tibanna/ec2_utils.py | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tibanna/core.py b/tibanna/core.py index c64d25172..6110d6d74 100644 --- a/tibanna/core.py +++ b/tibanna/core.py @@ -350,11 +350,14 @@ def kill_all(self, sfn=None): else: break - def log(self, exec_arn=None, job_id=None, exec_name=None, sfn=None, postrunjson=False, runjson=False, quiet=False): + def log(self, exec_arn=None, job_id=None, exec_name=None, sfn=None, postrunjson=False, + runjson=False, unicorn_input_json=False, quiet=False): if postrunjson: suffix = '.postrun.json' elif runjson: suffix = '.run.json' + elif unicorninputjson: + suffix = '.unicorn.input.json' else: suffix = '.log' if not sfn: diff --git a/tibanna/ec2_utils.py b/tibanna/ec2_utils.py index 196af0c95..71f921218 100644 --- a/tibanna/ec2_utils.py +++ b/tibanna/ec2_utils.py @@ -330,6 +330,7 @@ def input_dict(self): return self.unicorn_input.as_dict() def prelaunch(self, profile=None): + self.upload_run_json(self.input_dict, suffix='unicorn.input.json') self.check_dependency(**self.args.dependency) runjson = self.create_run_json_dict() self.upload_run_json(runjson) @@ -605,9 +606,9 @@ def create_run_json_dict(self): del(pre['config']['key_name']) return pre - def upload_run_json(self, runjson): + def upload_run_json(self, runjson, suffix = 'run.json'): jsonbody = json.dumps(runjson, indent=4, sort_keys=True) - jsonkey = self.jobid + '.run.json' + jsonkey = self.jobid + '.' + suffix # Keep log of the final json logger.info("jsonbody=\n" + jsonbody) # copy the json file to the s3 bucket @@ -619,7 +620,7 @@ def upload_run_json(self, runjson): try: res = s3.put_object(Body=jsonbody.encode('utf-8'), Bucket=self.cfg.json_bucket, Key=jsonkey) except Exception: - raise Exception("boto3 client error: Failed to upload run.json %s to s3: %s" % (jsonkey, str(res))) + raise Exception("boto3 client error: Failed to upload %s %s to s3: %s" % (suffix, jsonkey, str(res))) def create_userdata(self, profile=None): """Create a userdata script to pass to the instance. The userdata script is run_workflow.$JOBID.sh. From dba1ba1029b0b637526d411b645850d3eb83d9e3 Mon Sep 17 00:00:00 2001 From: SooLee Date: Fri, 20 Sep 2019 16:09:34 +0000 Subject: [PATCH 2/4] added runjson/unicorninputjson option to cli log --- tibanna/__main__.py | 11 ++++++++--- tibanna_4dn/__main__.py | 6 ++++-- tibanna_cgap/__main__.py | 6 ++++-- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tibanna/__main__.py b/tibanna/__main__.py index 92253ef6e..b41ffe7ab 100644 --- a/tibanna/__main__.py +++ b/tibanna/__main__.py @@ -99,7 +99,11 @@ def args(self): "your current default is %s)" % TIBANNA_DEFAULT_STEP_FUNCTION_NAME, 'default': TIBANNA_DEFAULT_STEP_FUNCTION_NAME}, {'flag': ["-p", "--postrunjson"], - 'help': "print out postrun json instead", 'action': "store_true"}], + 'help': "print out postrun json instead", 'action': "store_true"}, + {'flag': ["-p", "--runjson"], + 'help': "print out run json instead", 'action': "store_true"}, + {'flag': ["-p", "--unicorn-input-json"], + 'help': "print out unicorn input json instead", 'action': "store_true"}], 'add_user': [{'flag': ["-u", "--user"], 'help': "user to add to a Tibanna usergroup"}, @@ -300,9 +304,10 @@ def list_sfns(numbers=False): API().list_sfns(numbers=numbers) -def log(exec_arn=None, job_id=None, exec_name=None, sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, postrunjson=False): +def log(exec_arn=None, job_id=None, exec_name=None, sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, + postrunjson=False, runjson=False, unicorn_input_json=False): """print execution log or postrun json (-p) for a job""" - print(API().log(exec_arn, job_id, exec_name, sfn, postrunjson)) + print(API().log(exec_arn, job_id, exec_name, sfn, postrunjson, runjson, unicorn_input_json)) def kill_all(sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME): diff --git a/tibanna_4dn/__main__.py b/tibanna_4dn/__main__.py index dd30fc113..e51861ae5 100644 --- a/tibanna_4dn/__main__.py +++ b/tibanna_4dn/__main__.py @@ -68,8 +68,10 @@ def list_sfns(numbers=False): API().list_sfns(numbers=numbers) -def log(exec_arn=None, job_id=None, exec_name=None, sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, postrunjson=False): - API().log(exec_arn, job_id, exec_name, sfn, postrunjson) +def log(exec_arn=None, job_id=None, exec_name=None, sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, + postrunjson=False, runjson=False, unicorn_input_json=False): + """print execution log or postrun json (-p) for a job""" + print(API().log(exec_arn, job_id, exec_name, sfn, postrunjson, runjson, unicorn_input_json)) def kill_all(sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME): diff --git a/tibanna_cgap/__main__.py b/tibanna_cgap/__main__.py index 212998be0..87bd6734b 100644 --- a/tibanna_cgap/__main__.py +++ b/tibanna_cgap/__main__.py @@ -69,8 +69,10 @@ def list_sfns(numbers=False): API().list_sfns(numbers=numbers) -def log(exec_arn=None, job_id=None, exec_name=None, sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, postrunjson=False): - API().log(exec_arn, job_id, exec_name, sfn, postrunjson) +def log(exec_arn=None, job_id=None, exec_name=None, sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, + postrunjson=False, runjson=False, unicorn_input_json=False): + """print execution log or postrun json (-p) for a job""" + print(API().log(exec_arn, job_id, exec_name, sfn, postrunjson, runjson, unicorn_input_json)) def kill_all(sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME): From bc6f3351bfb0c3758ef02e9442b6dfa7126c581b Mon Sep 17 00:00:00 2001 From: SooLee Date: Fri, 20 Sep 2019 16:10:23 +0000 Subject: [PATCH 3/4] minor fix --- tibanna/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tibanna/__main__.py b/tibanna/__main__.py index b41ffe7ab..f0074ce89 100644 --- a/tibanna/__main__.py +++ b/tibanna/__main__.py @@ -100,9 +100,9 @@ def args(self): 'default': TIBANNA_DEFAULT_STEP_FUNCTION_NAME}, {'flag': ["-p", "--postrunjson"], 'help': "print out postrun json instead", 'action': "store_true"}, - {'flag': ["-p", "--runjson"], + {'flag': ["-r", "--runjson"], 'help': "print out run json instead", 'action': "store_true"}, - {'flag': ["-p", "--unicorn-input-json"], + {'flag': ["-u", "--unicorn-input-json"], 'help': "print out unicorn input json instead", 'action': "store_true"}], 'add_user': [{'flag': ["-u", "--user"], From 1c2ec590a55091cf6eccc574612f72bf2a3f7785 Mon Sep 17 00:00:00 2001 From: SooLee Date: Fri, 20 Sep 2019 19:24:48 +0000 Subject: [PATCH 4/4] typo --- tibanna/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tibanna/core.py b/tibanna/core.py index 6110d6d74..cc0212900 100644 --- a/tibanna/core.py +++ b/tibanna/core.py @@ -356,7 +356,7 @@ def log(self, exec_arn=None, job_id=None, exec_name=None, sfn=None, postrunjson= suffix = '.postrun.json' elif runjson: suffix = '.run.json' - elif unicorninputjson: + elif unicorn_input_json: suffix = '.unicorn.input.json' else: suffix = '.log'