Merge pull request #1687 from 4dn-dcic/utils

July Security Update

Author: willronchetti

.github/workflows/main.yml

@@ -77,6 +77,8 @@ jobs:
           TRAVIS_JOB_ID: ff-npm-x-test-${{ github.run_number }}-
           # This will be the new environment variable name.
           TEST_JOB_ID: ff-npm-test-${{ github.run_number }}-
+          # set global_env_bucket
+          GLOBAL_ENV_BUCKET: ${{ secrets.GLOBAL_ENV_BUCKET }}
         run: |
           make remote-test-npm
 
@@ -102,6 +104,8 @@ jobs:
           TRAVIS_JOB_ID: ff-unit-x-test-${{ github.run_number }}-
           # This will be the new environment variable name.
           TEST_JOB_ID: ff-unit-test-${{ github.run_number }}-
+          # set global_env_bucket
+          GLOBAL_ENV_BUCKET: ${{ secrets.GLOBAL_ENV_BUCKET }}
         run: |
           make remote-test-unit
 

Dockerfile

@@ -74,7 +74,7 @@ RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/b
 COPY . .
 
 # Build remaining back-end
-RUN poetry install && \
+RUN poetry install --no-dev -vvv && \
     python setup_eb.py develop && \
     make fix-dist-info
 

conftest.py

@@ -1,6 +1,10 @@
+import os
 import pytest
 import tempfile
 
+from dcicutils.misc_utils import PRINT
+
+
 
 def pytest_addoption(parser):
     parser.addoption("--es", action="store", default="", dest='es',
@@ -24,3 +28,38 @@ def pytest_configure():
     # because without it some of the filenames we generate end up being too long, and critical functionality
     # ends up failing. Some socket-related filenames, for example, seem to have length limits. -kmp 5-Jun-2020
     tempfile.tempdir = '/tmp'
+
+
+PRINT("=" * 80)
+PRINT("Configuring environment variables...")
+
+my_selected_account = os.environ.get("ACCOUNT_NUMBER")
+
+# TODO: Maybe make this test programmable in env_utils sometime. -kmp 21-Jul-2022
+desired_env = 'fourfront-mastertest'
+
+my_selected_env = os.environ.get("ENV_NAME")
+
+if not my_selected_account or my_selected_account == "643366669028":
+    PRINT("The legacy account is correctly selected for testing Fourfront.")
+elif not my_selected_env:
+    print("ENV_NAME was not set. It is being set to {desired_env}.")
+    os.environ['ENV_NAME'] = desired_env
+elif my_selected_env != desired_env:
+    PRINT(f"ENV_NAME must be set to {desired_env} (or left unset) for testing. (It is set to {my_selected_env}.)")
+    exit(1)
+else:
+    PRINT(f"Leaving ENV_NAME set to {desired_env}.")
+
+old_identity = os.environ.get("IDENTITY")
+new_identity = 'C4AppConfigFourfrontMastertestApplicationConfigurationfourfrontmastertest'
+if old_identity == new_identity:
+    PRINT(f"IDENTITY is already set to the desired value ({new_identity}). That value will be used.")
+elif old_identity:
+    PRINT(f"IDENTITY is set incompatibly for ENV_NAME={desired_env}.")
+    exit(1)
+else:
+    PRINT(f"The IDENTITY environment variable is being set to {new_identity} so you can assume its credentials.")
+    os.environ['IDENTITY'] = new_identity
+
+PRINT("=" * 80)

deploy/docker/production/assume_identity.py

@@ -6,9 +6,10 @@
 
 import os
 import logging
-from dcicutils.qa_utils import override_environ
+from dcicutils.misc_utils import override_environ
 from dcicutils.deployment_utils import BasicOrchestratedFourfrontIniFileManager
 from dcicutils.secrets_utils import assume_identity
+from dcicutils.env_utils import EnvUtils
 
 
 logging.basicConfig(level=logging.INFO)
@@ -34,6 +35,8 @@ def build_production_ini_from_global_application_configuration():
 
     # build production.ini
     with override_environ(**identity):
+        # load env_utils
+        EnvUtils.init()
 
         # TODO: this probably needs configuring but minimal
         FourfrontDockerIniFileManager.build_ini_file_from_template(

deploy/docker/production/entrypoint_deployment.bash

@@ -9,7 +9,7 @@ poetry run python -m assume_identity
 # Clear db/es on fourfront-mastertest if we run an "initial" deploy
 # Do nothing on other environments
 if [ -n "${INITIAL_DEPLOYMENT}" ]; then
-  poetry run clear-db-es-contents production.ini --app-name app --env fourfront_mastertest
+  poetry run clear-db-es-contents production.ini --app-name app --env fourfront-mastertest
 fi
 
 ## Create mapping

deploy/docker/production/nginx.conf

@@ -110,7 +110,7 @@ http {
             # the below code is a "hack" for doing "if cond1 AND cond2" - Will Oct 14 21
             # if we are a *.4dnucleome.org host
             set $is_https_host "";
-            if ($host ~* ((data|mastertest)\.4dnucleome\.org)) {
+            if ($host ~* ((data|staging|mastertest)\.4dnucleome\.org)) {
                 set $is_https_host YE;
             }
 

poetry.lock

@@ -1,7 +1,7 @@
 [tool.poetry]
 # Note: Various modules refer to this system as "encoded", not "fourfront".
 name = "encoded"
-version = "4.4.17"  # 4.0.0 introduced containerization
+version = "4.4.18"  # 4.0.0 introduced containerization
 description = "4DN-DCIC Fourfront"
 authors = ["4DN-DCIC Team <[email protected]>"]
 license = "MIT"
@@ -37,14 +37,14 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = ">=3.7.1,<3.9"
-awscli = ">=1.21.3,!=1.22.52"
-boto3 = "^1.21.5"
-botocore = "^1.24.5"
+awscli = ">=1.25.36"
+boto3 = "^1.24.36"
+botocore = "^1.27.36"
 certifi = ">=2021.5.30"
 chardet = "3.0.4"
 colorama = "0.3.3"
-dcicsnovault = "^5.6.1"
-dcicutils = "^3.14.0.2b38"
+dcicsnovault = "^6.0.0"
+dcicutils = "^4.0.0"
 elasticsearch = "6.8.1"
 elasticsearch-dsl = "^6.4.0"  # TODO: port code from cgap-portal to get rid of uses
 execnet = "1.4.1"
@@ -115,8 +115,8 @@ supervisor = "^4.2.4"
 
 [tool.poetry.dev-dependencies]
 # PyCharm says boto3-stubs contains useful type hints
-botocore-stubs = "^1.24.5"
-boto3-stubs = "^1.21.5"
+boto3-stubs = "^1.24.36"
+botocore-stubs = "^1.27.36"
 coverage = ">=6.2"
 codacy-coverage = ">=1.3.11"
 coveralls = ">=3.3.1"
@@ -161,6 +161,12 @@ wheel = ">=0.29.0"  # needed for distribution but not any particular version
 # See details at https://pytest.org/en/stable/customize.html
 
 [tool.poetry.scripts]
+# dcicutils commands
+add-image-tag = "dcicutils.ecr_scripts:add_image_tag_main"
+show-global-env-bucket = "dcicutils.env_scripts:show_global_env_bucket_main"
+show-image-manifest = "dcicutils.ecr_scripts:show_image_manifest_main"
+show-image-catalog = "dcicutils.ecr_scripts:show_image_catalog_main"
+unrelease-most-recent-image = "dcicutils.ecr_scripts:unrelease_most_recent_image_main"
 # snovault commands
 batchupgrade = "snovault.batchupgrade:main"
 create-mapping = "snovault.elasticsearch.create_mapping:main"
@@ -195,7 +201,6 @@ spreadsheet-to-json = "encoded.commands.spreadsheet_to_json:main"
 update-inserts-from-server = "encoded.commands.update_inserts_from_server:main"
 verify-item = "encoded.commands.verify_item:main"
 
-
 [paste.app_factory]
 main = "encoded:main"
 

pyproject.toml

@@ -7,9 +7,11 @@
 import subprocess
 import webtest
 
-from dcicutils.env_utils import get_mirror_env_from_context, is_stg_or_prd_env
+from dcicutils.env_utils import EnvUtils, get_mirror_env_from_context, is_stg_or_prd_env
 from dcicutils.ff_utils import get_health_page
 from dcicutils.log_utils import set_logging
+from dcicutils.secrets_utils import assume_identity
+from dcicutils.misc_utils import override_environ
 from codeguru_profiler_agent import Profiler
 from sentry_sdk.integrations.pyramid import PyramidIntegration
 from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
@@ -125,6 +127,15 @@ def main(global_config, **local_config):
     """
     This function returns a Pyramid WSGI application.
     """
+    # If running in production (not a unit test or local deploy), assume identity
+    # and resolve EnvUtils
+    if not local_config.get('testing', False):
+        identity = assume_identity()
+
+        # Assume GAC and load env utils (once)
+        with override_environ(**identity):
+            # load env_utils
+            EnvUtils.init()
 
     settings = global_config
     settings.update(local_config)
@@ -208,7 +219,9 @@ def main(global_config, **local_config):
     config.registry['aws_ipset'] = netaddr.IPSet(
         record['ip_prefix'] for record in aws_ip_ranges['prefixes'] if record['service'] == 'AMAZON')
 
-    if asbool(settings.get('testing', False)):
+    doing_testing = asbool(settings.get('testing', False))
+
+    if doing_testing:
         config.include('.tests.testing_views')
 
     # Load upgrades last so that all views (including testing views) are
@@ -220,7 +233,7 @@ def main(global_config, **local_config):
     if is_stg_or_prd_env(current_env):
         sentry_sdk.init("https://[email protected]/5379985",
                         integrations=[PyramidIntegration(), SqlalchemyIntegration()])
-    elif current_env is not None:
+    elif current_env is not None and not doing_testing:
         sentry_sdk.init("https://[email protected]/5373642",
                         integrations=[PyramidIntegration(), SqlalchemyIntegration()])
 

src/encoded/__init__.py

@@ -37,7 +37,7 @@ def _run_create_mapping(app, args):
 
     try:
         my_env = get_my_env(app)
-        deploy_cfg = {'SKIP': True}  # default
+        deploy_cfg = {'SKIP': True, 'ENV_NAME': my_env}  # default
         if is_beanstalk_env(my_env):
             deploy_cfg = CreateMappingOnDeployManager.get_deploy_config(env=my_env, args=args, log=log,
                                                                         client='create_mapping_on_deploy')
@@ -48,7 +48,7 @@ def _run_create_mapping(app, args):
             deploy_cfg['ENV_NAME'] = my_env
 
         # TODO: handle these better
-        elif my_env in ['fourfront_hotseat', 'fourfront_webdev', 'fourfront_mastertest']:
+        elif my_env in ['fourfront-hotseat', 'fourfront-webdev', 'fourfront-mastertest']:
             deploy_cfg['SKIP'] = False
             deploy_cfg['WIPE_ES'] = True
             deploy_cfg['STRICT'] = True