Implement dependency mapping for environment modules, galaxy packages, and Conda.

Author: jmchilton

lib/galaxy/tools/deps/resolvers/__init__.py

@@ -5,6 +5,9 @@
     abstractproperty,
 )
 
+import yaml
+
+from galaxy.util import listify
 from galaxy.util.dictifiable import Dictifiable
 
 from ..requirements import ToolRequirement
@@ -65,6 +68,72 @@ def _to_requirement(self, name, version=None):
         return ToolRequirement(name=name, type="package", version=version)
 
 
+class MappableDependencyResolver:
+    """Mix this into a ``DependencyResolver`` to allow mapping files.
+
+    Mapping files allow adapting generic requirements to specific local implementations.
+    """
+
+    def _setup_mapping(self, dependency_manager, **kwds):
+        mapping_files = dependency_manager.get_resolver_option(self, "mapping_file", explicit_resolver_options=kwds)
+        mappings = []
+        if mapping_files:
+            mapping_files = listify(mapping_files)
+            for mapping_file in mapping_files:
+                mappings.extend(MappableDependencyResolver._mapping_file_to_list(mapping_file))
+        self._mappings = mappings
+
+    @staticmethod
+    def _mapping_file_to_list(mapping_file):
+        with open(mapping_file, "r") as f:
+            raw_mapping = yaml.load(f)
+            return map(RequirementMapping.from_dict, raw_mapping)
+
+    def _expand_mappings(self, requirement):
+        for mapping in self._mappings:
+            if requirement.name == mapping.from_name:
+                if mapping.from_version is not None and mapping.from_version != requirement.version:
+                    continue
+
+                requirement = requirement.copy()
+                requirement.name = mapping.to_name
+                if mapping.to_version is not None:
+                    requirement.version = mapping.to_version
+
+                break
+
+        return requirement
+
+
+class RequirementMapping(object):
+
+    def __init__(self, from_name, from_version, to_name, to_version):
+        self.from_name = from_name
+        self.from_version = from_version
+        self.to_name = to_name
+        self.to_version = to_version
+
+    @staticmethod
+    def from_dict(raw_mapping):
+        from_raw = raw_mapping.get("from")
+        if isinstance(from_raw, dict):
+            from_name = from_raw.get("name")
+            from_version = str(from_raw.get("version"))
+        else:
+            from_name = from_raw
+            from_version = None
+
+        to_raw = raw_mapping.get("to")
+        if isinstance(to_raw, dict):
+            to_name = to_raw.get("name", from_name)
+            to_version = str(to_raw.get("version"))
+        else:
+            to_name = to_raw
+            to_version = None
+
+        return RequirementMapping(from_name, from_version, to_name, to_version)
+
+
 class SpecificationAwareDependencyResolver:
     """Mix this into a :class:`DependencyResolver` to implement URI specification matching.
 

lib/galaxy/tools/deps/resolvers/conda.py

@@ -29,6 +29,7 @@
     DependencyResolver,
     InstallableDependencyResolver,
     ListableDependencyResolver,
+    MappableDependencyResolver,
     MultipleDependencyResolver,
     NullDependency,
     SpecificationPatternDependencyResolver,
@@ -42,7 +43,7 @@
 log = logging.getLogger(__name__)
 
 
-class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver):
+class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver):
     dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install']
     resolver_type = "conda"
     config_options = {
@@ -57,6 +58,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
     _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+")
 
     def __init__(self, dependency_manager, **kwds):
+        self._setup_mapping(dependency_manager, **kwds)
         self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
         self.dependency_manager = dependency_manager
 
@@ -146,7 +148,7 @@ def resolve_all(self, requirements, **kwds):
 
         conda_targets = []
         for requirement in requirements:
-            requirement = self._expand_specs(requirement)
+            requirement = self._expand_requirement(requirement)
 
             version = requirement.version
             if self.versionless:
@@ -186,7 +188,7 @@ def merged_environment_name(self, conda_targets):
             return conda_targets[0].install_environment
 
     def resolve(self, requirement, **kwds):
-        requirement = self._expand_specs(requirement)
+        requirement = self._expand_requirement(requirement)
         name, version, type = requirement.name, requirement.version, requirement.type
 
         # Check for conda just not being there, this way we can enable
@@ -236,6 +238,9 @@ def resolve(self, requirement, **kwds):
             preserve_python_environment=preserve_python_environment,
         )
 
+    def _expand_requirement(self, requirement):
+        return self._expand_specs(self._expand_mappings(requirement))
+
     def list_dependencies(self):
         for install_target in installed_conda_targets(self.conda_context):
             name = install_target.package

lib/galaxy/tools/deps/resolvers/galaxy_packages.py

@@ -16,6 +16,7 @@
     Dependency,
     DependencyResolver,
     ListableDependencyResolver,
+    MappableDependencyResolver,
     NullDependency,
 )
 
@@ -103,9 +104,17 @@ def _galaxy_package_dep( self, path, version, name, exact ):
         return NullDependency(version=version, name=name)
 
 
-class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver):
+class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver, MappableDependencyResolver):
     resolver_type = "galaxy_packages"
 
+    def __init__(self, dependency_manager, **kwds):
+        super(GalaxyPackageDependencyResolver, self).__init__(dependency_manager, **kwds)
+        self._setup_mapping(dependency_manager, **kwds)
+
+    def resolve(self, requirement, **kwds):
+        requirement = self._expand_mappings(requirement)
+        return super(GalaxyPackageDependencyResolver, self).resolve(requirement, **kwds)
+
     def list_dependencies(self):
         base_path = self.base_path
         for package_name in listdir(base_path):

lib/galaxy/tools/deps/resolvers/modules.py

@@ -13,7 +13,12 @@
 
 from six import StringIO
 
-from ..resolvers import Dependency, DependencyResolver, NullDependency
+from ..resolvers import (
+    Dependency,
+    DependencyResolver,
+    MappableDependencyResolver,
+    NullDependency,
+)
 
 log = logging.getLogger( __name__ )
 
@@ -24,11 +29,12 @@
 UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s"
 
 
-class ModuleDependencyResolver(DependencyResolver):
+class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver):
     dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath']
     resolver_type = "modules"
 
     def __init__(self, dependency_manager, **kwds):
+        self._setup_mapping(dependency_manager, **kwds)
         self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
         find_by = kwds.get('find_by', 'avail')
         prefetch = _string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH))
@@ -52,6 +58,7 @@ def __default_modulespath(self):
         return module_path
 
     def resolve(self, requirement, **kwds):
+        requirement = self._expand_mappings(requirement)
         name, version, type = requirement.name, requirement.version, requirement.type
 
         if type != "package":

test/unit/tools/test_tool_deps.py

@@ -115,13 +115,7 @@ def __build_ts_test_package(base_path, script_contents=''):
 
 def test_module_dependency_resolver():
     with __test_base_path() as temp_directory:
-        module_script = os.path.join(temp_directory, "modulecmd")
-        __write_script(module_script, '''#!/bin/sh
-cat %s/example_output 1>&2;
-''' % temp_directory)
-        with open(os.path.join(temp_directory, "example_output"), "w") as f:
-            # Subset of module avail from MSI cluster.
-            f.write('''
+        module_script = _setup_module_command(temp_directory, '''
 -------------------------- /soft/modules/modulefiles ---------------------------
 JAGS/3.2.0-gcc45
 JAGS/3.3.0-gcc4.7.2
@@ -141,7 +135,7 @@ def test_module_dependency_resolver():
 advisor/2013/update2    intel/11.1.080          mkl/10.2.5.035
 advisor/2013/update3    intel/12.0              mkl/10.2.7.041
 ''')
-        resolver = ModuleDependencyResolver(None, modulecmd=module_script)
+        resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script)
         module = resolver.resolve( ToolRequirement( name="R", version=None, type="package" ) )
         assert module.module_name == "R"
         assert module.module_version is None
@@ -154,6 +148,74 @@ def test_module_dependency_resolver():
         assert isinstance(module, NullDependency)
 
 
+def test_module_resolver_with_mapping():
+    with __test_base_path() as temp_directory:
+        module_script = _setup_module_command(temp_directory, '''
+-------------------------- /soft/modules/modulefiles ---------------------------
+blast/2.24
+''')
+        mapping_file = os.path.join(temp_directory, "mapping")
+        with open(mapping_file, "w") as f:
+            f.write('''
+- from: blast+
+  to: blast
+''')
+
+        resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file)
+        module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) )
+        assert module.module_name == "blast"
+        assert module.module_version == "2.24", module.module_version
+
+
+def test_module_resolver_with_mapping_versions():
+    with __test_base_path() as temp_directory:
+        module_script = _setup_module_command(temp_directory, '''
+-------------------------- /soft/modules/modulefiles ---------------------------
+blast/2.22.0-mpi
+blast/2.23
+blast/2.24.0-mpi
+''')
+        mapping_file = os.path.join(temp_directory, "mapping")
+        with open(mapping_file, "w") as f:
+            f.write('''
+- from:
+    name: blast+
+    version: 2.24
+  to:
+    name: blast
+    version: 2.24.0-mpi
+- from:
+    name: blast
+    version: 2.22
+  to:
+    version: 2.22.0-mpi
+''')
+
+        resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file)
+        module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) )
+        assert module.module_name == "blast"
+        assert module.module_version == "2.24.0-mpi", module.module_version
+
+        resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file)
+        module = resolver.resolve( ToolRequirement( name="blast+", version="2.23", type="package" ) )
+        assert isinstance(module, NullDependency)
+
+        module = resolver.resolve( ToolRequirement( name="blast", version="2.22", type="package" ) )
+        assert module.module_name == "blast"
+        assert module.module_version == "2.22.0-mpi", module.module_version
+
+
+def _setup_module_command(temp_directory, contents):
+    module_script = os.path.join(temp_directory, "modulecmd")
+    __write_script(module_script, '''#!/bin/sh
+cat %s/example_output 1>&2;
+''' % temp_directory)
+    with open(os.path.join(temp_directory, "example_output"), "w") as f:
+        # Subset of module avail from MSI cluster.
+        f.write(contents)
+    return module_script
+
+
 def test_module_dependency():
     with __test_base_path() as temp_directory:
         # Create mock modulecmd script that just exports a variable
@@ -386,3 +448,9 @@ def __dependency_manager(xml_content):
         f.flush()
         dm = DependencyManager( default_base_path=base_path, conf_file=f.name )
         yield dm
+
+
+class _SimpleDependencyManager(object):
+
+    def get_resolver_option(self, resolver, key, explicit_resolver_options={}):
+        return explicit_resolver_options.get(key)