Merge branch 'staging'

dweinholz · dweinholz · commit 1bde91532dcc · 2024-09-02T13:22:09.000+02:00
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -11,7 +11,7 @@ repos:
 
   - repo: https://github.com/psf/black-pre-commit-mirror
 
-    rev: 24.4.2
+    rev: 24.8.0
     hooks:
       - id: black
         language_version: python3.11
@@ -35,7 +35,7 @@ repos:
           - --remove-all-unused-imports
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 7.1.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         args: [ "--config=setup.cfg" ]
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.12.4
+FROM python:3.12.5
 RUN echo "deb https://deb.debian.org/debian/ stable main" > /etc/apt/sources.list
 RUN apt-get update -y \
     && apt-get install -y build-essential python3-openstackclient vim\
diff --git a/docker-compose.bibigrid.host.yml b/docker-compose.bibigrid.host.yml
@@ -21,7 +21,7 @@ services:
   simplevm_client_redis:
     container_name: simplevm_client_redis
     network_mode: host  # Use the host network namespace
-    image: redis:7.2.5
+    image: redis:7.4.0
     expose:
       - "6379"
 
@@ -43,7 +43,7 @@ services:
 
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:8.13.4
+    image: docker.elastic.co/beats/filebeat:8.15.0
     network_mode: host  # Use the host network namespace
 
     env_file:
diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml
@@ -21,7 +21,7 @@ services:
 
   simplevm_client_redis:
     container_name: simplevm_client_redis
-    image: redis:7.2.5
+    image: redis:7.4.0
     expose:
       - "6379"
     networks:
@@ -44,7 +44,7 @@ services:
 
   # filebeat
   simplevm_filebeat:
-    image: docker.elastic.co/beats/filebeat:8.13.4
+    image: docker.elastic.co/beats/filebeat:8.15.0
     env_file:
       - .env
     volumes:
diff --git a/docker-compose.dev.bibigrid.yml b/docker-compose.dev.bibigrid.yml
@@ -21,7 +21,7 @@ services:
 
   client_redis:
     container_name: simplevm_client_redis
-    image: redis:7.2.5
+    image: redis:7.4.0
     expose:
       - "6379"
     networks:
diff --git a/docker-compose.dev.giessen.yml b/docker-compose.dev.giessen.yml
@@ -21,7 +21,7 @@ services:
 
   simplevm_client_giessen_redis:
     container_name: simplevm_client_giessen_redis
-    image: redis:7.2.5
+    image: redis:7.4.0
     expose:
       - "6379"
     networks:
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -21,7 +21,7 @@ services:
 
   client_redis:
     container_name: simplevm_client_redis
-    image: redis:7.2.5
+    image: redis:7.4.0
     expose:
       - "6379"
     networks:
diff --git a/docker-compose.host.yml b/docker-compose.host.yml
@@ -22,15 +22,15 @@ services:
   simplevm_client_redis:
     container_name: simplevm_client_redis
     network_mode: host  # Use the host network namespace
-    image: redis:7.2.5
+    image: redis:7.4.0
     expose:
       - "6379"
     networks:
       - portal
 
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:8.13.4
+    image: docker.elastic.co/beats/filebeat:8.15.0
     env_file:
       - .env
     volumes:
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,7 +21,7 @@ services:
 
   simplevm_client_redis:
     container_name: simplevm_client_redis
-    image: redis:7.2.5
+    image: redis:7.4.0
     expose:
       - "6379"
     networks:
@@ -30,7 +30,7 @@ services:
   # filebeat
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:8.13.4
+    image: docker.elastic.co/beats/filebeat:8.15.0
     env_file:
       - .env
     volumes:
diff --git a/requirements.txt b/requirements.txt
@@ -1,20 +1,21 @@
-setuptools==70.3.0
+setuptools==74.0.0
 thrift==0.20.0
 python-keystoneclient==5.4.0
 openstacksdk==3.1.0
 deprecated==1.2.14
 Click==8.1.7
-ansible==9.5.1
-flake8==7.1.0
-paramiko==3.4.0
+ansible==9.9.0
+flake8==7.1.1
+paramiko==3.4.1
 ruamel.yaml==0.18.6
 pyvim==3.0.3
-redis==5.0.7
+redis==5.0.8
 requests==2.32.3
-pyyaml==6.0.1
-pre-commit==3.7.1
-types-PyYAML==6.0.12.20240311
-sympy==1.12.1
+pyyaml==6.0.2
+pre-commit==3.8.0
+types-PyYAML==6.0.12.20240808
+sympy==1.13.2
 colorama==0.4.6
-types-redis==4.6.0.20240425
+types-redis==4.6.0.20240819
 gevent
+pytest
diff --git a/simple_vm_client/bibigrid_connector/test_bibigrid_connector.py b/simple_vm_client/bibigrid_connector/test_bibigrid_connector.py
@@ -6,7 +6,11 @@
 import requests
 
 from simple_vm_client.bibigrid_connector.bibigrid_connector import BibigridConnector
-from simple_vm_client.ttypes import ClusterInfo, ClusterInstance
+from simple_vm_client.ttypes import (
+    ClusterInfo,
+    ClusterInstance,
+    ClusterNotFoundException,
+)
 
 HOST = "example.com"
 PORT = 8080
@@ -200,7 +204,7 @@ def test_start_cluster(self, mock_logger_info, mock_post):
 
         # Call the method to test
         result = self.connector.start_cluster(
-            public_key=public_key,
+            public_keys=[public_key],
             master_instance=DEFAULT_MASTER_INSTANCE,
             worker_instances=DEFAULT_WORKER_INSTANCES,
             user=user,
@@ -271,13 +275,16 @@ def test_get_clusters_info(self, mock_logger_info, mock_get):
         self.assertEqual(result, [{"cluster-id": "fake_cluster_id"}])
         mock_logger_info.assert_called_once_with("Get clusters info")
 
-    @patch.object(BibigridConnector, "get_clusters_info")
+    @patch(
+        "simple_vm_client.bibigrid_connector.bibigrid_connector.BibigridConnector.get_clusters_info"
+    )
     @patch("simple_vm_client.bibigrid_connector.bibigrid_connector.logger.info")
     def test_get_cluster_info_none(self, mock_logger_info, mock_get_clusters_info):
         mock_get_clusters_info.return_value = []
-        result = self.connector.get_cluster_info("fake_cluster_id")
+        with self.assertRaises(ClusterNotFoundException):
+            self.connector.get_cluster_info("fake_cluster_id")
+
         mock_logger_info.assert_any_call("Get Cluster info from fake_cluster_id")
-        self.assertIsNone(result)
 
     @patch.object(BibigridConnector, "get_clusters_info")
     @patch("simple_vm_client.bibigrid_connector.bibigrid_connector.logger.info")
diff --git a/simple_vm_client/openstack_connector/openstack_connector.py b/simple_vm_client/openstack_connector/openstack_connector.py
@@ -6,6 +6,7 @@
 import sys
 import urllib
 import urllib.parse
+import threading
 from contextlib import closing
 from typing import Union
 from uuid import uuid4
@@ -54,6 +55,9 @@
 
 ALL_TEMPLATES = [BIOCONDA]
 
+lock_dict = {}
+lock_access = threading.Lock()
+
 
 class OpenStackConnector:
     def __init__(self, config_file: str):
@@ -520,18 +524,27 @@ def get_servers_by_bibigrid_id(self, bibigrid_id: str) -> list[Server]:
         servers: list[Server] = self.openstack_connection.list_servers(filters=filters)
         return servers
 
-    def get_active_image_by_os_version(self, os_version: str, os_distro: str) -> Image:
+    def get_active_image_by_os_version(
+        self, os_version: str, os_distro: Union[str, None]
+    ) -> Image:
         logger.info(f"Get active Image by os-version: {os_version}")
         images = self.openstack_connection.list_images()
         for image in images:
+
             image_os_version = image.get("os_version", None)
             image_os_distro = image.get("os_distro", None)
-            base_image_ref = image.get("properties", {}).get("base_image_ref", None)
+            image_properties = image.get("properties", None)
+            if image_properties:
+
+                base_image_ref = image_properties.get("base_image_ref", None)
+            else:
+                base_image_ref = None
             if (
                 os_version == image_os_version
                 and image.status == "active"
                 and base_image_ref is None
             ):
+
                 if os_distro and os_distro == image_os_distro:
                     return image
                 elif os_distro is None:
@@ -1025,33 +1038,41 @@ def get_or_create_vm_security_group(self, openstack_id):
 
     def get_or_create_project_security_group(self, project_name, project_id):
         security_group_name = f"{project_name}_{project_id}"
-        logger.info(
-            f"Check if Security Group for project - [{project_name}-{project_id}] exists... "
-        )
-        sec = self.openstack_connection.get_security_group(
-            name_or_id=security_group_name
-        )
-        if sec:
+
+        with lock_access:
+            if security_group_name not in lock_dict:
+                lock_dict[security_group_name] = threading.Lock()
+            lock = lock_dict[security_group_name]
+        
+        with lock:
             logger.info(
-                f"Security group [{project_name}-{project_id}]  already exists."
+                f"Check if Security Group for project - [{project_name}-{project_id}] exists... "
             )
-            return sec["id"]
+            sec = self.openstack_connection.get_security_group(
+                name_or_id=security_group_name
+            )
+            if sec:
+                logger.info(
+                    f"Security group [{project_name}-{project_id}]  already exists."
+                )
+                return sec["id"]
 
-        logger.info(
-            f"No security Group for [{project_name}-{project_id}]  exists. Creating.. "
-        )
-        new_security_group = self.openstack_connection.create_security_group(
-            name=security_group_name, description=f"{project_name} Security Group"
-        )
-        self.openstack_connection.network.create_security_group_rule(
-            direction="ingress",
-            protocol="tcp",
-            port_range_max=22,
-            port_range_min=22,
-            security_group_id=new_security_group["id"],
-            remote_group_id=new_security_group["id"],
-        )
-        return new_security_group["id"]
+            logger.info(
+                f"No security Group for [{project_name}-{project_id}]  exists. Creating.. "
+            )
+            new_security_group = self.openstack_connection.create_security_group(
+                name=security_group_name, description=f"{project_name} Security Group"
+            )
+            self.openstack_connection.network.create_security_group_rule(
+                direction="ingress",
+                protocol="tcp",
+                port_range_max=22,
+                port_range_min=22,
+                security_group_id=new_security_group["id"],
+                remote_group_id=new_security_group["id"],
+            )
+            return new_security_group["id"]
+        
 
     def get_limits(self) -> dict[str, str]:
         logger.info("Get Limits")
diff --git a/simple_vm_client/test_openstack_connector.py b/simple_vm_client/test_openstack_connector.py
@@ -4,7 +4,7 @@
 import tempfile
 import unittest
 from unittest import mock
-from unittest.mock import MagicMock, call, patch
+from unittest.mock import ANY, MagicMock, call, patch
 
 from openstack.block_storage.v3 import volume
 from openstack.block_storage.v3.limits import Limit
@@ -86,13 +86,17 @@
     id="image_id_2",
     status="active",
     name="image_2",
+    os_version="22.04",
+    os_distro="ubuntu",
     metadata={"os_version": "22.04", "os_distro": "ubuntu"},
     tags=["portalclient"],
 )
 INACTIVE_IMAGE = image_module.Image(
     id="image_inactive",
     status="building",
     name="image_inactive",
+    os_version="22.04",
+    os_distro="ubuntu",
     metadata={"os_version": "22.04", "os_distro": "ubuntu"},
     tags=["portalclient"],
 )
@@ -102,6 +106,8 @@
         id="image_id_1",
         status="inactive",
         name="image_1",
+        os_version="22.04",
+        os_distro="ubuntu",
         metadata={"os_version": "22.04", "os_distro": "ubuntu"},
         tags=["portalclient"],
     ),
@@ -110,6 +116,8 @@
         id="image_id_3",
         status="active",
         name="image_3",
+        os_version="22.04",
+        os_distro="ubuntu",
         metadata={"os_version": "22.04", "os_distro": "centos"},
         tags=["portalclient"],
     ),
@@ -1837,7 +1845,7 @@ def test_start_server(
             image=fake_image.id,
             flavor=fake_flavor.id,
             network=[fake_network.id],
-            key_name=server_keypair.name,
+            key_name=ANY,
             meta=metadata,
             volumes=["volume1", "volume2"],
             userdata="userdata",
@@ -1858,7 +1866,7 @@ def test_start_server(
         )
 
         self.openstack_connector.openstack_connection.create_keypair.assert_called_once_with(
-            name=server_keypair.name, public_key=public_key
+            name=ANY, public_key=public_key
         )
 
         mock_get_volumes.assert_called_once_with(
@@ -1867,9 +1875,9 @@ def test_start_server(
         )
 
         self.openstack_connector.openstack_connection.get_keypair.assert_called_once_with(
-            name_or_id=server_keypair.name
+            name_or_id=ANY
         )
-        mock_delete_keypair.assert_any_call(key_name=server_keypair.name)
+        mock_delete_keypair.assert_any_call(key_name=ANY)
 
         # Check the result
         self.assertEqual(result, server.id)
diff --git a/simple_vm_client/test_virtualmachinehandler.py b/simple_vm_client/test_virtualmachinehandler.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM python:3.12.4`
	`1`	`+FROM python:3.12.5`
`2`	`2`	`RUN echo "deb https://deb.debian.org/debian/ stable main" > /etc/apt/sources.list`
`3`	`3`	`RUN apt-get update -y \`
`4`	`4`	`&& apt-get install -y build-essential python3-openstackclient vim\`