Merge branch 'staging'

dweinholz · dweinholz · commit 60cb29a43062 · 2025-09-17T07:32:06.000+02:00
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v5
 
       - name: Set up Python 3.11
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: 3.13
 
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.13.6
+FROM python:3.13.7
 RUN echo "deb https://deb.debian.org/debian/ stable main" > /etc/apt/sources.list
 RUN apt-get update -y \
     && apt-get install -y build-essential python3-openstackclient vim\
diff --git a/Dockerfile-tests b/Dockerfile-tests
@@ -1,4 +1,4 @@
-FROM python:3.13.6
+FROM python:3.13.7
 WORKDIR /code
 
 # Copy all requirements files at once
diff --git a/docker-compose.bibigrid.host.yml b/docker-compose.bibigrid.host.yml
@@ -20,7 +20,7 @@ services:
   simplevm_client_redis:
     container_name: simplevm_client_redis
     network_mode: host  # Use the host network namespace
-    image: redis:8.2.0
+    image: redis:8.2.1
     expose:
       - "6379"
 
@@ -42,7 +42,7 @@ services:
 
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:9.1.2
+    image: docker.elastic.co/beats/filebeat:9.1.3
     network_mode: host  # Use the host network namespace
 
     env_file:
diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml
@@ -20,7 +20,7 @@ services:
 
   simplevm_client_redis:
     container_name: simplevm_client_redis
-    image: redis:8.2.0
+    image: redis:8.2.1
     expose:
       - "6379"
     networks:
@@ -43,7 +43,7 @@ services:
 
   # filebeat
   simplevm_filebeat:
-    image: docker.elastic.co/beats/filebeat:9.1.2
+    image: docker.elastic.co/beats/filebeat:9.1.3
     env_file:
       - .env
     volumes:
diff --git a/docker-compose.dev.bibigrid.yml b/docker-compose.dev.bibigrid.yml
@@ -20,7 +20,7 @@ services:
 
   client_redis:
     container_name: simplevm_client_redis
-    image: redis:8.2.0
+    image: redis:8.2.1
     expose:
       - "6379"
     networks:
diff --git a/docker-compose.dev.giessen.yml b/docker-compose.dev.giessen.yml
@@ -20,7 +20,7 @@ services:
 
   simplevm_client_giessen_redis:
     container_name: simplevm_client_giessen_redis
-    image: redis:8.2.0
+    image: redis:8.2.1
     expose:
       - "6379"
     networks:
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -20,7 +20,7 @@ services:
 
   client_redis:
     container_name: simplevm_client_redis
-    image: redis:8.2.0
+    image: redis:8.2.1
     expose:
       - "6379"
     networks:
diff --git a/docker-compose.host.yml b/docker-compose.host.yml
@@ -21,15 +21,15 @@ services:
   simplevm_client_redis:
     container_name: simplevm_client_redis
     network_mode: host  # Use the host network namespace
-    image: redis:8.2.0
+    image: redis:8.2.1
     expose:
       - "6379"
     networks:
       - portal
 
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:9.1.2
+    image: docker.elastic.co/beats/filebeat:9.1.3
     env_file:
       - .env
     volumes:
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -20,7 +20,7 @@ services:
 
   simplevm_client_redis:
     container_name: simplevm_client_redis
-    image: redis:8.2.0
+    image: redis:8.2.1
     expose:
       - "6379"
     networks:
@@ -29,7 +29,7 @@ services:
   # filebeat
   simplevm_client_filebeat:
     container_name: simplevm_client_filebeat
-    image: docker.elastic.co/beats/filebeat:9.1.2
+    image: docker.elastic.co/beats/filebeat:9.1.3
     env_file:
       - .env
     volumes:
diff --git a/portal_client.thrift b/portal_client.thrift
@@ -8,6 +8,7 @@ const string VERSION= '1.0.0'
 struct MetadataUser {
   1: required string user_id,
   2: optional list<string> public_keys,
+  3: optional string unix_name
 }
 
 struct MetadataUserData {
diff --git a/requirements-test.txt b/requirements-test.txt
@@ -1,4 +1,4 @@
-pytest==8.4.1
-coverage==7.10.3
-pytest-cov==6.2.1
+pytest==8.4.2
+coverage==7.10.6
+pytest-cov==7.0.0
 
diff --git a/requirements.txt b/requirements.txt
@@ -1,22 +1,22 @@
 setuptools==80.9.0
 thrift==0.22.0
-openstacksdk==4.6.0
+openstacksdk==4.7.1
 deprecated==1.2.18
 Click==8.2.2
 ansible==11.9.0
 flake8==7.3.0
 paramiko==4.0.0
-ruamel.yaml==0.18.14
+ruamel.yaml==0.18.15
 pyvim==3.0.3
 redis==6.4.0
-requests==2.32.4
+requests==2.32.5
 pyyaml==6.0.2
 pre-commit==4.3.0
-types-PyYAML==6.0.12.20250809
+types-PyYAML==6.0.12.20250915
 sympy==1.14.0
 colorama==0.4.6
-gevent==25.5.1
+gevent==25.8.2
 types-redis==4.6.0.20241004
-oslo.utils==9.0.0
+oslo.utils==9.1.0
 passlib==1.7.4
 apscheduler==3.11.0
diff --git a/simple_vm_client/VirtualMachineHandler.py b/simple_vm_client/VirtualMachineHandler.py
@@ -552,6 +552,8 @@ def start_cluster(
         metadata: ClusterInstanceMetadata,
         shared_volume: ClusterVolume = None,
     ) -> ClusterMessage:
+        
+
         return self.bibigrid_connector.start_cluster(
             public_keys=public_keys,
             master_instance=master_instance,
diff --git a/simple_vm_client/metadata_connector/metadata_connector.py b/simple_vm_client/metadata_connector/metadata_connector.py
@@ -15,116 +15,133 @@
 class MetadataConnector:
     def __init__(self, config_file: str):
         logger.info("Initializing Metadata Connector")
+        self.session = requests.Session()
+        self.ACTIVATED = False
+        self.METADATA_WRITE_TOKEN = None
+        self.METADATA_BASE_URL = None
         self.load_config_yml(config_file)
         self.is_metadata_server_available()
 
     def load_config_yml(self, config_file: str) -> None:
-        logger.info(f"Load config file openstack config - {config_file}")
+        logger.info(f"Loading config file: {config_file}")
         with open(config_file, "r") as ymlfile:
             cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader)
-            logger.info("Config file loaded")
+
             if "metadata_server" not in cfg:
-                logger.info("Metadata Server configuration not found. Skipping.")
-                self.ACTIVATED = False
+                logger.warning("Metadata server configuration not found. Skipping.")
                 return
-            if not cfg["metadata_server"].get("activated", False):
+
+            metadata_cfg = cfg["metadata_server"]
+            if not metadata_cfg.get("activated", False):
                 logger.info(
-                    "Metadata Server Config available but deactivated. Skipping.."
+                    "Metadata server config available but deactivated. Skipping."
                 )
-                self.ACTIVATED = False
                 return
+
+            self.METADATA_USE_HTTPS = metadata_cfg.get("use_https", False)
+            self.METADATA_SERVER_HOST = metadata_cfg.get("host")
+            self.METADATA_SERVER_PORT = metadata_cfg.get("port")
+
+            if not self.METADATA_SERVER_HOST or not self.METADATA_SERVER_PORT:
+                logger.error("Host or port missing in metadata server configuration!")
+                return
+
+            scheme = "https" if self.METADATA_USE_HTTPS else "http"
+            self.METADATA_BASE_URL = (
+                f"{scheme}://{self.METADATA_SERVER_HOST}:{self.METADATA_SERVER_PORT}/"
+            )
+
+            logger.info("Metadata configuration loaded")
             self.ACTIVATED = True
-            self.METADATA_USE_HTTPS = cfg["metadata_server"].get("use_https", False)
-            self.METADATA_SERVER_HOST = cfg["metadata_server"]["host"]
-            self.METADATA_SERVER_PORT = cfg["metadata_server"]["port"]
-            if self.METADATA_USE_HTTPS:
-                self.METADATA_BASE_URL = (
-                    f"https://{self.METADATA_SERVER_HOST}:{self.METADATA_SERVER_PORT}/"
-                )
-            else:
-                self.METADATA_BASE_URL = (
-                    f"http://{self.METADATA_SERVER_HOST}:{self.METADATA_SERVER_PORT}/"
-                )
-        logger.info("Metadata Config Loaded")
+
         self.load_env_config()
 
     def load_env_config(self):
-        required_keys = [
-            "METADATA_WRITE_TOKEN",
-        ]
-        missing_keys = [key for key in required_keys if key not in os.environ]
-        if missing_keys:
-            missing_keys_str = ", ".join(missing_keys)
-            logger.error(
-                f"MetadataServer missing keys {missing_keys_str} not provided in env!"
-            )
-        self.METADATA_WRITE_TOKEN = os.environ.get("METADATA_WRITE_TOKEN")
-        logger.info("Metadata Environment loaded")
+        token = os.environ.get("METADATA_WRITE_TOKEN")
+        if not token:
+            logger.error("Environment variable METADATA_WRITE_TOKEN is missing!")
+            self.ACTIVATED = False
+            return
+
+        self.METADATA_WRITE_TOKEN = token
+        self.session.headers.update({"X-Auth-Token": self.METADATA_WRITE_TOKEN})
+        logger.info("Metadata environment configuration loaded")
 
-    def remove_metadata(self, ip: str):
+    def remove_metadata(self, ip: str) -> bool:
         if not self.ACTIVATED:
-            logger.info("Metadata Server not activated. Skipping.")
-            return
+            logger.info("Metadata server not activated. Skipping.")
+            return False
 
-        logger.info(f"Removing Metadata for: {ip}")
+        logger.info(f"Removing metadata for {ip}")
         remove_metadata_url = urljoin(self.METADATA_BASE_URL, f"metadata/{ip}")
 
         try:
-            response = requests.delete(
-                remove_metadata_url,
-                timeout=(30, 30),
-                headers={
-                    "X-Auth-Token": self.METADATA_WRITE_TOKEN,
-                },
-            )
+            response = self.session.delete(remove_metadata_url, timeout=(30, 30))
             response.raise_for_status()
             logger.info(f"Metadata removed successfully for {ip}")
+            return True
+        except requests.exceptions.HTTPError as e:
+            logger.error(f"HTTP error while removing metadata for {ip}: {e}")
+        except requests.exceptions.Timeout:
+            logger.error(f"Timeout while removing metadata for {ip}")
         except requests.exceptions.RequestException as e:
             logger.error(f"Failed to remove metadata for {ip}: {e}")
+        return False
 
     def _serialize_metadata(self, metadata: VirtualMachineServerMetadata) -> str:
-        metadata_dict = thrift_to_dict(metadata)
-        return json.dumps(metadata_dict)
+        try:
+            metadata_dict = thrift_to_dict(metadata)
+            return json.dumps(metadata_dict)
+        except Exception as e:
+            logger.error(f"Failed to serialize metadata: {e}")
+            raise
 
-    def set_metadata(self, ip: str, metadata: VirtualMachineServerMetadata):
+    def set_metadata(self, ip: str, metadata: VirtualMachineServerMetadata) -> bool:
         if not ip:
-            return
+            logger.error("IP address not provided, cannot set metadata.")
+            return False
         if not self.ACTIVATED:
-            logger.info("Metadata Server not activated. Skipping.")
-            return
+            logger.info("Metadata server not activated. Skipping.")
+            return False
 
-        logger.info(f"Setting Metadata for {ip}")
+        logger.info(f"Setting metadata for {ip}")
         set_metadata_url = urljoin(self.METADATA_BASE_URL, f"metadata/{ip}")
+
         try:
-            serialized_data = self._serialize_metadata(metadata=metadata)
-            response = requests.post(
+            serialized_data = self._serialize_metadata(metadata)
+            response = self.session.post(
                 set_metadata_url,
                 data=serialized_data,
                 timeout=(30, 30),
-                headers={
-                    "X-Auth-Token": self.METADATA_WRITE_TOKEN,
-                    "Content-Type": "application/json",
-                },
+                headers={"Content-Type": "application/json"},
             )
             response.raise_for_status()
             logger.info(f"Metadata set successfully for {ip}")
+            return True
+        except requests.exceptions.HTTPError as e:
+            logger.error(f"HTTP error while setting metadata for {ip}: {e}")
+        except requests.exceptions.Timeout:
+            logger.error(f"Timeout while setting metadata for {ip}")
         except requests.exceptions.RequestException as e:
             logger.error(f"Failed to set metadata for {ip}: {e}")
+        return False
 
-    def is_metadata_server_available(self):
-        logger.info("Metadata Server checking health...")
+    def is_metadata_server_available(self) -> bool:
+        logger.info("Checking metadata server health...")
 
         if not self.ACTIVATED:
-            logger.info("Metadata Server not activated. Skipping.")
+            logger.info("Metadata server not activated. Skipping health check.")
             return False
 
         health_url = urljoin(self.METADATA_BASE_URL, "health")
 
         try:
-            response = requests.get(health_url, timeout=(30, 30))
+            response = self.session.get(health_url, timeout=(30, 30))
             response.raise_for_status()
-            logger.info(f"Metadata Health Check --- {response.json()}")
+            logger.info(f"Metadata health check successful: {response.json()}")
             return True
+        except requests.exceptions.Timeout:
+            logger.error("Timeout during metadata server health check")
         except requests.exceptions.RequestException as e:
             logger.error(f"Health check failed: {e}")
-            return False
+        return False
diff --git a/simple_vm_client/metadata_connector/test_metadata_connector.py b/simple_vm_client/metadata_connector/test_metadata_connector.py
diff --git a/simple_vm_client/ttypes.py b/simple_vm_client/ttypes.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM python:3.13.6`
	`1`	`+FROM python:3.13.7`
`2`	`2`	`RUN echo "deb https://deb.debian.org/debian/ stable main" > /etc/apt/sources.list`
`3`	`3`	`RUN apt-get update -y \`
`4`	`4`	`&& apt-get install -y build-essential python3-openstackclient vim\`