Merge branch 'miltilingual-suuport' into sonarcloud-patch-1

Author: hcvdwerf

.github/workflows/test.yml

@@ -48,20 +48,36 @@ jobs:
           pip install -r requirements.txt
           pip install -r dev-requirements.txt
           pip install -e .
-      - name: Setup CKAN extensions (harvest, scheming, dcat)
+      - name: Setup CKAN extensions (harvest, scheming, dcat, dataset-series, fluent)
         run: |
           # Harvest v1.6.1 from GitHub
           git clone https://github.com/ckan/ckanext-harvest
           cd ckanext-harvest
           git checkout tags/v1.6.1
           pip install -e .
           pip install -r requirements.txt
-      
-          # Scheming (Civity fork)
-          pip install -e 'git+https://github.com/CivityNL/[email protected]#egg=ckanext-scheming[requirements]'
-      
-          # DCAT v2.4.0 from PyPI
-          pip install ckanext-dcat==2.4.0
+          cd ..
+
+          # Scheming release 3.1.0
+          pip install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-scheming[requirements]'
+
+          # DCAT extension maintained for the GDI user portal
+          pip install -e 'git+https://github.com/GenomicDataInfrastructure/[email protected]#egg=ckanext-dcat'
+          if [ -f "${PIP_SRC}/ckanext-dcat/requirements.txt" ]; then
+            pip install -r "${PIP_SRC}/ckanext-dcat/requirements.txt"
+          fi
+
+          # Dataset series extension
+          pip install -e 'git+https://github.com/GenomicDataInfrastructure/[email protected]#egg=ckanext-dataset-series'
+          if [ -f "${PIP_SRC}/ckanext-dataset-series/requirements.txt" ]; then
+            pip install -r "${PIP_SRC}/ckanext-dataset-series/requirements.txt"
+          fi
+
+          # Fluent extension for multilingual support
+          pip install -e 'git+https://github.com/ckan/ckanext-fluent.git#egg=ckanext-fluent'
+          if [ -f "${PIP_SRC}/ckanext-fluent/requirements.txt" ]; then
+            pip install -r "${PIP_SRC}/ckanext-fluent/requirements.txt"
+          fi
       - name: Setup extension
         run: |
           sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
@@ -74,8 +90,10 @@ jobs:
           coverage xml -o coverage.xml
       - name: Install unzip
         run: apt-get update && apt-get install -y unzip
-      - name: SonarCloud Scan
-        uses: SonarSource/[email protected]
+      - name: Sonar scan
+        uses: SonarSource/sonarqube-scan-action@v6
+
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+          SONAR_HOST_URL: https://sonarcloud.io

.gitignore

@@ -78,4 +78,10 @@ __pycache__
 
 *.egg-info
 
-venv
+venv
+.idea/.gitignore
+.idea/ckanext-gdi-userportal.iml
+.idea/misc.xml
+.idea/modules.xml
+.idea/vcs.xml
+.idea/inspectionProfiles/profiles_settings.xml

ckanext/gdi_userportal/helpers.py

@@ -0,0 +1,117 @@
+# SPDX-FileCopyrightText: 2024 PNED G.I.E.
+# SPDX-FileContributor: Stichting Health-RI
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+import ckan.plugins.toolkit as tk
+
+
+def _ensure_data_dict(data: dict[str, Any] | None, package_id: str | None) -> dict[str, Any]:
+    """Return a dataset dict for the current helper call.
+
+    When a dataset dict is not provided but an id is, fetch it via the
+    ``package_show`` action. Failures are ignored so the helper still returns
+    a sensible default instead of aborting rendering.
+    """
+    if data is not None:
+        return data
+
+    if package_id:
+        try:
+            return tk.get_action("package_show")(
+                {"ignore_auth": True}, {"id": package_id}
+            )
+        except (tk.ObjectNotFound, tk.NotAuthorized):
+            pass
+
+    return {}
+
+
+def _value_from_extras(data_dict: dict[str, Any], field_name: str) -> Any:
+    extras = data_dict.get("extras")
+    if isinstance(extras, dict):
+        return extras.get(field_name)
+    if isinstance(extras, Iterable):
+        for extra in extras:
+            if not isinstance(extra, dict):
+                continue
+            if extra.get("key") == field_name:
+                return extra.get("value")
+    return None
+
+
+def _extract_field_value(field: dict[str, Any], data_dict: dict[str, Any]) -> Any:
+    field_name = field.get("field_name")
+    if not field_name or not data_dict:
+        return None
+
+    if field_name in data_dict:
+        return data_dict[field_name]
+
+    return _value_from_extras(data_dict, field_name)
+
+
+def _is_missing_value(value: Any) -> bool:
+    if value is None:
+        return True
+
+    if isinstance(value, str):
+        return value.strip() == ""
+
+    if isinstance(value, dict):
+        return not value or all(_is_missing_value(v) for v in value.values())
+
+    if isinstance(value, (list, tuple, set)):
+        return not value or all(_is_missing_value(v) for v in value)
+
+    return False
+
+
+def scheming_missing_required_fields(
+    pages: list[dict[str, Any]],
+    data: dict[str, Any] | None = None,
+    package_id: str | None = None,
+) -> list[list[str]]:
+    """Return a list of missing required fields grouped per form page.
+
+    This helper acts as the base implementation expected by
+    ``ckanext-fluent``.  It mirrors the behaviour from the forked
+    ckanext-scheming version previously used in this project and makes sure
+    chained helpers can extend the result again.
+    """
+    data_dict = _ensure_data_dict(data, package_id)
+
+    missing_per_page: list[list[str]] = []
+
+    for page in pages or []:
+        page_missing: list[str] = []
+        for field in page.get("fields", []):
+            # Ignore non-required fields early.
+            if not tk.h.scheming_field_required(field):
+                continue
+
+            value = _extract_field_value(field, data_dict)
+
+            # Repeating subfields can contain a list of child values; treat the
+            # field as present when at least one entry contains data.
+            if field.get("repeating_subfields") and isinstance(value, list):
+                if any(not _is_missing_value(item) for item in value):
+                    continue
+            elif not _is_missing_value(value):
+                continue
+
+            if field_name := field.get("field_name"):
+                page_missing.append(field_name)
+
+        missing_per_page.append(page_missing)
+
+    return missing_per_page
+
+
+def get_helpers() -> dict[str, Any]:
+    return {"scheming_missing_required_fields": scheming_missing_required_fields}

ckanext/gdi_userportal/logic/action/get.py

@@ -28,7 +28,8 @@ def enhanced_package_search(context, data_dict) -> Dict:
     lang = toolkit.request.headers.get("Accept-Language")
     translations = get_translations(values_to_translate, lang=lang)
     result["results"] = [
-        replace_package(package, translations) for package in result["results"]
+        replace_package(package, translations, lang=lang)
+        for package in result["results"]
     ]
     if "search_facets" in result.keys():
         result["search_facets"] = replace_search_facets(
@@ -43,4 +44,4 @@ def enhanced_package_show(context, data_dict) -> Dict:
     values_to_translate = collect_values_to_translate(result)
     lang = toolkit.request.headers.get("Accept-Language")
     translations = get_translations(values_to_translate, lang=lang)
-    return replace_package(result, translations)
+    return replace_package(result, translations, lang=lang)

ckanext/gdi_userportal/logic/action/post.py

@@ -25,15 +25,17 @@ def enhanced_package_search(context, data_dict=None) -> Dict:
     try:
         result = toolkit.get_action("package_search")(context, data_dict)
         values_to_translate = collect_values_to_translate(result)
-        translations = get_translations(values_to_translate)
+        lang = toolkit.request.headers.get("Accept-Language")
+        translations = get_translations(values_to_translate, lang=lang)
 
         result["results"] = [
-            replace_package(package, translations) for package in result["results"]
+            replace_package(package, translations, lang=lang)
+            for package in result["results"]
         ]
 
         if "search_facets" in result:
             result["search_facets"] = replace_search_facets(
-                result["search_facets"], translations
+                result["search_facets"], translations, lang=lang
             )
 
         return result

ckanext/gdi_userportal/logic/action/translation_utils.py

@@ -6,9 +6,9 @@
 
 from dataclasses import dataclass
 import logging
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
-from ckan.common import config, request
+from ckan.common import config
 
 # -*- coding: utf-8 -*-
 from ckan.plugins import toolkit
@@ -23,6 +23,13 @@
     "dcat_type",
 ]
 RESOURCE_REPLACE_FIELDS = ["format", "language"]
+TRANSLATED_SUFFIX = "_translated"
+LANGUAGE_VALUE_FIELDS = {
+    "population_coverage",
+    "publisher_note",
+    "provenance",
+    "rights",
+}
 DEFAULT_FALLBACK_LANG = "en"
 SUPPORTED_LANGUAGES = {DEFAULT_FALLBACK_LANG, "nl"}
 
@@ -121,13 +128,18 @@ def collect_values_to_translate(data: Any) -> List:
     return list(set(values_to_translate))
 
 
-def replace_package(data, translation_dict):
+def replace_package(data, translation_dict, lang: Optional[str] = None):
+    preferred_lang = _get_language(lang)
+
+    _apply_translated_properties(data, preferred_lang)
+
     data = _translate_fields(data, PACKAGE_REPLACE_FIELDS, translation_dict)
     resources = data.get("resources", [])
     data["resources"] = [
         _translate_fields(item, RESOURCE_REPLACE_FIELDS, translation_dict)
         for item in resources
     ]
+
     return data
 
 
@@ -156,11 +168,77 @@ def _change_facet(facet, translation_dict):
 
 
 def replace_search_facets(data, translation_dict, lang):
+    preferred_lang = _get_language(lang)
     new_facets = {}
     for key, facet in data.items():
         title = facet["title"]
-        new_facets[key] = {"title": get_translations([title], lang=lang).get(title, title)}
+        new_facets[key] = {
+            "title": get_translations([title], lang=preferred_lang).get(title, title)
+        }
         new_facets[key]["items"] = [
             _change_facet(item, translation_dict) for item in facet["items"]
         ]
     return new_facets
+
+
+def _apply_translated_properties(data: Any, preferred_lang: str, fallback_lang: str = DEFAULT_FALLBACK_LANG):
+    if not isinstance(data, (dict, list)):
+        return data
+
+    if isinstance(data, list):
+        return [
+            _apply_translated_properties(item, preferred_lang, fallback_lang)
+            if isinstance(item, (dict, list))
+            else item
+            for item in data
+        ]
+
+    for key, value in list(data.items()):
+        if isinstance(value, dict):
+            _apply_translated_properties(value, preferred_lang, fallback_lang)
+        elif isinstance(value, list):
+            data[key] = [
+                _apply_translated_properties(item, preferred_lang, fallback_lang)
+                if isinstance(item, (dict, list))
+                else item
+                for item in value
+            ]
+
+    for key, value in list(data.items()):
+        if key.endswith(TRANSLATED_SUFFIX) and isinstance(value, dict):
+            base_key = key[:-len(TRANSLATED_SUFFIX)]
+            merged_values = value.copy()
+            existing_value = data.get(base_key)
+            if isinstance(existing_value, dict):
+                merged_values.update(existing_value)
+            data[base_key] = _select_translated_value(
+                merged_values, preferred_lang, fallback_lang
+            )
+        elif key in LANGUAGE_VALUE_FIELDS and isinstance(value, dict):
+            data[key] = _select_translated_value(value, preferred_lang, fallback_lang)
+
+    return data
+
+
+def _select_translated_value(values: Dict[str, Any], preferred_lang: str, fallback_lang: str) -> Any:
+    if not isinstance(values, dict):
+        return values
+
+    for lang in (preferred_lang, fallback_lang):
+        translated = values.get(lang)
+        if _has_content(translated):
+            return translated
+
+    for translated in values.values():
+        if _has_content(translated):
+            return translated
+
+    return next(iter(values.values()), "")
+
+
+def _has_content(value: Any) -> bool:
+    if value is None:
+        return False
+    if isinstance(value, str):
+        return bool(value.strip())
+    return bool(value) if isinstance(value, (list, dict)) else True

ckanext/gdi_userportal/plugin.py

@@ -6,6 +6,7 @@
 import json
 import ckan.plugins as plugins
 import ckan.plugins.toolkit as toolkit
+from ckanext.gdi_userportal.helpers import get_helpers as get_portal_helpers
 from ckanext.gdi_userportal.logic.action.get import (
     enhanced_package_search,
     enhanced_package_show,
@@ -50,6 +51,7 @@ class GdiUserPortalPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IActions)
     plugins.implements(plugins.IPackageController)
     plugins.implements(plugins.IValidators)
+    plugins.implements(plugins.ITemplateHelpers, inherit=True)
     plugins.implements(plugins.IMiddleware, inherit=True)
     plugins.implements(plugins.IConfigurable, inherit=True)
 
@@ -74,8 +76,14 @@ def update_config(self, config_):
     def update_config_schema(self, schema):
         ignore_missing = toolkit.get_validator("ignore_missing")
         unicode_safe = toolkit.get_validator("unicode_safe")
+        int_validator = toolkit.get_validator("int_validator")
         schema.update(
-            {"ckanext.gdi_userportal.intro_text": [ignore_missing, unicode_safe]}
+            {
+                "ckanext.gdi_userportal.intro_text": [ignore_missing, unicode_safe],
+                "ckan.harvest.timeout": [ignore_missing, int_validator],
+                "ckan.harvest.mq.type": [ignore_missing, unicode_safe],
+                "ckan.harvest.mq.hostname": [ignore_missing, unicode_safe],
+            }
         )
         return schema
 
@@ -105,6 +113,9 @@ def get_actions(self):
             "enhanced_package_show": enhanced_package_show,
         }
 
+    def get_helpers(self):
+        return get_portal_helpers()
+
     def read(self, entity):
         pass