Merge pull request #80 from GenomicDataInfrastructure/Upgrade-DCAT-AP3

Upgrade dcat ap3

Author: brunopacheco1

.github/workflows/test.yml

@@ -44,8 +44,8 @@ jobs:
         pip install --upgrade pytest-rerunfailures
         pip install -e 'git+https://github.com/CivityNL/[email protected]#egg=ckanext-scheming[requirements]'
         pip install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-harvest[requirements]'
-        pip install -e 'git+https://github.com/ckan/ckanext-dcat.git@v1.5.1#egg=ckanext-dcat[requirements]'
-        pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v1.5.1/requirements.txt
+        pip install -e 'git+https://github.com/ckan/ckanext-dcat.git@v2.0.0#egg=ckanext-dcat[requirements]'
+        pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v2.0.0/requirements.txt
         python3 setup.py develop
     - name: Setup extension
       run: |

README.md

@@ -108,8 +108,8 @@ otherwise run:
 ```commandline
 pip install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-scheming[requirements]'
 pip install -e 'git+https://github.com/ckan/[email protected]#egg=ckanext-harvest[requirements]'
-pip install -e 'git+https://github.com/ckan/ckanext-dcat.git@v1.5.1#egg=ckanext-dcat'
-pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v1.5.1/requirements.txt
+pip install -e 'git+https://github.com/ckan/ckanext-dcat.git@v2.0.0#egg=ckanext-dcat'
+pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/v2.0.0/requirements.txt
 ```
 
 ## Tests

ckanext/fairdatapoint/processors.py

@@ -35,7 +35,7 @@ def catalogs(self) -> Iterable[Dict]:
         for catalog_ref in self._catalogs():
             catalog_dict = {}
             for profile_class in self._profiles:
-                profile = profile_class(self.g, self.compatibility_mode)
+                profile = profile_class(graph= self.g,compatibility_mode = self.compatibility_mode)
                 profile.parse_dataset(catalog_dict, catalog_ref)
 
             yield catalog_dict

ckanext/fairdatapoint/profiles.py

@@ -8,7 +8,7 @@
 import json
 import logging
 
-from ckanext.dcat.profiles import EuropeanDCATAP2Profile
+from ckanext.dcat.profiles import EuropeanDCATAP3Profile
 from ckan.plugins import toolkit
 from ckan import model
 import dateutil.parser as dateparser
@@ -21,46 +21,6 @@
 
 VCARD = Namespace("http://www.w3.org/2006/vcard/ns#")
 
-
-def _convert_extras_to_declared_schema_fields(dataset_dict: Dict) -> Dict:
-    """
-    Compares the extras dictionary with the declared schema.
-    Updates the declared schema fields with the values that match from the extras.
-    Remove the extras that are present on the declared schema.
-    :param dataset_dict:
-    :return: dataset_dict - Updated dataset_dict
-    """
-    # Use the correct dataset type, Defaults to 'dataset'
-    dataset_type = dataset_dict.get('type', 'dataset')
-    # Gets the full Schema definition of the correct dataset type
-    context = {'model': model, 'session': model.Session}
-    data_dict = {'type': dataset_type}
-    full_schema_dict = toolkit.get_action('scheming_dataset_schema_show')(context, data_dict)
-
-    dataset_fields = {x.get('field_name'): x.get('preset') for x in full_schema_dict.get('dataset_fields', [])}
-
-    # Populate the declared schema fields, if they are present in the extras
-    for extra_dict in dataset_dict.get('extras', []):
-        field_key = extra_dict.get('key')
-        field_value = extra_dict.get('value')
-        if field_key in dataset_fields:
-            preset = dataset_fields[field_key]
-            if preset == 'multiple_text' and field_value:
-                try:
-                    dataset_dict[field_key] = json.loads(field_value)
-                except JSONDecodeError:
-                    dataset_dict[field_key] = field_value
-            elif preset == 'date' and field_value:
-                dataset_dict[field_key] = convert_datetime_string(field_value)
-            else:
-                dataset_dict[field_key] = field_value
-
-    # Remove the extras that have been populated into the declared schema fields
-    dataset_dict['extras'] = [d for d in dataset_dict['extras'] if d.get('key') not in dataset_fields]
-
-    return dataset_dict
-
-
 def validate_tags(values_list: List[Dict]) -> List:
     """
     Validates tags strings to contain allowed characters, replaces others with spaces
@@ -85,31 +45,18 @@ def validate_tags(values_list: List[Dict]) -> List:
     return tags
 
 
-def convert_datetime_string(date_value: str) -> datetime:
-    """
-    Converts datestrings (e.g. '2023-10-06T10:12:55.614000+00:00') to datetime class instance
-    """
-    try:
-        date_value = dateparser.parse(date_value, yearfirst=True)
-        if date_value.tzinfo is not None:
-            date_value = date_value.astimezone(timezone.utc)
-    except ParserError:
-        log.error(f'A date field string value {date_value} can not be parsed to a date')
-    return date_value
-
-
-class FAIRDataPointDCATAPProfile(EuropeanDCATAP2Profile):
+class FAIRDataPointDCATAPProfile(EuropeanDCATAP3Profile):
     """
     An RDF profile for FAIR data points
     """
 
     def parse_dataset(self, dataset_dict: Dict, dataset_ref: URIRef) -> Dict:
         super(FAIRDataPointDCATAPProfile, self).parse_dataset(dataset_dict, dataset_ref)
-        dataset_dict = self._parse_contact_point(dataset_dict, dataset_ref)
 
+        #dataset_dict = self._parse_contact_point(dataset_dict, dataset_ref)
         dataset_dict = self._parse_creator(dataset_dict, dataset_ref)
 
-        dataset_dict = _convert_extras_to_declared_schema_fields(dataset_dict)
+       ## dataset_dict = _convert_extras_to_declared_schema_fields(dataset_dict)
 
         dataset_dict['tags'] = validate_tags(dataset_dict['tags'])
 
@@ -123,10 +70,10 @@ def _contact_point_details(self, subject, predicate) -> List:
 
         for agent in self.g.objects(subject, predicate):
             contact = {
-                'contact_uri': (str(agent) if isinstance(agent, URIRef)
+                'uri': (str(agent) if isinstance(agent, URIRef)
                         else self._get_vcard_property_value(agent, VCARD.hasUID)),
-                'contact_name': self._get_vcard_property_value(agent, VCARD.hasFN, VCARD.fn),
-                'contact_email': self._without_mailto(self._get_vcard_property_value(agent, VCARD.hasEmail))}
+                'name': self._get_vcard_property_value(agent, VCARD.hasFN, VCARD.fn),
+                'email': self._without_mailto(self._get_vcard_property_value(agent, VCARD.hasEmail))}
 
             contact_list.append(contact)
 
@@ -156,16 +103,16 @@ def _parse_creator(self, dataset_dict: Dict, dataset_ref: URIRef) -> Dict:
             creator_name = graph.value(creator_ref, FOAF.name)
 
             if creator_identifier:
-                creator['creator_identifier'] = str(creator_identifier)
+                creator['identifier'] = str(creator_identifier)
             if creator_name:
-                creator['creator_name'] = str(creator_name)
+                creator['name'] = str(creator_name)
             else:
                 # If the creator is a URI, use it as the identifier
                 if isinstance(creator_ref, URIRef):
-                    creator['creator_identifier'] = str(creator_ref)
-                    creator['creator_name'] = str(creator_ref)
+                    creator['identifier'] = str(creator_ref)
+                    creator['name'] = str(creator_ref)
                 else:
-                    creator['creator_name'] = str(creator_ref)
+                    creator['name'] = str(creator_ref)
 
             creators.append(creator)
 

ckanext/fairdatapoint/tests/test_data/scheming/presets/gdi_presets.yaml

@@ -0,0 +1,13 @@
+#SPDX-FileCopyrightText: 2024 Stichting Health-RI
+#
+#SPDX-License-Identifier: Apache-2.0
+
+scheming_version: 2
+about: GDI scheming field presets
+about_url: https://github.com/GenomicDataInfrastructure/gdi-userportal-ckanext-gdi-userportal/
+presets:
+  - preset_name: datetime_flex
+    values:
+      form_snippet: datetime.html
+      display_snippet: datetime.html
+      validators: scheming_isodatetime_flex convert_to_json_if_datetime

ckanext/fairdatapoint/tests/test_data/scheming/schemas/gdi_userportal.yaml

@@ -0,0 +1,106 @@
+#SPDX-FileCopyrightText: 2024 PNED G.I.E.
+#
+#SPDX-License-Identifier: Apache-2.0
+
+scheming_version: 2
+dataset_type: dataset
+about: DCAT-AP 3 compatible schema
+about_url: http://github.com/ckan/ckanext-dcat
+
+dataset_fields:
+- field_name: issued
+  label:
+    en: Issued Date
+    nl: Uitgegeven Datum
+  preset: datetime_flex
+  help_text:
+    en: "[dct:issued] This property contains the date of formal issuance (e.g., publication) of the Dataset."
+    nl: "[dct:issued] Deze eigenschap bevat de datum van formele uitgave (bijv. publicatie) van de Dataset."
+
+- field_name: modified
+  label:
+    en: Modification Date
+    nl: Datum Wijziging
+  preset: datetime_flex
+  help_text:
+    en: "[dct:modified] This property contains the most recent date on which the Dataset was changed or modified."
+    nl: "[dct:modified] Deze eigenschap bevat de meest recente datum waarop de Dataset is gewijzigd of gewijzigd."
+
+- field_name: has_version
+  label:
+    en: Has Version
+    nl: Bevat Versie
+  preset: multiple_text
+  help_inline: true
+  help_text:
+    en: "[dct:hasVersion] This property refers to a related Dataset that is a version, edition, or adaptation of the described Dataset."
+    nl: "[dct:hasVersion] Deze eigenschap verwijst naar een gerelateerde Dataset die een versie, editie of aanpassing is van de beschreven Dataset."
+
+- field_name: temporal_start
+  label:
+    en: Temporal Start Date
+    nl: Begindatum tijdsperiode
+  help_inline: true
+  help_text:
+    en: "[dct:temporal] This property refers to a temporal period that the Dataset covers."
+    nl: "[dct:temporal] Deze eigenschap verwijst naar een tijdsperiode die door de Dataset wordt gedekt."
+  preset: datetime_flex
+
+- field_name: temporal_end
+  label:
+    en: Temporal End Date
+    nl: Einddatum tijdsperiode
+  help_inline: true
+  help_text:
+    en: "[dct:temporal] This property refers to a temporal period that the Dataset covers."
+    nl: "[dct:temporal] Deze eigenschap verwijst naar een tijdsperiode die door de Dataset wordt gedekt."
+  preset: datetime_flex
+
+- field_name: creator
+  label: Creator
+  repeating_subfields:
+
+    - field_name: uri
+      label: Creator URI
+
+    - field_name: name
+      label: Creator Name
+
+    - field_name: email
+      label: Creator Email
+      display_snippet: email.html
+
+    - field_name: url
+      label: Creator URL
+      display_snippet: link.html
+
+    - field_name: type
+      label: Creator Type
+
+    - field_name: identifier
+      label: Creator Identifier
+      help_text:
+        en: Unique identifier for the creator, such as a ROR ID.
+        nl: Unieke identificatie voor de maker, zoals een ROR-ID.
+  help_text:
+    en: Entity responsible for producing the dataset.
+    nl: Entiteit die verantwoordelijk is voor het produceren van de dataset.
+
+resource_fields:
+- field_name: issued
+  label:
+    en: Issued Date
+    nl: Uitgegeven Datum
+  preset: datetime_flex
+  help_text:
+    en: "[dct:issued] This property contains the date of formal issuance (e.g., publication) of the Resource."
+    nl: "[dct:issued] Deze eigenschap bevat de datum van formele uitgave (bijv. publicatie) van de Resource."
+
+- field_name: modified
+  label:
+      en: Modification Date
+      nl: Datum Wijziging
+  preset: datetime_flex
+  help_text:
+      en: "[dct:modified] This property contains the most recent date on which the Resource was changed or modified."
+      nl: "[dct:modified] Deze eigenschap bevat de meest recente datum waarop de Resource is gewijzigd of gewijzigd."