Merge remote-tracking branch 'origin/dev' into yt/sc-48774/fragment_list_consolidation

Author: shaunrd0

.github/workflows/build-ubuntu20.04-backwards-compatibility.yml

@@ -59,7 +59,7 @@ jobs:
           - ubuntu-20.04
         # Note: v2_1_0 arrays were never created so its currently skipped
         # Note: This matrix is used to set the value of TILEDB_COMPATIBILITY_VERSION
-        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0", "v2_24_0"]
+        tiledb_version: ["v1_4_0", "v1_5_0", "v1_6_0", "v1_7_0", "v2_0_0", "v2_2_0", "v2_2_3", "v2_3_0", "v2_4_0", "v2_5_0", "v2_6_0", "v2_7_0", "v2_8_3", "v2_9_1", "v2_10_0", "v2_11_0", "v2_12_3", "v2_13_2", "v2_14_0", "v2_15_0", "v2_16_3", "v2_17_5", "v2_18_3", "v2_19_1", "v2_20_1", "v2_21_1", "v2_22_0", "v2_23_0", "v2_24_0", "v2_25_0"]
     timeout-minutes: 30
     name: ${{ matrix.tiledb_version }}
     steps:

.github/workflows/build-windows.yml

@@ -136,6 +136,8 @@ jobs:
         uses: seanmiddleditch/gha-setup-ninja@v4
       - name: Prevent vcpkg from building debug variants
         run: python $env:GITHUB_WORKSPACE/scripts/ci/patch_vcpkg_triplets.py
+      - name: Install dependencies from pip
+        run: python -m pip install pybind11[global]
 
       - name: Configure TileDB
         shell: pwsh

.github/workflows/release.yml

@@ -208,5 +208,5 @@ jobs:
         uses: TileDB-Inc/github-actions/open-issue@main
         with:
           name: Release failed
-          label: bug
+          label: release
           assignee: KiterLuc,teo-tsirpanis,davisp

CMakeLists.txt

@@ -286,6 +286,12 @@ endif()
 
 list(APPEND TILEDB_C_API_RELATIVE_HEADERS
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/api_external_common.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array/array_api_external.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_schema_api_deprecated.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_schema_api_experimental.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_schema_api_external.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/array_type_enum.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/array_schema/layout_enum.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/attribute/attribute_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/attribute/attribute_api_external_experimental.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/buffer/buffer_api_external.h"
@@ -521,6 +527,7 @@ if (TILEDB_TESTS)
   # C API basics
   add_dependencies(tests unit_capi_config unit_capi_context)
   add_dependencies(tests unit_capi_array)
+  add_dependencies(tests unit_capi_array_schema)
   add_dependencies(tests unit_capi_buffer)
   add_dependencies(tests unit_capi_buffer_list)
   add_dependencies(tests unit_capi_data_order)

HISTORY.md

@@ -1,3 +1,67 @@
+# TileDB v2.25.0 Release Notes
+
+## Announcements
+
+* TileDB 2.25, includes the new current domain feature which allows to specify an area of the domain that is considered to be active for sparse arrays.
+
+## Deprecation announcements
+
+* The HDFS backend is no longer officially tested by TileDB. As announced before, it is scheduled to be removed in version 2.28, to be released in Q4 2024. [#5085](https://github.com/TileDB-Inc/TileDB/pull/5085)
+* Support for reading sparse fragments in dense arrays will be removed in version 2.27. Writting sparse fragments in dense arrays was removed in version 2.5. [#5116](https://github.com/TileDB-Inc/TileDB/pull/5116)
+* Support for returning the same results multiple times in sparse reads when ranges overlap will be removed in version 2.27. This was possible by setting `sm.merge_overlapping_ranges_experimental` to `false`, but the default `true` behavior has been there since version 2.17.
+
+## New features
+
+* REST support for current domain. [#5136](https://github.com/TileDB-Inc/TileDB/pull/5136)
+* Disallow writing outside of the current domain. [#5165](https://github.com/TileDB-Inc/TileDB/pull/5165)
+* Current domain: disallow reading outside of current domain. [#5168](https://github.com/TileDB-Inc/TileDB/pull/5168)
+
+## Improvements
+
+* Improve memory consumption for tile structures in dense reader. [#5046](https://github.com/TileDB-Inc/TileDB/pull/5046)
+
+## Defects removed
+
+* Fail early when trying to add members with relative URIs in remote groups. [#5025](https://github.com/TileDB-Inc/TileDB/pull/5025)
+* Correct defective return value in `Posix::ls_with_sizes`. [#5037](https://github.com/TileDB-Inc/TileDB/pull/5037)
+* Prevent constructing attribute with invalid cell_val_num. [#4952](https://github.com/TileDB-Inc/TileDB/pull/4952)
+* Do not mask failures when listing a directory fails on POSIX. [#5043](https://github.com/TileDB-Inc/TileDB/pull/5043)
+* Fix write queries using `sm.var_offsets.extra_element=true`. [#5033](https://github.com/TileDB-Inc/TileDB/pull/5033)
+* Fix segfaults in WebP queries ran in parallel. [#5065](https://github.com/TileDB-Inc/TileDB/pull/5065)
+* Fix exceptions with message: ```unknown exception type; no further information```. [#5080](https://github.com/TileDB-Inc/TileDB/pull/5080)
+* Fix check for out of bounds dimension in Dimension::dimension_ptr. [#5094](https://github.com/TileDB-Inc/TileDB/pull/5094)
+* Fix array latest schema selection for same MS timestamps schemas. [#5143](https://github.com/TileDB-Inc/TileDB/pull/5143)
+* Fix serialization issue with schema evolution for query v3. [#5154](https://github.com/TileDB-Inc/TileDB/pull/5154)
+
+## Configuration changes
+
+* Add `vfs.s3.storage_class` config option to set the storage class of newly uploaded S3 objects. [#5053](https://github.com/TileDB-Inc/TileDB/pull/5053)
+* Add `rest.custom_headers.*` config option to set custom headers on REST requests. [#5104](https://github.com/TileDB-Inc/TileDB/pull/5104)
+* Add `rest.payer_namespace` config option to set the namespace to be charged for REST requests. [#5105](https://github.com/TileDB-Inc/TileDB/pull/5105)
+
+## API changes
+
+### C API
+
+* Add CurrentDomain API support. [#5041](https://github.com/TileDB-Inc/TileDB/pull/5041)
+
+### C++ API
+
+* Current Domain CPP API implementation. [#5056](https://github.com/TileDB-Inc/TileDB/pull/5056)
+
+## Build System Changes
+
+* Backwards compatibility with older CMake versions for libfaketime. [#5049](https://github.com/TileDB-Inc/TileDB/pull/5049)
+* Automatic downloading of vcpkg can be disabled by enabling the `TILEDB_DISABLE_AUTO_VCPKG` CMake option, in addition to setting the environment variable with trhe same name. [#5048](https://github.com/TileDB-Inc/TileDB/pull/5048)
+* Improve embedding of `magic.mgc` and allow compiling with any libmagic version. [#4989](https://github.com/TileDB-Inc/TileDB/pull/4989)
+
+## Internal Improvements
+
+* Implement actualize function that orders data underlying `alt_var_length_view` [#5087](https://github.com/TileDB-Inc/TileDB/pull/5087)
+* Implement a partitioning function to partition cells to fit into fixed size bins [#5092](https://github.com/TileDB-Inc/TileDB/pull/5092)
+* Implementation of a `chunk_view` class to provide a subset of C++23 chunk_view, suitable for supporting external sort. [#5035](https://github.com/TileDB-Inc/TileDB/pull/5035)
+* Tests that the chunks in a `chunk_view` can be separately sorted. [#5052](https://github.com/TileDB-Inc/TileDB/pull/5052)
+
 # TileDB v2.24.2 Release Notes
 
 ## Defects removed

cmake/Options/TileDBToolchain.cmake

@@ -36,7 +36,7 @@ if (NOT DEFINED CMAKE_TOOLCHAIN_FILE)
         # Inspired from https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.10.3/cmake-modules/AzureVcpkg.cmake
         message("TILEDB_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.")
         # To help with resolving conflicts, when you update the commit, also update its date.
-        set(VCPKG_COMMIT_STRING 72010900b7cee36cea77aebb97695095c9358eaf) # 2023-12-05
+        set(VCPKG_COMMIT_STRING 7aeffc91033ad35cc4e2c152f213a866ec6c11ac) # 2024-07-13
         message("Vcpkg commit string used: ${VCPKG_COMMIT_STRING}")
         include(FetchContent)
         FetchContent_Declare(

examples/c_api/current_domain.c

@@ -74,7 +74,7 @@ void create_array() {
   tiledb_ndrectangle_set_range_for_name(ctx, ndrect, "d1", &range);
 
   // Assign the rectangle to the current domain
-  tiledb_current_domain_set_ndrectangle(current_domain, ndrect);
+  tiledb_current_domain_set_ndrectangle(ctx, current_domain, ndrect);
 
   // Create a single attribute "a" so each cell can store an integer
   tiledb_attribute_t* a;
@@ -119,21 +119,21 @@ void print_current_domain() {
 
   // Check if current domain is empty
   uint32_t is_empty;
-  tiledb_current_domain_get_is_empty(current_domain, &is_empty);
+  tiledb_current_domain_get_is_empty(ctx, current_domain, &is_empty);
 
   if (is_empty) {
     printf("Current domain: empty\n");
   } else {
     // Get current domain type
     tiledb_current_domain_type_t current_domain_type;
-    tiledb_current_domain_get_type(current_domain, &current_domain_type);
+    tiledb_current_domain_get_type(ctx, current_domain, &current_domain_type);
 
     if (current_domain_type == TILEDB_NDRECTANGLE) {
       printf("Current domain type: NDRECTANGLE\n");
 
       // Get the ND rectangle
       tiledb_ndrectangle_t* ndrect;
-      tiledb_current_domain_get_ndrectangle(current_domain, &ndrect);
+      tiledb_current_domain_get_ndrectangle(ctx, current_domain, &ndrect);
 
       tiledb_range_t range;
       tiledb_ndrectangle_get_range_from_name(ctx, ndrect, "d1", &range);
@@ -143,6 +143,23 @@ void print_current_domain() {
           *(int*)range.min,
           *(int*)range.max);
 
+      // Get datatype of range
+      tiledb_datatype_t dtype;
+      tiledb_ndrectangle_get_dtype(ctx, ndrect, 0, &dtype);
+      const char* dtype_str;
+      tiledb_datatype_to_str(dtype, &dtype_str);
+      printf("Range 0 dtype: %s\n", dtype_str);
+
+      // Get datatype of range by name
+      tiledb_ndrectangle_get_dtype_from_name(ctx, ndrect, "d1", &dtype);
+      tiledb_datatype_to_str(dtype, &dtype_str);
+      printf("Range 0 dtype by name: %s\n", dtype_str);
+
+      // Get dim num
+      uint32_t ndim;
+      tiledb_ndrectangle_get_dim_num(ctx, ndrect, &ndim);
+      printf("Range 0 dtype by name: %d\n", ndim);
+
       // Clean up
       tiledb_ndrectangle_free(&ndrect);
     } else {
@@ -191,7 +208,7 @@ void expand_current_domain() {
   tiledb_ndrectangle_set_range_for_name(ctx, ndrect, "d1", &range);
 
   // Set the rectangle to the current domain
-  tiledb_current_domain_set_ndrectangle(new_current_domain, ndrect);
+  tiledb_current_domain_set_ndrectangle(ctx, new_current_domain, ndrect);
 
   // Expand the current domain
   tiledb_array_schema_evolution_expand_current_domain(

examples/cpp_api/current_domain.cc

@@ -113,6 +113,17 @@ void print_current_domain(Context& ctx) {
   // Print the range
   std::cout << "Current domain range: [" << range[0] << ", " << range[1] << "]"
             << std::endl;
+
+  // Print datatype of range 0
+  std::cout << "Current domain range 0 datatype: "
+            << tiledb::impl::type_to_str(ndrect.range_dtype(0)) << std::endl;
+
+  // Print datatype of range d1
+  std::cout << "Current domain range 0 datatype: "
+            << tiledb::impl::type_to_str(ndrect.range_dtype("d1")) << std::endl;
+
+  // Print dim num
+  std::cout << "Current domain dim num: " << ndrect.dim_num() << std::endl;
 }
 
 void expand_current_domain(Context& ctx) {

ports/README.md

@@ -26,13 +26,13 @@ $ git commit
 
 After copying the port, add an entry to the table below. You should also contribute your changes to vcpkg and/or the upstream package repository.
 
+For ease of review when patching existing ports, you are recommended to make one commit that copies the upstream port unchanged, and another commit that makes the changes you need.
+
 ## List of port overlays
 
-| Port                       | Reason                                                                                                  |
-|----------------------------|---------------------------------------------------------------------------------------------------------|
-| `libmagic`                 | Updating to the upstream port deferred due to failures.                                                 |
-| `pcre2`                    | To be removed alongside libmagic.                                                                       |
-| `azure-storage-common-cpp` | Patching to disable default features on libxml2 (https://github.com/Azure/azure-sdk-for-cpp/pull/5221). |
-| `libfaketime`              | Port does not yet exist upstream                                                                        |
-| `vcpkg-cmake-config`       | Patching to fix build issues with CMake 3.29.1. (https://github.com/microsoft/vcpkg/pull/38017)         |
-| `google-cloud-cpp`         | Patching to remove dependency on GMock. (https://github.com/microsoft/vcpkg/pull/39802)                 |
+| Port          | Reason                                                                                                                     |
+|---------------|----------------------------------------------------------------------------------------------------------------------------|
+| `aws-c-io`    | Patching to fix MinGW build failures. (https://github.com/awslabs/aws-c-io/pull/664)                                       |
+| `aws-sdk-cpp` | Patching to fix MinGW build failures, and to avoid building test-only SDKs (https://github.com/aws/aws-sdk-cpp/pull/3061). |
+| `libmagic`    | Updating to the upstream port deferred due to failures.                                                                    |
+| `libfaketime` | Port does not yet exist upstream                                                                                           |

ports/aws-c-io/fix-pointer-mismatch.patch

@@ -0,0 +1,22 @@
+diff --git a/source/windows/secure_channel_tls_handler.c b/source/windows/secure_channel_tls_handler.c
+index b62b1a0..48f4556 100644
+--- a/source/windows/secure_channel_tls_handler.c
++++ b/source/windows/secure_channel_tls_handler.c
+@@ -53,7 +53,7 @@ struct secure_channel_ctx {
+     struct aws_tls_ctx ctx;
+     struct aws_string *alpn_list;
+     SCHANNEL_CRED credentials;
+-    PCERT_CONTEXT pcerts;
++    PCCERT_CONTEXT pcerts;
+     HCERTSTORE cert_store;
+     HCERTSTORE custom_trust_store;
+     HCRYPTPROV crypto_provider;
+@@ -188,7 +188,7 @@ static int s_manually_verify_peer_cert(struct aws_channel_handler *handler) {
+     int result = AWS_OP_ERR;
+     CERT_CONTEXT *peer_certificate = NULL;
+     HCERTCHAINENGINE engine = NULL;
+-    CERT_CHAIN_CONTEXT *cert_chain_ctx = NULL;
++    PCCERT_CHAIN_CONTEXT cert_chain_ctx = NULL;
+ 
+     /* get the peer's certificate so we can validate it.*/
+     SECURITY_STATUS status =