Skip to content

Commit ba66ff5

Browse files
morningmanwsjz
morningman
and
wsjz
authored
[fix](multi-catalog)fix paimon meta properties convert (#37249) (#37958)
bp #37249 Co-authored-by: slothever <[email protected]>
1 parent b6e5281 commit ba66ff5

File tree

9 files changed

+204
-13
lines changed

9 files changed

+204
-13
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,11 @@
2020
import org.apache.doris.common.DdlException;
2121
import org.apache.doris.common.security.authentication.AuthenticationConfig;
2222
import org.apache.doris.common.security.authentication.HadoopUGI;
23+
import org.apache.doris.datasource.CatalogProperty;
2324
import org.apache.doris.datasource.ExternalCatalog;
2425
import org.apache.doris.datasource.InitCatalogLog;
2526
import org.apache.doris.datasource.SessionContext;
27+
import org.apache.doris.datasource.property.PropertyConverter;
2628
import org.apache.doris.datasource.property.constants.HMSProperties;
2729
import org.apache.doris.datasource.property.constants.PaimonProperties;
2830
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
@@ -55,8 +57,11 @@ public abstract class PaimonExternalCatalog extends ExternalCatalog {
5557
PaimonProperties.WAREHOUSE
5658
);
5759

58-
public PaimonExternalCatalog(long catalogId, String name, String comment) {
60+
public PaimonExternalCatalog(long catalogId, String name, String resource,
61+
Map<String, String> props, String comment) {
5962
super(catalogId, name, InitCatalogLog.Type.PAIMON, comment);
63+
props = PropertyConverter.convertToMetaProperties(props);
64+
catalogProperty = new CatalogProperty(resource, props);
6065
}
6166

6267
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
package org.apache.doris.datasource.paimon;
1919

20-
import org.apache.doris.datasource.CatalogProperty;
2120
import org.apache.doris.datasource.property.PropertyConverter;
2221
import org.apache.doris.datasource.property.constants.CosProperties;
2322
import org.apache.doris.datasource.property.constants.ObsProperties;
@@ -34,8 +33,7 @@ public class PaimonFileExternalCatalog extends PaimonExternalCatalog {
3433

3534
public PaimonFileExternalCatalog(long catalogId, String name, String resource,
3635
Map<String, String> props, String comment) {
37-
super(catalogId, name, comment);
38-
catalogProperty = new CatalogProperty(resource, props);
36+
super(catalogId, name, resource, props, comment);
3937
}
4038

4139
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonHMSExternalCatalog.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
package org.apache.doris.datasource.paimon;
1919

2020
import org.apache.doris.common.DdlException;
21-
import org.apache.doris.datasource.CatalogProperty;
2221
import org.apache.doris.datasource.property.constants.HMSProperties;
2322
import org.apache.doris.datasource.property.constants.PaimonProperties;
2423

@@ -37,8 +36,7 @@ public class PaimonHMSExternalCatalog extends PaimonExternalCatalog {
3736

3837
public PaimonHMSExternalCatalog(long catalogId, String name, String resource,
3938
Map<String, String> props, String comment) {
40-
super(catalogId, name, comment);
41-
catalogProperty = new CatalogProperty(resource, props);
39+
super(catalogId, name, resource, props, comment);
4240
}
4341

4442
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import org.apache.doris.common.credentials.CloudCredential;
2121
import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
2222
import org.apache.doris.common.util.LocationPath;
23-
import org.apache.doris.common.util.Util;
2423
import org.apache.doris.datasource.CatalogMgr;
2524
import org.apache.doris.datasource.InitCatalogLog.Type;
2625
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
@@ -250,13 +249,14 @@ private static Map<String, String> convertToS3Properties(Map<String, String> pro
250249
return s3Properties;
251250
}
252251

253-
private static String checkRegion(String endpoint, String region, String regionKey) {
252+
public static String checkRegion(String endpoint, String region, String regionKey) {
254253
if (Strings.isNullOrEmpty(region)) {
255254
region = S3Properties.getRegionOfEndpoint(endpoint);
256255
}
257256
if (Strings.isNullOrEmpty(region)) {
258-
String errorMsg = String.format("Required property '%s' when region is not in endpoint.", regionKey);
259-
Util.logAndThrowRuntimeException(LOG, errorMsg, new IllegalArgumentException(errorMsg));
257+
String errorMsg = String.format("No '%s' info found, using SDK default region: us-east-1", regionKey);
258+
LOG.warn(errorMsg);
259+
return "us-east-1";
260260
}
261261
return region;
262262
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ private static Map<String, String> getBeAWSPropertiesFromS3(Map<String, String>
5555
Map<String, String> beProperties = new HashMap<>();
5656
String endpoint = properties.get(S3Properties.ENDPOINT);
5757
beProperties.put(S3Properties.Env.ENDPOINT, endpoint);
58-
String region = S3Properties.getRegionOfEndpoint(endpoint);
58+
String region = PropertyConverter.checkRegion(endpoint, properties.get(S3Properties.Env.REGION),
59+
S3Properties.Env.REGION);
5960
beProperties.put(S3Properties.Env.REGION, properties.getOrDefault(S3Properties.REGION, region));
6061
if (properties.containsKey(S3Properties.ACCESS_KEY)) {
6162
beProperties.put(S3Properties.Env.ACCESS_KEY, properties.get(S3Properties.ACCESS_KEY));

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import java.util.HashMap;
4141
import java.util.List;
4242
import java.util.Map;
43+
import java.util.regex.Pattern;
4344

4445
public class S3Properties extends BaseProperties {
4546

@@ -76,6 +77,8 @@ public class S3Properties extends BaseProperties {
7677
WebIdentityTokenCredentialsProvider.class.getName(),
7778
IAMInstanceCredentialsProvider.class.getName());
7879

80+
private static final Pattern IPV4_PORT_PATTERN = Pattern.compile("((?:\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5})");
81+
7982
public static Map<String, String> credentialToMap(CloudCredentialWithEndpoint credential) {
8083
Map<String, String> resMap = new HashMap<>();
8184
resMap.put(S3Properties.ENDPOINT, credential.getEndpoint());
@@ -125,11 +128,18 @@ public static CloudCredentialWithEndpoint getEnvironmentCredentialWithEndpoint(M
125128
}
126129
String endpoint = props.get(Env.ENDPOINT);
127130
String region = props.getOrDefault(Env.REGION, S3Properties.getRegionOfEndpoint(endpoint));
131+
props.putIfAbsent(Env.REGION, PropertyConverter.checkRegion(endpoint, region, Env.REGION));
128132
return new CloudCredentialWithEndpoint(endpoint, region, credential);
129133
}
130134

131135
public static String getRegionOfEndpoint(String endpoint) {
132-
String[] endpointSplit = endpoint.split("\\.");
136+
if (IPV4_PORT_PATTERN.matcher(endpoint).find()) {
137+
// if endpoint contains '192.168.0.1:8999', return null region
138+
return null;
139+
}
140+
String[] endpointSplit = endpoint.replace("http://", "")
141+
.replace("https://", "")
142+
.split("\\.");
133143
if (endpointSplit.length < 2) {
134144
return null;
135145
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@
3838
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
3939
import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog;
4040
import org.apache.doris.datasource.property.constants.CosProperties;
41+
import org.apache.doris.datasource.property.constants.DLFProperties;
4142
import org.apache.doris.datasource.property.constants.GCSProperties;
43+
import org.apache.doris.datasource.property.constants.GlueProperties;
4244
import org.apache.doris.datasource.property.constants.HMSProperties;
4345
import org.apache.doris.datasource.property.constants.MinioProperties;
4446
import org.apache.doris.datasource.property.constants.ObsProperties;
@@ -677,4 +679,103 @@ public void testS3PropertiesConvertor() {
677679
Assertions.assertEquals("region", beProperties.get(S3Properties.Env.REGION));
678680
Assertions.assertEquals("false", beProperties.get(PropertyConverter.USE_PATH_STYLE));
679681
}
682+
683+
@Test
684+
public void testMetaPropertiesConvertor() {
685+
// test region parser
686+
Assertions.assertNull(S3Properties.getRegionOfEndpoint("http://192.168.2.30:9099/com.region.test/dir"));
687+
Assertions.assertEquals("cn-beijing",
688+
S3Properties.getRegionOfEndpoint("http://dlf.cn-beijing.aliyuncs.com/com.region.test/dir"));
689+
Assertions.assertEquals("oss-cn-beijing",
690+
S3Properties.getRegionOfEndpoint("http://oss-cn-beijing.aliyuncs.com/com.region.test/dir"));
691+
Assertions.assertEquals("us-east-1",
692+
S3Properties.getRegionOfEndpoint("http://s3.us-east-1.amazonaws.com/com.region.test/dir"));
693+
694+
//1. dlf
695+
Map<String, String> props = new HashMap<>();
696+
// iceberg.catalog.type
697+
props.put("type", "hms");
698+
props.put("hive.metastore.type", "dlf");
699+
props.put(DLFProperties.PROXY_MODE, "DLF_ONLY");
700+
props.put(DLFProperties.ENDPOINT, "dlf.cn-beijing.aliyuncs.com");
701+
props.put(DLFProperties.UID, "20239444");
702+
props.put(DLFProperties.ACCESS_KEY, "akk");
703+
props.put(DLFProperties.SECRET_KEY, "skk");
704+
props.put(DLFProperties.REGION, "cn-beijing");
705+
props.put(DLFProperties.ACCESS_PUBLIC, "false");
706+
Map<String, String> res = PropertyConverter.convertToMetaProperties(new HashMap<>(props));
707+
Assertions.assertEquals(25, res.size());
708+
Assertions.assertEquals("akk", res.get(S3Properties.Env.ACCESS_KEY));
709+
Assertions.assertEquals("skk", res.get(S3Properties.Env.SECRET_KEY));
710+
Assertions.assertEquals("akk", res.get(DataLakeConfig.CATALOG_ACCESS_KEY_ID));
711+
Assertions.assertEquals("skk", res.get(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET));
712+
Assertions.assertEquals("dlf.cn-beijing.aliyuncs.com", res.get(DataLakeConfig.CATALOG_ENDPOINT));
713+
Assertions.assertEquals("oss-cn-beijing-internal.aliyuncs.com", res.get(S3Properties.Env.ENDPOINT));
714+
Assertions.assertEquals("cn-beijing", res.get(DataLakeConfig.CATALOG_REGION_ID));
715+
Assertions.assertEquals("oss-cn-beijing", res.get(S3Properties.Env.REGION));
716+
717+
props.put(DLFProperties.ACCESS_PUBLIC, "true");
718+
res = PropertyConverter.convertToMetaProperties(new HashMap<>(props));
719+
Assertions.assertEquals(25, res.size());
720+
Assertions.assertEquals("oss-cn-beijing.aliyuncs.com", res.get(S3Properties.Env.ENDPOINT));
721+
Assertions.assertEquals("oss-cn-beijing", res.get(S3Properties.Env.REGION));
722+
723+
props.put(OssProperties.OSS_HDFS_ENABLED, "true");
724+
res = PropertyConverter.convertToMetaProperties(new HashMap<>(props));
725+
Assertions.assertEquals(28, res.size());
726+
Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem", res.get("fs.oss.impl"));
727+
Assertions.assertEquals("com.aliyun.jindodata.oss.OSS", res.get("fs.AbstractFileSystem.oss.impl"));
728+
Assertions.assertEquals("false", res.get(DataLakeConfig.CATALOG_CREATE_DEFAULT_DB));
729+
Assertions.assertEquals("cn-beijing", res.get(S3Properties.Env.REGION));
730+
731+
// 2. glue
732+
Map<String, String> props2 = new HashMap<>();
733+
props2.put("hive.metastore.type", "glue");
734+
props2.put("aws.glue.endpoint", "glue.us-east-1.amazonaws.com");
735+
props2.put("aws.glue.access-key", "akk");
736+
props2.put("aws.glue.secret-key", "skk");
737+
props2.put("aws.region", "us-east-1");
738+
res = PropertyConverter.convertToMetaProperties(props2);
739+
Assertions.assertEquals(16, res.size());
740+
Assertions.assertEquals("akk", res.get(S3Properties.Env.ACCESS_KEY));
741+
Assertions.assertEquals("skk", res.get(S3Properties.Env.SECRET_KEY));
742+
Assertions.assertEquals("s3.us-east-1.amazonaws.com", res.get(S3Properties.Env.ENDPOINT));
743+
Assertions.assertEquals("us-east-1", res.get(S3Properties.Env.REGION));
744+
745+
Map<String, String> props3 = new HashMap<>();
746+
props3.put("hive.metastore.type", "glue");
747+
props3.put(GlueProperties.ENDPOINT, "glue.us-east-1.amazonaws.com");
748+
props3.put(GlueProperties.ACCESS_KEY, "akk");
749+
props3.put(GlueProperties.SECRET_KEY, "skk");
750+
res = PropertyConverter.convertToMetaProperties(props3);
751+
Assertions.assertEquals(16, res.size());
752+
Assertions.assertEquals("akk", res.get(S3Properties.Env.ACCESS_KEY));
753+
Assertions.assertEquals("skk", res.get(S3Properties.Env.SECRET_KEY));
754+
Assertions.assertEquals("s3.us-east-1.amazonaws.com", res.get(S3Properties.Env.ENDPOINT));
755+
Assertions.assertEquals("us-east-1", res.get(S3Properties.Env.REGION));
756+
757+
// 3. s3 env
758+
Map<String, String> props4 = new HashMap<>();
759+
props4.put("hive.metastore.type", "hms");
760+
props4.put(S3Properties.Env.ENDPOINT, "s3.us-west-2.amazonaws.com");
761+
props4.put(S3Properties.Env.ACCESS_KEY, "akk");
762+
props4.put(S3Properties.Env.SECRET_KEY, "skk");
763+
res = PropertyConverter.convertToMetaProperties(new HashMap<>(props4));
764+
Assertions.assertEquals(9, res.size());
765+
Assertions.assertEquals("akk", res.get(S3Properties.Env.ACCESS_KEY));
766+
Assertions.assertEquals("skk", res.get(S3Properties.Env.SECRET_KEY));
767+
Assertions.assertEquals("s3.us-west-2.amazonaws.com", res.get(S3Properties.Env.ENDPOINT));
768+
Assertions.assertEquals("us-west-2", res.get(S3Properties.Env.REGION));
769+
770+
props4.put(S3Properties.Env.ENDPOINT, "http://172.23.56.19:9033");
771+
res = PropertyConverter.convertToMetaProperties(new HashMap<>(props4));
772+
Assertions.assertEquals(9, res.size());
773+
Assertions.assertEquals("http://172.23.56.19:9033", res.get(S3Properties.Env.ENDPOINT));
774+
Assertions.assertEquals("us-east-1", res.get(S3Properties.Env.REGION));
775+
776+
props4.put(S3Properties.Env.REGION, "north");
777+
res = PropertyConverter.convertToMetaProperties(new HashMap<>(props4));
778+
Assertions.assertEquals(9, res.size());
779+
Assertions.assertEquals("north", res.get(S3Properties.Env.REGION));
780+
}
680781
}

‎regression-test/data/external_table_p0/paimon/test_paimon_minio.out

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
-- This file is automatically generated. You should know what you did if you want to edit this
2+
-- !no_region1 --
3+
2024-01-02T10:04:05.100 2024-01-02T02:04:05.123456
4+
5+
-- !no_region2 --
6+
1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123 2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450 2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456
7+
8+
-- !region1 --
9+
2024-01-02T10:04:05.100 2024-01-02T02:04:05.123456
10+
11+
-- !region2 --
12+
1 2024-01-02T10:04:05.100 2024-01-02T10:04:05.120 2024-01-02T10:04:05.123 2024-01-02T10:04:05.123400 2024-01-02T10:04:05.123450 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T10:04:05.123456 2024-01-02T02:04:05.100 2024-01-02T02:04:05.120 2024-01-02T02:04:05.123 2024-01-02T02:04:05.123400 2024-01-02T02:04:05.123450 2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456 2024-01-02T02:04:05.123456

‎regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_paimon_minio", "p0,external,doris,external_docker,external_docker_doris") {
19+
String enabled = context.config.otherConfigs.get("enablePaimonTest")
20+
if (enabled != null && enabled.equalsIgnoreCase("true")) {
21+
String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
22+
String catalog_name = "test_paimon_minio"
23+
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
24+
String table_name = "ts_scale_orc"
25+
26+
sql """drop catalog if exists ${catalog_name}"""
27+
28+
sql """
29+
CREATE CATALOG ${catalog_name} PROPERTIES (
30+
'type' = 'paimon',
31+
'warehouse' = 's3://warehouse/wh',
32+
's3.endpoint' = 'http://${externalEnvIp}:${minio_port}',
33+
's3.access_key' = 'admin',
34+
's3.secret_key' = 'password',
35+
's3.path.style.access' = 'true'
36+
);
37+
"""
38+
sql """switch `${catalog_name}`"""
39+
sql """show databases; """
40+
sql """use `${catalog_name}`.`flink_paimon`"""
41+
order_qt_no_region1 """select ts1,ts19 from ${table_name} """
42+
order_qt_no_region2 """select * from ${table_name} """
43+
sql """drop catalog if exists ${catalog_name}"""
44+
45+
sql """drop catalog if exists ${catalog_name}_with_region"""
46+
sql """
47+
CREATE CATALOG ${catalog_name}_with_region PROPERTIES (
48+
'type' = 'paimon',
49+
'warehouse' = 's3://warehouse/wh',
50+
's3.endpoint' = 'http://${externalEnvIp}:${minio_port}',
51+
's3.access_key' = 'admin',
52+
's3.secret_key' = 'password',
53+
's3.region' = 'us-west-2',
54+
's3.path.style.access' = 'true'
55+
);
56+
"""
57+
sql """switch `${catalog_name}_with_region`"""
58+
sql """show databases; """
59+
sql """use `${catalog_name}_with_region`.`flink_paimon`"""
60+
order_qt_region1 """select ts1,ts19 from ${table_name} """
61+
order_qt_region2 """select * from ${table_name} """
62+
sql """drop catalog if exists ${catalog_name}_with_region"""
63+
}
64+
}
65+
66+

0 commit comments

Comments
 (0)