From be9f4d6c85ea2a24ee7d95f944de1b56a5d05ada Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 30 Aug 2022 18:21:32 +0800 Subject: [PATCH 01/14] util: add CA check and refine tests Signed-off-by: lance6716 --- dumpling/export/config.go | 3 +- util/security.go | 88 +++++++++++++++---- util/security_test.go | 180 +++++++++++++++++++++++++++++++------- util/tls_test/ca.key | 8 -- util/tls_test/ca.pem | 8 -- util/tls_test/client1.key | 8 -- util/tls_test/client1.pem | 10 --- util/tls_test/client2.key | 8 -- util/tls_test/client2.pem | 10 --- util/tls_test/generate.sh | 42 --------- util/tls_test/server.key | 8 -- util/tls_test/server.pem | 10 --- 12 files changed, 224 insertions(+), 159 deletions(-) delete mode 100644 util/tls_test/ca.key delete mode 100644 util/tls_test/ca.pem delete mode 100644 util/tls_test/client1.key delete mode 100644 util/tls_test/client1.pem delete mode 100644 util/tls_test/client2.key delete mode 100644 util/tls_test/client2.pem delete mode 100644 util/tls_test/generate.sh delete mode 100644 util/tls_test/server.key delete mode 100644 util/tls_test/server.pem diff --git a/dumpling/export/config.go b/dumpling/export/config.go index 204f889a5a091..4e8a587ed8cbb 100644 --- a/dumpling/export/config.go +++ b/dumpling/export/config.go @@ -656,7 +656,7 @@ func registerTLSConfig(conf *Config) error { } } } - tlsConfig, err = util.ToTLSConfigWithVerifyByRawbytes(conf.Security.SSLCABytes, + tlsConfig, err = util.NewTLSConfigWithVerifyCN(conf.Security.SSLCABytes, conf.Security.SSLCertBytes, conf.Security.SSLKEYBytes, []string{}) if err != nil { return errors.Trace(err) @@ -666,6 +666,7 @@ func registerTLSConfig(conf *Config) error { if conf.Host == "127.0.0.1" || len(conf.Security.SSLCertBytes) == 0 || len(conf.Security.SSLKEYBytes) == 0 { tlsConfig.InsecureSkipVerify = true } + // TODO: separate name err = mysql.RegisterTLSConfig("dumpling-tls-target", tlsConfig) if err != nil { return errors.Trace(err) diff --git a/util/security.go b/util/security.go index 0a958767a50de..82c789ab1d91d 100644 --- a/util/security.go +++ b/util/security.go @@ -58,7 +58,7 @@ func addVerifyPeerCertificate(tlsCfg *tls.Config, verifyCN []string) { } } } - return errors.Errorf("client certificate authentication failed. The Common Name from the client certificate %v was not found in the configuration cluster-verify-cn with value: %s", cns, verifyCN) + return errors.Errorf("client certificate authentication failed. The CommonName from the client certificate %v was not found in the configuration cluster-verify-cn with value: %s", cns, verifyCN) } } } @@ -107,34 +107,92 @@ func ToTLSConfigWithVerify(caPath, certPath, keyPath string, verifyCN []string) return tlsCfg, nil } -// ToTLSConfigWithVerifyByRawbytes constructs a `*tls.Config` from the CA, certification and key bytes -// and add verify for CN. -func ToTLSConfigWithVerifyByRawbytes(caData, certData, keyData []byte, verifyCN []string) (*tls.Config, error) { - var certificates []tls.Certificate +// NewTLSConfigWithVerifyCN constructs a `*tls.Config` from given data: +// - caData: represents the MySQL server's CA certificate. If not empty, the TLS connection only allows this CA. +// - certData, keyData: represents the client's certificate and key. If not empty, the TLS connection will use this +// certificate. Otherwise, it will use self-generated certificate. +// - verifyCN: represents the Common Name that the MySQL server's certificate is expected to have. If not empty, the TLS +// connection only allows the certificates with these CN. +func NewTLSConfigWithVerifyCN(caData, certData, keyData []byte, verifyCN []string) (*tls.Config, error) { + var ( + certificates []tls.Certificate + certPool *x509.CertPool + verifyFuncs []func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error + ) + if len(certData) != 0 && len(keyData) != 0 { // Generate a key pair from your pem-encoded cert and key ([]byte). cert, err := tls.X509KeyPair(certData, keyData) if err != nil { - return nil, errors.New("failed to generate cert") + return nil, errors.Wrap(err, "failed to generate cert") } certificates = []tls.Certificate{cert} } - // Create a certificate pool from CA - certPool := x509.NewCertPool() - // Append the certificates from the CA - if !certPool.AppendCertsFromPEM(caData) { - return nil, errors.New("failed to append ca certs") - } /* #nosec G402 */ tlsCfg := &tls.Config{ MinVersion: tls.VersionTLS10, Certificates: certificates, - RootCAs: certPool, - ClientCAs: certPool, NextProtos: []string{"h2", "http/1.2"}, // specify `h2` to let Go use HTTP/2. } - addVerifyPeerCertificate(tlsCfg, verifyCN) + + if len(caData) != 0 { + certPool = x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caData) { + return nil, errors.New("failed to append ca certs") + } + tlsCfg.RootCAs = certPool + tlsCfg.ClientCAs = certPool + + // check the received MySQL server certificate during TLS handshake + verifyFuncs = append(verifyFuncs, func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return errors.New("no certificates available to verify") + } + + cert, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return err + } + + opts := x509.VerifyOptions{Roots: certPool} + if _, err = cert.Verify(opts); err != nil { + return errors.Wrap(err, "different CA is used") + } + return nil + }) + } + + if len(verifyCN) != 0 { + checkCN := make(map[string]struct{}) + for _, cn := range verifyCN { + cn = strings.TrimSpace(cn) + checkCN[cn] = struct{}{} + } + tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert + // check the received CommonName of MySQL server certificate's verify chain during TLS handshake + verifyFuncs = append(verifyFuncs, func(_ [][]byte, verifiedChains [][]*x509.Certificate) error { + cns := make([]string, 0, len(verifiedChains)) + for _, chains := range verifiedChains { + for _, chain := range chains { + cns = append(cns, chain.Subject.CommonName) + if _, match := checkCN[chain.Subject.CommonName]; match { + return nil + } + } + } + return errors.Errorf("MySQL server certificate authentication failed. The Common Name from the certificate %v was not found in the configuration values: %s", cns, verifyCN) + }) + } + + tlsCfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + for _, f := range verifyFuncs { + if err := f(rawCerts, verifiedChains); err != nil { + return err + } + } + return nil + } return tlsCfg, nil } diff --git a/util/security_test.go b/util/security_test.go index 753cccfb48d98..7075b990c659d 100644 --- a/util/security_test.go +++ b/util/security_test.go @@ -15,19 +15,25 @@ package util_test import ( + "bytes" "context" + "crypto/rand" + "crypto/rsa" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "fmt" "io" - "io/ioutil" + "math/big" "net" "net/http" "net/http/httptest" "net/url" "os" - "path" "path/filepath" "testing" + "time" "github.com/pingcap/tidb/util" "github.com/stretchr/testify/require" @@ -80,7 +86,7 @@ func TestInvalidTLS(t *testing.T) { _, err := util.NewTLS(caPath, "", "", "localhost", nil) require.Regexp(t, "could not read ca certificate:.*", err.Error()) - err = ioutil.WriteFile(caPath, []byte("invalid ca content"), 0644) + err = os.WriteFile(caPath, []byte("invalid ca content"), 0644) require.NoError(t, err) _, err = util.NewTLS(caPath, "", "", "localhost", nil) require.Regexp(t, "failed to append ca certs", err.Error()) @@ -90,37 +96,62 @@ func TestInvalidTLS(t *testing.T) { _, err = util.NewTLS(caPath, certPath, keyPath, "localhost", nil) require.Regexp(t, "could not load client key pair: open.*", err.Error()) - err = ioutil.WriteFile(certPath, []byte("invalid cert content"), 0644) + err = os.WriteFile(certPath, []byte("invalid cert content"), 0644) require.NoError(t, err) - err = ioutil.WriteFile(keyPath, []byte("invalid key content"), 0600) + err = os.WriteFile(keyPath, []byte("invalid key content"), 0600) require.NoError(t, err) _, err = util.NewTLS(caPath, certPath, keyPath, "localhost", nil) require.Regexp(t, "could not load client key pair: tls.*", err.Error()) } -func TestCheckCN(t *testing.T) { - dir, err := os.Getwd() - require.NoError(t, err) +func TestVerifyCN(t *testing.T) { + caData, certs, keys := generateCerts(t, []string{"server", "client1", "client2"}) + serverCert, serverKey := certs[0], keys[0] + client1Cert, client1Key := certs[1], keys[1] + client2Cert, client2Key := certs[2], keys[2] - dir = path.Join(dir, "tls_test") - caPath, certPath, keyPath := getTestCertFile(dir, "server") // only allow client1 to visit - serverTLS, err := util.ToTLSConfigWithVerify(caPath, certPath, keyPath, []string{"client1"}) + serverTLS, err := util.NewTLSConfigWithVerifyCN(caData, serverCert, serverKey, []string{"client1"}) require.NoError(t, err) + port := 9292 + url := fmt.Sprintf("https://127.0.0.1:%d", port) + ctx, cancel := context.WithCancel(context.Background()) + server := runServer(ctx, serverTLS, port, t) + defer func() { + cancel() + server.Close() + }() - caPath1, certPath1, keyPath1 := getTestCertFile(dir, "client1") - caData, certData, keyData := loadTLSContent(t, caPath1, certPath1, keyPath1) - clientTLS1, err := util.ToTLSConfigWithVerifyByRawbytes(caData, certData, keyData, []string{}) + clientTLS1, err := util.NewTLSConfigWithVerifyCN(caData, client1Cert, client1Key, []string{}) require.NoError(t, err) - - _, err = util.ToTLSConfigWithVerifyByRawbytes(caData, []byte{}, []byte{}, []string{}) + resp, err := util.ClientWithTLS(clientTLS1).Get(url) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) require.NoError(t, err) + require.Equal(t, "This an example server", string(body)) + require.NoError(t, resp.Body.Close()) - caPath2, certPath2, keyPath2 := getTestCertFile(dir, "client2") - clientTLS2, err := util.ToTLSConfigWithVerify(caPath2, certPath2, keyPath2, nil) + // client2 can't visit server + clientTLS2, err := util.NewTLSConfigWithVerifyCN(caData, client2Cert, client2Key, []string{}) require.NoError(t, err) + resp, err = util.ClientWithTLS(clientTLS2).Get(url) + require.ErrorContains(t, err, "tls: bad certificate") + if resp != nil { + require.NoError(t, resp.Body.Close()) + } +} - port := 9292 +func TestWithOrWithoutCA(t *testing.T) { + caData, certs, keys := generateCerts(t, []string{"server", "client"}) + serverCert, serverKey := certs[0], keys[0] + clientCert, clientKey := certs[1], keys[1] + + caData2, certs2, keys2 := generateCerts(t, []string{"client2"}) + clientCert2, clientKey2 := certs2[0], keys2[0] + + serverTLS, err := util.NewTLSConfigWithVerifyCN(caData, serverCert, serverKey, nil) + require.NoError(t, err) + port := 9293 url := fmt.Sprintf("https://127.0.0.1:%d", port) ctx, cancel := context.WithCancel(context.Background()) server := runServer(ctx, serverTLS, port, t) @@ -129,6 +160,9 @@ func TestCheckCN(t *testing.T) { server.Close() }() + // test only CA + clientTLS1, err := util.NewTLSConfigWithVerifyCN(caData, nil, nil, nil) + require.NoError(t, err) resp, err := util.ClientWithTLS(clientTLS1).Get(url) require.NoError(t, err) body, err := io.ReadAll(resp.Body) @@ -136,9 +170,32 @@ func TestCheckCN(t *testing.T) { require.Equal(t, "This an example server", string(body)) require.NoError(t, resp.Body.Close()) - // client2 can't visit server + // test without CA + clientTLS2, err := util.NewTLSConfigWithVerifyCN(nil, clientCert, clientKey, nil) + require.NoError(t, err) + // inject CA to imitate our generated CA is a trusted CA + clientTLS2.RootCAs = clientTLS1.RootCAs resp, err = util.ClientWithTLS(clientTLS2).Get(url) - require.Regexp(t, ".*tls: bad certificate", err.Error()) + require.NoError(t, err) + body, err = io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "This an example server", string(body)) + require.NoError(t, resp.Body.Close()) + + // test wrong CA should fail + _ = clientCert2 + _ = clientKey2 + clientTLS3, err := util.NewTLSConfigWithVerifyCN(caData2, nil, nil, []string{}) + require.NoError(t, err) + // inject CA to imitate our generated CA is a trusted CA + certPool := x509.NewCertPool() + ok := certPool.AppendCertsFromPEM(caData) + require.True(t, ok) + ok = certPool.AppendCertsFromPEM(caData2) + require.True(t, ok) + clientTLS3.RootCAs = certPool + resp, err = util.ClientWithTLS(clientTLS3).Get(url) + require.ErrorContains(t, err, "different CA is used") if resp != nil { require.NoError(t, resp.Body.Close()) } @@ -163,20 +220,81 @@ func runServer(ctx context.Context, tlsCfg *tls.Config, port int, t *testing.T) return server } -func getTestCertFile(dir, role string) (string, string, string) { - return path.Join(dir, "ca.pem"), path.Join(dir, fmt.Sprintf("%s.pem", role)), path.Join(dir, fmt.Sprintf("%s.key", role)) -} +// generateCerts returns the PEM contents of a CA certificate and some certificates and private keys per Common Name in +// commonNames. +// thanks to https://shaneutt.com/blog/golang-ca-and-signed-cert-go/. +func generateCerts(t *testing.T, commonNames []string) (caCert []byte, certs [][]byte, keys [][]byte) { + caPrivKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + ca := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: pkix.Name{ + Organization: []string{"test"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } -func loadTLSContent(t *testing.T, caPath, certPath, keyPath string) (caData, certData, keyData []byte) { - // NOTE we make sure the file exists,so we don't need to check the error - var err error - caData, err = ioutil.ReadFile(caPath) + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey) require.NoError(t, err) - certData, err = ioutil.ReadFile(certPath) + caPEM := new(bytes.Buffer) + err = pem.Encode(caPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) require.NoError(t, err) - keyData, err = ioutil.ReadFile(keyPath) + caPrivKeyPEM := new(bytes.Buffer) + err = pem.Encode(caPrivKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(caPrivKey), + }) require.NoError(t, err) - return + + caBytes = caPEM.Bytes() + + for _, cn := range commonNames { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1658), + Subject: pkix.Name{ + Organization: []string{"test"}, + CommonName: cn, + }, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + certPrivKey, err2 := rsa.GenerateKey(rand.Reader, 4096) + require.NoError(t, err2) + + certBytes, err2 := x509.CreateCertificate(rand.Reader, cert, ca, &certPrivKey.PublicKey, caPrivKey) + require.NoError(t, err2) + + certPEM := new(bytes.Buffer) + err2 = pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + require.NoError(t, err2) + + certPrivKeyPEM := new(bytes.Buffer) + err2 = pem.Encode(certPrivKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey), + }) + require.NoError(t, err2) + certs = append(certs, certPEM.Bytes()) + keys = append(keys, certPrivKeyPEM.Bytes()) + } + + return caBytes, certs, keys } diff --git a/util/tls_test/ca.key b/util/tls_test/ca.key deleted file mode 100644 index 6da9f0bc02c48..0000000000000 --- a/util/tls_test/ca.key +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN EC PARAMETERS----- -BggqhkjOPQMBBw== ------END EC PARAMETERS----- ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIMO+fZeShFQVJa6kU2F90yju134dyKHh4f22esdhHlxboAoGCCqGSM49 -AwEHoUQDQgAEPy5/g45R5YKfYtVh6sF+SMGfZ9Ng0E7ZZcB8YSQzjNfJ7JMiw1Wj -BfsnY7kRjjUiXdxgr9vZEA3nCqUAkKIhFg== ------END EC PRIVATE KEY----- diff --git a/util/tls_test/ca.pem b/util/tls_test/ca.pem deleted file mode 100644 index f5cc7ea53f15b..0000000000000 --- a/util/tls_test/ca.pem +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBGTCBwAIJAPXAD+MhNtlhMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMMCWxvY2Fs -aG9zdDAgFw0yMDAzMTExMzA0MDJaGA8yMjkzMTIyNTEzMDQwMlowFDESMBAGA1UE -AwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEPy5/g45R5YKf -YtVh6sF+SMGfZ9Ng0E7ZZcB8YSQzjNfJ7JMiw1WjBfsnY7kRjjUiXdxgr9vZEA3n -CqUAkKIhFjAKBggqhkjOPQQDAgNIADBFAiBpx/nUnXE4kHKA+7JYQDsligGXgvJA -7UMM8ryLGtNYxgIhAIZPPCTHBBka67Xj3DlGqbsRVUIwVLeHucUkmKeYfIX6 ------END CERTIFICATE----- diff --git a/util/tls_test/client1.key b/util/tls_test/client1.key deleted file mode 100644 index de1925c473811..0000000000000 --- a/util/tls_test/client1.key +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN EC PARAMETERS----- -BggqhkjOPQMBBw== ------END EC PARAMETERS----- ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIG/udmMi583GVhwCIxtQoBM09qe9gZfGMM0c3Z5CxzPHoAoGCCqGSM49 -AwEHoUQDQgAEpAv4b38dVtwyMvE0mAQq8d3CYZwS6EzatpeSpbSUmzY9mkpHQ8l0 -Y/VGeg/JyITa6uJjocU8E8LcwQLl0o3Waw== ------END EC PRIVATE KEY----- diff --git a/util/tls_test/client1.pem b/util/tls_test/client1.pem deleted file mode 100644 index 21d1ea65f698f..0000000000000 --- a/util/tls_test/client1.pem +++ /dev/null @@ -1,10 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBaTCCAQ+gAwIBAgIJAIIv2AIyeo1DMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM -CWxvY2FsaG9zdDAgFw0yMDAzMTExMzA0MDJaGA8yMjkzMTIyNTEzMDQwMlowEjEQ -MA4GA1UEAwwHY2xpZW50MTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABKQL+G9/ -HVbcMjLxNJgEKvHdwmGcEuhM2raXkqW0lJs2PZpKR0PJdGP1RnoPyciE2uriY6HF -PBPC3MEC5dKN1mujSjBIMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATALBgNV -HQ8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMAoGCCqGSM49 -BAMCA0gAMEUCIFb9MwwqTmecR8dzIROKpeNMAZFp55ckPTO2VO4j0eLnAiEAuXRJ -YSH5E8mMJ3u/xwOneEMso2GF54xNKt667Ilxz6A= ------END CERTIFICATE----- diff --git a/util/tls_test/client2.key b/util/tls_test/client2.key deleted file mode 100644 index 03427c72517a0..0000000000000 --- a/util/tls_test/client2.key +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN EC PARAMETERS----- -BggqhkjOPQMBBw== ------END EC PARAMETERS----- ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIHP4IWExGDB0azQdiqnM0TXw1h825IQjiQe5dmA6UQ2CoAoGCCqGSM49 -AwEHoUQDQgAE7bJIe6h8a6a0ySiQABSl9CDwOg6K1skf2EGtyFCNd09jX8KArx7Y -VleH/fX0sMljmksELyawI9taOBFi3MaPsw== ------END EC PRIVATE KEY----- diff --git a/util/tls_test/client2.pem b/util/tls_test/client2.pem deleted file mode 100644 index d31b16453e271..0000000000000 --- a/util/tls_test/client2.pem +++ /dev/null @@ -1,10 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBaTCCAQ+gAwIBAgIJAIIv2AIyeo1EMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM -CWxvY2FsaG9zdDAgFw0yMDAzMTExMzA0MDJaGA8yMjkzMTIyNTEzMDQwMlowEjEQ -MA4GA1UEAwwHY2xpZW50MjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABO2ySHuo -fGumtMkokAAUpfQg8DoOitbJH9hBrchQjXdPY1/CgK8e2FZXh/319LDJY5pLBC8m -sCPbWjgRYtzGj7OjSjBIMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATALBgNV -HQ8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMAoGCCqGSM49 -BAMCA0gAMEUCIQD4uiAwz1uxtfY03imTC1B9+GJn49uCWiVKb69y7osOagIgF9k0 -w2IUhboI3nJMl7dyj+fCNCFKFIMogLfPUZUu64A= ------END CERTIFICATE----- diff --git a/util/tls_test/generate.sh b/util/tls_test/generate.sh deleted file mode 100644 index c0198cb7663c8..0000000000000 --- a/util/tls_test/generate.sh +++ /dev/null @@ -1,42 +0,0 @@ -#! /bin/bash -# Copyright 2022 PingCAP, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this script used for generating tls file for unit test -# execute: cd pkg/utils/tls_test && sh generate.sh && cd - - - cat - > "ipsan.cnf" < /dev/null - -for role in server client1 client2; do - openssl ecparam -out "$role.key" -name prime256v1 -genkey - openssl req -new -batch -sha256 -subj "/CN=${role}" -key "$role.key" -out "$role.csr" - openssl x509 -req -sha256 -days 100000 -extensions EXT -extfile "ipsan.cnf" -in "$role.csr" -CA "ca.pem" -CAkey "ca.key" -CAcreateserial -out "$role.pem" 2> /dev/null -done diff --git a/util/tls_test/server.key b/util/tls_test/server.key deleted file mode 100644 index 199fe939797d1..0000000000000 --- a/util/tls_test/server.key +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN EC PARAMETERS----- -BggqhkjOPQMBBw== ------END EC PARAMETERS----- ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIJAYZp8J7QW4QE8xqrbpZloRIOd6rGJZROZjTFtYxl2PoAoGCCqGSM49 -AwEHoUQDQgAEpmwUvzuPv41DPIRJlseO8L52nnJag/DCWdE+MXHWnPxkdcrojnS+ -tLOacGI7R6OmUBen3R55InBppEPQJVDoMg== ------END EC PRIVATE KEY----- diff --git a/util/tls_test/server.pem b/util/tls_test/server.pem deleted file mode 100644 index 208a6c0069b8d..0000000000000 --- a/util/tls_test/server.pem +++ /dev/null @@ -1,10 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIBaDCCAQ6gAwIBAgIJAIIv2AIyeo1CMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM -CWxvY2FsaG9zdDAgFw0yMDAzMTExMzA0MDJaGA8yMjkzMTIyNTEzMDQwMlowETEP -MA0GA1UEAwwGc2VydmVyMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEpmwUvzuP -v41DPIRJlseO8L52nnJag/DCWdE+MXHWnPxkdcrojnS+tLOacGI7R6OmUBen3R55 -InBppEPQJVDoMqNKMEgwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMAsGA1Ud -DwQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwCgYIKoZIzj0E -AwIDSAAwRQIgHoMs+/kHmRys8q/BvL/iqZq7zpqXgskG3FKWRsBHILECIQClnmS1 -0yqkuSHb9/ScG4fivhsuih6grGZrFRji8mtOkA== ------END CERTIFICATE----- From 9c326061a8ccee9c719bf3aac92c3f18efd4726e Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 30 Aug 2022 18:22:25 +0800 Subject: [PATCH 02/14] remove debug Signed-off-by: lance6716 --- util/security_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/util/security_test.go b/util/security_test.go index 7075b990c659d..29a6edec459a6 100644 --- a/util/security_test.go +++ b/util/security_test.go @@ -146,8 +146,7 @@ func TestWithOrWithoutCA(t *testing.T) { serverCert, serverKey := certs[0], keys[0] clientCert, clientKey := certs[1], keys[1] - caData2, certs2, keys2 := generateCerts(t, []string{"client2"}) - clientCert2, clientKey2 := certs2[0], keys2[0] + caData2, _, _ := generateCerts(t, nil) serverTLS, err := util.NewTLSConfigWithVerifyCN(caData, serverCert, serverKey, nil) require.NoError(t, err) @@ -183,8 +182,6 @@ func TestWithOrWithoutCA(t *testing.T) { require.NoError(t, resp.Body.Close()) // test wrong CA should fail - _ = clientCert2 - _ = clientKey2 clientTLS3, err := util.NewTLSConfigWithVerifyCN(caData2, nil, nil, []string{}) require.NoError(t, err) // inject CA to imitate our generated CA is a trusted CA From 9a34203f688f6e4c5d364c8702bc8b520301291b Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 31 Aug 2022 11:09:10 +0800 Subject: [PATCH 03/14] refine comment Signed-off-by: lance6716 --- dumpling/export/config.go | 2 +- util/security.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dumpling/export/config.go b/dumpling/export/config.go index 4e8a587ed8cbb..71ec85da59752 100644 --- a/dumpling/export/config.go +++ b/dumpling/export/config.go @@ -666,7 +666,7 @@ func registerTLSConfig(conf *Config) error { if conf.Host == "127.0.0.1" || len(conf.Security.SSLCertBytes) == 0 || len(conf.Security.SSLKEYBytes) == 0 { tlsConfig.InsecureSkipVerify = true } - // TODO: separate name + // TODO: use separate name when dumpling as DM library err = mysql.RegisterTLSConfig("dumpling-tls-target", tlsConfig) if err != nil { return errors.Trace(err) diff --git a/util/security.go b/util/security.go index 82c789ab1d91d..0994481f7f3a7 100644 --- a/util/security.go +++ b/util/security.go @@ -58,7 +58,7 @@ func addVerifyPeerCertificate(tlsCfg *tls.Config, verifyCN []string) { } } } - return errors.Errorf("client certificate authentication failed. The CommonName from the client certificate %v was not found in the configuration cluster-verify-cn with value: %s", cns, verifyCN) + return errors.Errorf("client certificate authentication failed. The Common Name from the client certificate %v was not found in the configuration cluster-verify-cn with value: %s", cns, verifyCN) } } } @@ -181,7 +181,7 @@ func NewTLSConfigWithVerifyCN(caData, certData, keyData []byte, verifyCN []strin } } } - return errors.Errorf("MySQL server certificate authentication failed. The Common Name from the certificate %v was not found in the configuration values: %s", cns, verifyCN) + return errors.Errorf("client certificate authentication failed. The Common Name from the client certificate %v was not found in the configuration cluster-verify-cn with value: %s", cns, verifyCN) }) } From 9c6c0d7c8566eb933f6752a9195ac0a7a26f6d76 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Thu, 1 Sep 2022 13:17:58 +0800 Subject: [PATCH 04/14] add comment Signed-off-by: lance6716 --- util/security.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/security.go b/util/security.go index 0994481f7f3a7..e59d8129d7358 100644 --- a/util/security.go +++ b/util/security.go @@ -145,6 +145,7 @@ func NewTLSConfigWithVerifyCN(caData, certData, keyData []byte, verifyCN []strin tlsCfg.ClientCAs = certPool // check the received MySQL server certificate during TLS handshake + // thanks to https://cloud.google.com/sql/docs/mysql/samples/cloud-sql-mysql-databasesql-sslcerts verifyFuncs = append(verifyFuncs, func(rawCerts [][]byte, _ [][]*x509.Certificate) error { if len(rawCerts) == 0 { return errors.New("no certificates available to verify") @@ -157,7 +158,7 @@ func NewTLSConfigWithVerifyCN(caData, certData, keyData []byte, verifyCN []strin opts := x509.VerifyOptions{Roots: certPool} if _, err = cert.Verify(opts); err != nil { - return errors.Wrap(err, "different CA is used") + return errors.Wrap(err, "can't verify certificate, maybe different CA is used") } return nil }) From 7a956244bdd5e3f5d7f5ed15f8e162cea6ea3648 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Mon, 5 Sep 2022 13:23:02 +0800 Subject: [PATCH 05/14] fix TODO Signed-off-by: lance6716 --- dumpling/export/config.go | 6 ++++-- dumpling/export/dump.go | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dumpling/export/config.go b/dumpling/export/config.go index 71ec85da59752..a0e50f067299d 100644 --- a/dumpling/export/config.go +++ b/dumpling/export/config.go @@ -17,6 +17,7 @@ import ( "github.com/coreos/go-semver/semver" "github.com/docker/go-units" "github.com/go-sql-driver/mysql" + "github.com/google/uuid" "github.com/pingcap/errors" "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/br/pkg/version" @@ -103,6 +104,7 @@ type Config struct { User string Password string `json:"-"` Security struct { + DriveTLSName string `json:"-"` CAPath string CertPath string KeyPath string @@ -666,8 +668,8 @@ func registerTLSConfig(conf *Config) error { if conf.Host == "127.0.0.1" || len(conf.Security.SSLCertBytes) == 0 || len(conf.Security.SSLKEYBytes) == 0 { tlsConfig.InsecureSkipVerify = true } - // TODO: use separate name when dumpling as DM library - err = mysql.RegisterTLSConfig("dumpling-tls-target", tlsConfig) + conf.Security.DriveTLSName = "dumpling" + uuid.NewString() + err = mysql.RegisterTLSConfig(conf.Security.DriveTLSName, tlsConfig) if err != nil { return errors.Trace(err) } diff --git a/dumpling/export/dump.go b/dumpling/export/dump.go index 1310c37ffa314..37825895a10de 100644 --- a/dumpling/export/dump.go +++ b/dumpling/export/dump.go @@ -1236,6 +1236,9 @@ func (d *Dumper) Close() error { if d.dbHandle != nil { return d.dbHandle.Close() } + if d.conf.Security.DriveTLSName != "" { + mysql.DeregisterTLSConfig(d.conf.Security.DriveTLSName) + } return nil } From 5872527c9eea0b6b03765595783d4299b6485ae8 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Mon, 5 Sep 2022 15:33:04 +0800 Subject: [PATCH 06/14] fix dumpling/lightning invocation Signed-off-by: lance6716 --- br/pkg/lightning/common/security.go | 47 ++++++++++++------- br/pkg/lightning/common/security_test.go | 28 ++++------- br/pkg/lightning/config/config.go | 46 ++++++++++++++++-- br/pkg/lightning/lightning.go | 13 ++++- dumpling/export/config.go | 60 +++++++++++------------- util/security.go | 7 +-- 6 files changed, 125 insertions(+), 76 deletions(-) diff --git a/br/pkg/lightning/common/security.go b/br/pkg/lightning/common/security.go index 7f55c07603919..aa3b387fdb513 100644 --- a/br/pkg/lightning/common/security.go +++ b/br/pkg/lightning/common/security.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/br/pkg/httputil" + "github.com/pingcap/tidb/util" "github.com/tikv/client-go/v2/config" pd "github.com/tikv/pd/client" "google.golang.org/grpc" @@ -32,12 +33,15 @@ import ( ) type TLS struct { - caPath string - certPath string - keyPath string - inner *tls.Config - client *http.Client - url string + caPath string + certPath string + keyPath string + caBytes []byte + certBytes []byte + keyBytes []byte + inner *tls.Config + client *http.Client + url string } // ToTLSConfig constructs a `*tls.Config` from the CA, certification and key @@ -89,25 +93,28 @@ func ToTLSConfig(caPath, certPath, keyPath string) (*tls.Config, error) { // certificate and key paths. // // If the CA path is empty, returns an instance where TLS is disabled. -func NewTLS(caPath, certPath, keyPath, host string) (*TLS, error) { - if len(caPath) == 0 { +func NewTLS(caPath, certPath, keyPath, host string, caBytes, certBytes, keyBytes []byte) (*TLS, error) { + if len(caBytes) == 0 && len(certBytes) == 0 && len(keyBytes) == 0 { return &TLS{ inner: nil, client: &http.Client{}, url: "http://" + host, }, nil } - inner, err := ToTLSConfig(caPath, certPath, keyPath) + inner, err := util.NewTLSConfigWithVerifyCN(caBytes, certBytes, keyBytes, nil) if err != nil { return nil, errors.Trace(err) } return &TLS{ - caPath: caPath, - certPath: certPath, - keyPath: keyPath, - inner: inner, - client: httputil.NewClient(inner), - url: "https://" + host, + caPath: caPath, + certPath: certPath, + keyPath: keyPath, + caBytes: caBytes, + certBytes: certBytes, + keyBytes: keyBytes, + inner: inner, + client: httputil.NewClient(inner), + url: "https://" + host, }, nil } @@ -158,12 +165,16 @@ func (tc *TLS) GetJSON(ctx context.Context, path string, v interface{}) error { func (tc *TLS) ToPDSecurityOption() pd.SecurityOption { return pd.SecurityOption{ - CAPath: tc.caPath, - CertPath: tc.certPath, - KeyPath: tc.keyPath, + CAPath: tc.caPath, + CertPath: tc.certPath, + KeyPath: tc.keyPath, + SSLCABytes: tc.caBytes, + SSLCertBytes: tc.certBytes, + SSLKEYBytes: tc.keyBytes, } } +// TODO: TiKV does not support pass in content func (tc *TLS) ToTiKVSecurityConfig() config.Security { return config.Security{ ClusterSSLCA: tc.caPath, diff --git a/br/pkg/lightning/common/security_test.go b/br/pkg/lightning/common/security_test.go index 3359935b73bf0..ec3e04a8fd8ca 100644 --- a/br/pkg/lightning/common/security_test.go +++ b/br/pkg/lightning/common/security_test.go @@ -42,7 +42,7 @@ func TestGetJSONInsecure(t *testing.T) { u, err := url.Parse(mockServer.URL) require.NoError(t, err) - tls, err := common.NewTLS("", "", "", u.Host) + tls, err := common.NewTLS("", "", "", u.Host, nil, nil, nil) require.NoError(t, err) var result struct{ Path string } @@ -73,15 +73,8 @@ func TestGetJSONSecure(t *testing.T) { func TestInvalidTLS(t *testing.T) { tempDir := t.TempDir() caPath := filepath.Join(tempDir, "ca.pem") - _, err := common.NewTLS(caPath, "", "", "localhost") - require.Regexp(t, "could not read ca certificate:.*", err.Error()) - err = os.WriteFile(caPath, []byte("invalid ca content"), 0o644) - require.NoError(t, err) - _, err = common.NewTLS(caPath, "", "", "localhost") - require.Regexp(t, "failed to append ca certs", err.Error()) - - err = os.WriteFile(caPath, []byte(`-----BEGIN CERTIFICATE----- + caContent := []byte(`-----BEGIN CERTIFICATE----- MIIBITCBxwIUf04/Hucshr7AynmgF8JeuFUEf9EwCgYIKoZIzj0EAwIwEzERMA8G A1UEAwwIYnJfdGVzdHMwHhcNMjIwNDEzMDcyNDQxWhcNMjIwNDE1MDcyNDQxWjAT MREwDwYDVQQDDAhicl90ZXN0czBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABL+X @@ -90,20 +83,19 @@ wczUg0AbaFFaCI+FAk3K9vbB9JeIORgGKS+F1TKip5tvm96g7S5lq8SgY38SXVc3 ze4ZnCkwJdP2VdpI3WZsoI7zAiEAjP8X1c0iFwYxdAbQAveX+9msVrzyUpZOohi4 RtgQTNI= -----END CERTIFICATE----- -`), 0o644) +`) + err := os.WriteFile(caPath, caContent, 0o644) require.NoError(t, err) certPath := filepath.Join(tempDir, "test.pem") keyPath := filepath.Join(tempDir, "test.key") - tls, err := common.NewTLS(caPath, certPath, keyPath, "localhost") - _, err = tls.TLSConfig().GetCertificate(nil) - require.Regexp(t, "could not load client key pair: open.*", err.Error()) - err = os.WriteFile(certPath, []byte("invalid cert content"), 0o644) + certContent := []byte("invalid cert content") + err = os.WriteFile(certPath, certContent, 0o644) require.NoError(t, err) - err = os.WriteFile(keyPath, []byte("invalid key content"), 0o600) + keyContent := []byte("invalid key content") + err = os.WriteFile(keyPath, keyContent, 0o600) require.NoError(t, err) - tls, err = common.NewTLS(caPath, certPath, keyPath, "localhost") - _, err = tls.TLSConfig().GetCertificate(nil) - require.Regexp(t, "could not load client key pair: tls.*", err.Error()) + _, err = common.NewTLS(caPath, certPath, keyPath, "localhost", caContent, certContent, keyContent) + require.ErrorContains(t, err, "tls: failed to find any PEM data in certificate input") } diff --git a/br/pkg/lightning/config/config.go b/br/pkg/lightning/config/config.go index f88e7693d6664..f16bfaeacc1bb 100644 --- a/br/pkg/lightning/config/config.go +++ b/br/pkg/lightning/config/config.go @@ -32,11 +32,13 @@ import ( "github.com/BurntSushi/toml" "github.com/docker/go-units" gomysql "github.com/go-sql-driver/mysql" + "github.com/google/uuid" "github.com/pingcap/errors" "github.com/pingcap/tidb/br/pkg/lightning/common" "github.com/pingcap/tidb/br/pkg/lightning/log" tidbcfg "github.com/pingcap/tidb/config" "github.com/pingcap/tidb/parser/mysql" + "github.com/pingcap/tidb/util" filter "github.com/pingcap/tidb/util/table-filter" router "github.com/pingcap/tidb/util/table-router" "go.uber.org/atomic" @@ -155,7 +157,18 @@ func (cfg *Config) String() string { func (cfg *Config) ToTLS() (*common.TLS, error) { hostPort := net.JoinHostPort(cfg.TiDB.Host, strconv.Itoa(cfg.TiDB.StatusPort)) - return common.NewTLS(cfg.Security.CAPath, cfg.Security.CertPath, cfg.Security.KeyPath, hostPort) + if err := cfg.Security.LoadTLSContent(); err != nil { + return nil, err + } + return common.NewTLS( + cfg.Security.CAPath, + cfg.Security.CertPath, + cfg.Security.KeyPath, + hostPort, + cfg.Security.CABytes, + cfg.Security.CertBytes, + cfg.Security.KeyBytes, + ) } type Lightning struct { @@ -559,6 +572,30 @@ type Security struct { // TLSConfigName is used to set tls config for lightning in DM, so we don't expose this field to user // DM may running many lightning instances at same time, so we need to set different tls config name for each lightning TLSConfigName string `toml:"-" json:"-"` + + // When DM/engine uses lightning as a library, it can directly pass in the content + CABytes []byte `toml:"-" json:"-"` + CertBytes []byte `toml:"-" json:"-"` + KeyBytes []byte `toml:"-" json:"-"` +} + +// LoadTLSContent loads the file content from CA/Cert/Key. This function should be called every time before using the content, +// to support certificate rotation. +func (sec *Security) LoadTLSContent() error { + var err error + load := func(path string, target *[]byte) { + if err != nil { + return + } + if path != "" { + *target, err = os.ReadFile(path) + } + } + + load(sec.CAPath, &sec.CABytes) + load(sec.CertPath, &sec.CertBytes) + load(sec.KeyPath, &sec.KeyBytes) + return err } // RegisterMySQL registers the TLS config with name "cluster" or security.TLSConfigName @@ -567,7 +604,10 @@ func (sec *Security) RegisterMySQL() error { if sec == nil { return nil } - tlsConfig, err := common.ToTLSConfig(sec.CAPath, sec.CertPath, sec.KeyPath) + if err := sec.LoadTLSContent(); err != nil { + return err + } + tlsConfig, err := util.NewTLSConfigWithVerifyCN(sec.CABytes, sec.CertBytes, sec.KeyBytes, nil) if err != nil { return errors.Trace(err) } @@ -1153,7 +1193,7 @@ func (cfg *Config) CheckAndAdjustSecurity() error { case "": if len(cfg.TiDB.Security.CAPath) > 0 { if cfg.TiDB.Security.TLSConfigName == "" { - cfg.TiDB.Security.TLSConfigName = "cluster" // adjust this the default value + cfg.TiDB.Security.TLSConfigName = uuid.NewString() // adjust this the default value } cfg.TiDB.TLS = cfg.TiDB.Security.TLSConfigName } else { diff --git a/br/pkg/lightning/lightning.go b/br/pkg/lightning/lightning.go index d7b964d44dfad..b1954b81232d4 100644 --- a/br/pkg/lightning/lightning.go +++ b/br/pkg/lightning/lightning.go @@ -96,7 +96,18 @@ func New(globalCfg *config.GlobalConfig) *Lightning { os.Exit(1) } - tls, err := common.NewTLS(globalCfg.Security.CAPath, globalCfg.Security.CertPath, globalCfg.Security.KeyPath, globalCfg.App.StatusAddr) + if err := globalCfg.Security.LoadTLSContent(); err != nil { + log.L().Fatal("failed to load TLS certificates", zap.Error(err)) + } + tls, err := common.NewTLS( + globalCfg.Security.CAPath, + globalCfg.Security.CertPath, + globalCfg.Security.KeyPath, + globalCfg.App.StatusAddr, + globalCfg.Security.CABytes, + globalCfg.Security.CertBytes, + globalCfg.Security.KeyBytes, + ) if err != nil { log.L().Fatal("failed to load TLS certificates", zap.Error(err)) } diff --git a/dumpling/export/config.go b/dumpling/export/config.go index a0e50f067299d..dc5593d4d39f2 100644 --- a/dumpling/export/config.go +++ b/dumpling/export/config.go @@ -4,11 +4,10 @@ package export import ( "context" - "crypto/tls" "encoding/json" "fmt" - "io/ioutil" "net" + "os" "strconv" "strings" "text/template" @@ -211,8 +210,8 @@ func (conf *Config) GetDSN(db string) string { hostPort := net.JoinHostPort(conf.Host, strconv.Itoa(conf.Port)) dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?collation=utf8mb4_general_ci&readTimeout=%s&writeTimeout=30s&interpolateParams=true&maxAllowedPacket=0", conf.User, conf.Password, hostPort, db, conf.ReadTimeout) - if len(conf.Security.CAPath) > 0 { - dsn += "&tls=dumpling-tls-target" + if conf.Security.DriveTLSName != "" { + dsn += "&tls=" + conf.Security.DriveTLSName } if conf.AllowCleartextPasswords { dsn += "&allowCleartextPasswords=1" @@ -637,44 +636,39 @@ func adjustConfig(conf *Config, fns ...func(*Config) error) error { } func registerTLSConfig(conf *Config) error { + if len(conf.Security.CAPath) == 0 && len(conf.Security.CertPath) == 0 && len(conf.Security.KeyPath) == 0 { + return nil + } + + var err error if len(conf.Security.CAPath) > 0 { - var err error - var tlsConfig *tls.Config - if len(conf.Security.SSLCABytes) == 0 { - conf.Security.SSLCABytes, err = ioutil.ReadFile(conf.Security.CAPath) - if err != nil { - return errors.Trace(err) - } - if len(conf.Security.CertPath) > 0 { - conf.Security.SSLCertBytes, err = ioutil.ReadFile(conf.Security.CertPath) - if err != nil { - return errors.Trace(err) - } - } - if len(conf.Security.KeyPath) > 0 { - conf.Security.SSLKEYBytes, err = ioutil.ReadFile(conf.Security.KeyPath) - if err != nil { - return errors.Trace(err) - } - } - } - tlsConfig, err = util.NewTLSConfigWithVerifyCN(conf.Security.SSLCABytes, - conf.Security.SSLCertBytes, conf.Security.SSLKEYBytes, []string{}) + conf.Security.SSLCABytes, err = os.ReadFile(conf.Security.CAPath) if err != nil { return errors.Trace(err) } - // NOTE for local test(use a self-signed or invalid certificate), we don't need to check CA file. - // see more here https://github.com/go-sql-driver/mysql#tls - if conf.Host == "127.0.0.1" || len(conf.Security.SSLCertBytes) == 0 || len(conf.Security.SSLKEYBytes) == 0 { - tlsConfig.InsecureSkipVerify = true + } + if len(conf.Security.CertPath) > 0 { + conf.Security.SSLCertBytes, err = os.ReadFile(conf.Security.CertPath) + if err != nil { + return errors.Trace(err) } - conf.Security.DriveTLSName = "dumpling" + uuid.NewString() - err = mysql.RegisterTLSConfig(conf.Security.DriveTLSName, tlsConfig) + } + if len(conf.Security.KeyPath) > 0 { + conf.Security.SSLKEYBytes, err = os.ReadFile(conf.Security.KeyPath) if err != nil { return errors.Trace(err) } } - return nil + + tlsConfig, err2 := util.NewTLSConfigWithVerifyCN(conf.Security.SSLCABytes, + conf.Security.SSLCertBytes, conf.Security.SSLKEYBytes, []string{}) + if err2 != nil { + return errors.Trace(err2) + } + + conf.Security.DriveTLSName = "dumpling" + uuid.NewString() + err = mysql.RegisterTLSConfig(conf.Security.DriveTLSName, tlsConfig) + return errors.Trace(err) } func validateSpecifiedSQL(conf *Config) error { diff --git a/util/security.go b/util/security.go index e59d8129d7358..75a287992f747 100644 --- a/util/security.go +++ b/util/security.go @@ -131,9 +131,10 @@ func NewTLSConfigWithVerifyCN(caData, certData, keyData []byte, verifyCN []strin /* #nosec G402 */ tlsCfg := &tls.Config{ - MinVersion: tls.VersionTLS10, - Certificates: certificates, - NextProtos: []string{"h2", "http/1.2"}, // specify `h2` to let Go use HTTP/2. + MinVersion: tls.VersionTLS10, + Certificates: certificates, + InsecureSkipVerify: true, + NextProtos: []string{"h2", "http/1.2"}, // specify `h2` to let Go use HTTP/2. } if len(caData) != 0 { From 17a20d7e21b9463b9542edd7f59b79d374f4e9a6 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Mon, 5 Sep 2022 16:09:35 +0800 Subject: [PATCH 07/14] fix for lightning Signed-off-by: lance6716 --- br/pkg/lightning/config/config.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/br/pkg/lightning/config/config.go b/br/pkg/lightning/config/config.go index f16bfaeacc1bb..a480710764568 100644 --- a/br/pkg/lightning/config/config.go +++ b/br/pkg/lightning/config/config.go @@ -607,6 +607,10 @@ func (sec *Security) RegisterMySQL() error { if err := sec.LoadTLSContent(); err != nil { return err } + if len(sec.CABytes) == 0 && len(sec.CertBytes) == 0 && len(sec.KeyBytes) == 0 { + return nil + } + tlsConfig, err := util.NewTLSConfigWithVerifyCN(sec.CABytes, sec.CertBytes, sec.KeyBytes, nil) if err != nil { return errors.Trace(err) @@ -1191,7 +1195,9 @@ func (cfg *Config) CheckAndAdjustSecurity() error { switch cfg.TiDB.TLS { case "": - if len(cfg.TiDB.Security.CAPath) > 0 { + if len(cfg.TiDB.Security.CAPath) > 0 || len(cfg.TiDB.Security.CABytes) > 0 || + len(cfg.TiDB.Security.CertPath) > 0 || len(cfg.TiDB.Security.CertBytes) > 0 || + len(cfg.TiDB.Security.KeyPath) > 0 || len(cfg.TiDB.Security.KeyBytes) > 0 { if cfg.TiDB.Security.TLSConfigName == "" { cfg.TiDB.Security.TLSConfigName = uuid.NewString() // adjust this the default value } From fa86991b00bb847f4902ad2776304c146e15ebb6 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Mon, 5 Sep 2022 17:00:17 +0800 Subject: [PATCH 08/14] fix UT Signed-off-by: lance6716 --- br/pkg/lightning/config/config_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/br/pkg/lightning/config/config_test.go b/br/pkg/lightning/config/config_test.go index 845b325e7fcfd..18ccaeb2c6326 100644 --- a/br/pkg/lightning/config/config_test.go +++ b/br/pkg/lightning/config/config_test.go @@ -279,6 +279,7 @@ func TestAdjustWillBatchImportRatioInvalid(t *testing.T) { } func TestAdjustSecuritySection(t *testing.T) { + uuidHolder := "" testCases := []struct { input string expectedCA string @@ -302,7 +303,7 @@ func TestAdjustSecuritySection(t *testing.T) { ca-path = "/path/to/ca.pem" `, expectedCA: "/path/to/ca.pem", - expectedTLS: "cluster", + expectedTLS: uuidHolder, }, { input: ` @@ -321,7 +322,7 @@ func TestAdjustSecuritySection(t *testing.T) { ca-path = "/path/to/ca2.pem" `, expectedCA: "/path/to/ca2.pem", - expectedTLS: "cluster", + expectedTLS: uuidHolder, }, { input: ` @@ -330,7 +331,7 @@ func TestAdjustSecuritySection(t *testing.T) { ca-path = "/path/to/ca2.pem" `, expectedCA: "/path/to/ca2.pem", - expectedTLS: "cluster", + expectedTLS: uuidHolder, }, { input: ` @@ -356,7 +357,11 @@ func TestAdjustSecuritySection(t *testing.T) { err = cfg.Adjust(context.Background()) require.NoError(t, err, comment) require.Equal(t, tc.expectedCA, cfg.TiDB.Security.CAPath, comment) - require.Equal(t, tc.expectedTLS, cfg.TiDB.TLS, comment) + if tc.expectedTLS == uuidHolder { + require.NotEmpty(t, cfg.TiDB.TLS, comment) + } else { + require.Equal(t, tc.expectedTLS, cfg.TiDB.TLS, comment) + } } // test different tls config name cfg := config.NewConfig() From 2e94424779bf5652231562131b98f7e359c01926 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 6 Sep 2022 11:53:38 +0800 Subject: [PATCH 09/14] fix lightning cert rorate Signed-off-by: lance6716 --- br/pkg/lightning/common/security.go | 126 +++++++++++++--------------- br/pkg/lightning/config/config.go | 3 - br/pkg/lightning/lightning.go | 3 - 3 files changed, 58 insertions(+), 74 deletions(-) diff --git a/br/pkg/lightning/common/security.go b/br/pkg/lightning/common/security.go index aa3b387fdb513..a74be5af308bf 100644 --- a/br/pkg/lightning/common/security.go +++ b/br/pkg/lightning/common/security.go @@ -17,7 +17,6 @@ package common import ( "context" "crypto/tls" - "crypto/x509" "net" "net/http" "net/http/httptest" @@ -44,78 +43,30 @@ type TLS struct { url string } -// ToTLSConfig constructs a `*tls.Config` from the CA, certification and key -// paths. -// -// If the CA path is empty, returns nil. -func ToTLSConfig(caPath, certPath, keyPath string) (*tls.Config, error) { - if len(caPath) == 0 { - return nil, nil - } - - // Create a certificate pool from CA - certPool := x509.NewCertPool() - ca, err := os.ReadFile(caPath) - if err != nil { - return nil, errors.Annotate(err, "could not read ca certificate") - } - - // Append the certificates from the CA - if !certPool.AppendCertsFromPEM(ca) { - return nil, errors.New("failed to append ca certs") - } - - tlsConfig := &tls.Config{ - RootCAs: certPool, - NextProtos: []string{"h2", "http/1.1"}, // specify `h2` to let Go use HTTP/2. - MinVersion: tls.VersionTLS12, - } - - if len(certPath) != 0 && len(keyPath) != 0 { - loadCert := func() (*tls.Certificate, error) { - cert, err := tls.LoadX509KeyPair(certPath, keyPath) - if err != nil { - return nil, errors.Annotate(err, "could not load client key pair") - } - return &cert, nil - } - tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { - return loadCert() - } - tlsConfig.GetCertificate = func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - return loadCert() - } - } - return tlsConfig, nil -} - // NewTLS constructs a new HTTP client with TLS configured with the CA, // certificate and key paths. // // If the CA path is empty, returns an instance where TLS is disabled. func NewTLS(caPath, certPath, keyPath, host string, caBytes, certBytes, keyBytes []byte) (*TLS, error) { - if len(caBytes) == 0 && len(certBytes) == 0 && len(keyBytes) == 0 { - return &TLS{ - inner: nil, - client: &http.Client{}, - url: "http://" + host, - }, nil - } - inner, err := util.NewTLSConfigWithVerifyCN(caBytes, certBytes, keyBytes, nil) - if err != nil { - return nil, errors.Trace(err) - } - return &TLS{ + ret := &TLS{ caPath: caPath, certPath: certPath, keyPath: keyPath, caBytes: caBytes, certBytes: certBytes, keyBytes: keyBytes, - inner: inner, - client: httputil.NewClient(inner), - url: "https://" + host, - }, nil + client: &http.Client{}, + } + if err := ret.buildInnerTLSFields(); err != nil { + return nil, err + } + + if ret.inner != nil { + ret.url = "https://" + host + } else { + ret.url = "http://" + host + } + return ret, nil } // NewTLSFromMockServer constructs a new TLS instance from the certificates of @@ -128,6 +79,40 @@ func NewTLSFromMockServer(server *httptest.Server) *TLS { } } +func (tc *TLS) loadFileContent() error { + var err error + load := func(path string, target *[]byte) { + if err != nil { + return + } + if path != "" { + *target, err = os.ReadFile(path) + } + } + + load(tc.caPath, &tc.caBytes) + load(tc.certPath, &tc.certBytes) + load(tc.keyPath, &tc.keyBytes) + return err +} + +func (tc *TLS) buildInnerTLSFields() error { + if err := tc.loadFileContent(); err != nil { + return errors.Trace(err) + } + if len(tc.caBytes) == 0 && len(tc.certBytes) == 0 && len(tc.keyBytes) == 0 { + return nil + } + + inner, err := util.NewTLSConfigWithVerifyCN(tc.caBytes, tc.certBytes, tc.keyBytes, nil) + if err != nil { + return errors.Trace(err) + } + tc.inner = inner + tc.client = httputil.NewClient(inner) + return nil +} + // WithHost creates a new TLS instance with the host replaced. func (tc *TLS) WithHost(host string) *TLS { var url string @@ -136,15 +121,14 @@ func (tc *TLS) WithHost(host string) *TLS { } else { url = "http://" + host } - return &TLS{ - inner: tc.inner, - client: tc.client, - url: url, - } + shallowClone := *tc + shallowClone.url = url + return &shallowClone } // ToGRPCDialOption constructs a gRPC dial option. func (tc *TLS) ToGRPCDialOption() grpc.DialOption { + _ = tc.buildInnerTLSFields() if tc.inner != nil { return grpc.WithTransportCredentials(credentials.NewTLS(tc.inner)) } @@ -153,6 +137,7 @@ func (tc *TLS) ToGRPCDialOption() grpc.DialOption { // WrapListener places a TLS layer on top of the existing listener. func (tc *TLS) WrapListener(l net.Listener) net.Listener { + _ = tc.buildInnerTLSFields() if tc.inner == nil { return l } @@ -160,10 +145,13 @@ func (tc *TLS) WrapListener(l net.Listener) net.Listener { } func (tc *TLS) GetJSON(ctx context.Context, path string, v interface{}) error { + _ = tc.buildInnerTLSFields() return GetJSON(ctx, tc.client, tc.url+path, v) } +// ToPDSecurityOption converts the TLS configuration to a PD security option. func (tc *TLS) ToPDSecurityOption() pd.SecurityOption { + _ = tc.loadFileContent() return pd.SecurityOption{ CAPath: tc.caPath, CertPath: tc.certPath, @@ -174,7 +162,8 @@ func (tc *TLS) ToPDSecurityOption() pd.SecurityOption { } } -// TODO: TiKV does not support pass in content +// ToTiKVSecurityConfig converts the TLS configuration to a TiKV security config. +// TODO: TiKV does not support pass in content. func (tc *TLS) ToTiKVSecurityConfig() config.Security { return config.Security{ ClusterSSLCA: tc.caPath, @@ -185,5 +174,6 @@ func (tc *TLS) ToTiKVSecurityConfig() config.Security { } func (tc *TLS) TLSConfig() *tls.Config { + _ = tc.buildInnerTLSFields() return tc.inner } diff --git a/br/pkg/lightning/config/config.go b/br/pkg/lightning/config/config.go index a480710764568..30c421192a828 100644 --- a/br/pkg/lightning/config/config.go +++ b/br/pkg/lightning/config/config.go @@ -157,9 +157,6 @@ func (cfg *Config) String() string { func (cfg *Config) ToTLS() (*common.TLS, error) { hostPort := net.JoinHostPort(cfg.TiDB.Host, strconv.Itoa(cfg.TiDB.StatusPort)) - if err := cfg.Security.LoadTLSContent(); err != nil { - return nil, err - } return common.NewTLS( cfg.Security.CAPath, cfg.Security.CertPath, diff --git a/br/pkg/lightning/lightning.go b/br/pkg/lightning/lightning.go index b1954b81232d4..3770f7c8f07a4 100644 --- a/br/pkg/lightning/lightning.go +++ b/br/pkg/lightning/lightning.go @@ -96,9 +96,6 @@ func New(globalCfg *config.GlobalConfig) *Lightning { os.Exit(1) } - if err := globalCfg.Security.LoadTLSContent(); err != nil { - log.L().Fatal("failed to load TLS certificates", zap.Error(err)) - } tls, err := common.NewTLS( globalCfg.Security.CAPath, globalCfg.Security.CertPath, From 8e86b021637128057275faff57a76a13c1329e70 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Tue, 6 Sep 2022 20:59:55 +0800 Subject: [PATCH 10/14] change API Signed-off-by: lance6716 --- br/pkg/lightning/common/security.go | 76 +++----- br/pkg/lightning/config/config.go | 32 +-- dumpling/export/config.go | 39 ++-- go.mod | 2 +- util/security.go | 290 +++++++++++++++++----------- util/security_test.go | 103 +++++----- 6 files changed, 269 insertions(+), 273 deletions(-) diff --git a/br/pkg/lightning/common/security.go b/br/pkg/lightning/common/security.go index a74be5af308bf..c7c25e2d9f046 100644 --- a/br/pkg/lightning/common/security.go +++ b/br/pkg/lightning/common/security.go @@ -20,7 +20,6 @@ import ( "net" "net/http" "net/http/httptest" - "os" "github.com/pingcap/errors" "github.com/pingcap/tidb/br/pkg/httputil" @@ -48,25 +47,35 @@ type TLS struct { // // If the CA path is empty, returns an instance where TLS is disabled. func NewTLS(caPath, certPath, keyPath, host string, caBytes, certBytes, keyBytes []byte) (*TLS, error) { - ret := &TLS{ + inner, err := util.NewTLSConfig( + util.WithCAPath(caPath), + util.WithCertAndKeyPath(certPath, keyPath), + util.WithCAContent(caBytes), + util.WithCertAndKeyContent(certBytes, keyBytes), + ) + if err != nil { + return nil, errors.Trace(err) + } + + if inner == nil { + return &TLS{ + inner: nil, + client: &http.Client{}, + url: "http://" + host, + }, nil + } + + return &TLS{ caPath: caPath, certPath: certPath, keyPath: keyPath, caBytes: caBytes, certBytes: certBytes, keyBytes: keyBytes, - client: &http.Client{}, - } - if err := ret.buildInnerTLSFields(); err != nil { - return nil, err - } - - if ret.inner != nil { - ret.url = "https://" + host - } else { - ret.url = "http://" + host - } - return ret, nil + inner: inner, + client: httputil.NewClient(inner), + url: "https://" + host, + }, nil } // NewTLSFromMockServer constructs a new TLS instance from the certificates of @@ -79,40 +88,6 @@ func NewTLSFromMockServer(server *httptest.Server) *TLS { } } -func (tc *TLS) loadFileContent() error { - var err error - load := func(path string, target *[]byte) { - if err != nil { - return - } - if path != "" { - *target, err = os.ReadFile(path) - } - } - - load(tc.caPath, &tc.caBytes) - load(tc.certPath, &tc.certBytes) - load(tc.keyPath, &tc.keyBytes) - return err -} - -func (tc *TLS) buildInnerTLSFields() error { - if err := tc.loadFileContent(); err != nil { - return errors.Trace(err) - } - if len(tc.caBytes) == 0 && len(tc.certBytes) == 0 && len(tc.keyBytes) == 0 { - return nil - } - - inner, err := util.NewTLSConfigWithVerifyCN(tc.caBytes, tc.certBytes, tc.keyBytes, nil) - if err != nil { - return errors.Trace(err) - } - tc.inner = inner - tc.client = httputil.NewClient(inner) - return nil -} - // WithHost creates a new TLS instance with the host replaced. func (tc *TLS) WithHost(host string) *TLS { var url string @@ -128,7 +103,6 @@ func (tc *TLS) WithHost(host string) *TLS { // ToGRPCDialOption constructs a gRPC dial option. func (tc *TLS) ToGRPCDialOption() grpc.DialOption { - _ = tc.buildInnerTLSFields() if tc.inner != nil { return grpc.WithTransportCredentials(credentials.NewTLS(tc.inner)) } @@ -137,7 +111,6 @@ func (tc *TLS) ToGRPCDialOption() grpc.DialOption { // WrapListener places a TLS layer on top of the existing listener. func (tc *TLS) WrapListener(l net.Listener) net.Listener { - _ = tc.buildInnerTLSFields() if tc.inner == nil { return l } @@ -145,13 +118,11 @@ func (tc *TLS) WrapListener(l net.Listener) net.Listener { } func (tc *TLS) GetJSON(ctx context.Context, path string, v interface{}) error { - _ = tc.buildInnerTLSFields() return GetJSON(ctx, tc.client, tc.url+path, v) } // ToPDSecurityOption converts the TLS configuration to a PD security option. func (tc *TLS) ToPDSecurityOption() pd.SecurityOption { - _ = tc.loadFileContent() return pd.SecurityOption{ CAPath: tc.caPath, CertPath: tc.certPath, @@ -174,6 +145,5 @@ func (tc *TLS) ToTiKVSecurityConfig() config.Security { } func (tc *TLS) TLSConfig() *tls.Config { - _ = tc.buildInnerTLSFields() return tc.inner } diff --git a/br/pkg/lightning/config/config.go b/br/pkg/lightning/config/config.go index 30c421192a828..6e42878778d88 100644 --- a/br/pkg/lightning/config/config.go +++ b/br/pkg/lightning/config/config.go @@ -576,39 +576,19 @@ type Security struct { KeyBytes []byte `toml:"-" json:"-"` } -// LoadTLSContent loads the file content from CA/Cert/Key. This function should be called every time before using the content, -// to support certificate rotation. -func (sec *Security) LoadTLSContent() error { - var err error - load := func(path string, target *[]byte) { - if err != nil { - return - } - if path != "" { - *target, err = os.ReadFile(path) - } - } - - load(sec.CAPath, &sec.CABytes) - load(sec.CertPath, &sec.CertBytes) - load(sec.KeyPath, &sec.KeyBytes) - return err -} - // RegisterMySQL registers the TLS config with name "cluster" or security.TLSConfigName // for use in `sql.Open()`. This method is goroutine-safe. func (sec *Security) RegisterMySQL() error { if sec == nil { return nil } - if err := sec.LoadTLSContent(); err != nil { - return err - } - if len(sec.CABytes) == 0 && len(sec.CertBytes) == 0 && len(sec.KeyBytes) == 0 { - return nil - } - tlsConfig, err := util.NewTLSConfigWithVerifyCN(sec.CABytes, sec.CertBytes, sec.KeyBytes, nil) + tlsConfig, err := util.NewTLSConfig( + util.WithCAPath(sec.CAPath), + util.WithCertAndKeyPath(sec.CertPath, sec.KeyPath), + util.WithCAContent(sec.CABytes), + util.WithCertAndKeyContent(sec.CertBytes, sec.KeyBytes), + ) if err != nil { return errors.Trace(err) } diff --git a/dumpling/export/config.go b/dumpling/export/config.go index dc5593d4d39f2..980de0d8807f5 100644 --- a/dumpling/export/config.go +++ b/dumpling/export/config.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "net" - "os" "strconv" "strings" "text/template" @@ -109,7 +108,7 @@ type Config struct { KeyPath string SSLCABytes []byte `json:"-"` SSLCertBytes []byte `json:"-"` - SSLKEYBytes []byte `json:"-"` + SSLKeyBytes []byte `json:"-"` } LogLevel string @@ -636,34 +635,18 @@ func adjustConfig(conf *Config, fns ...func(*Config) error) error { } func registerTLSConfig(conf *Config) error { - if len(conf.Security.CAPath) == 0 && len(conf.Security.CertPath) == 0 && len(conf.Security.KeyPath) == 0 { - return nil - } - - var err error - if len(conf.Security.CAPath) > 0 { - conf.Security.SSLCABytes, err = os.ReadFile(conf.Security.CAPath) - if err != nil { - return errors.Trace(err) - } - } - if len(conf.Security.CertPath) > 0 { - conf.Security.SSLCertBytes, err = os.ReadFile(conf.Security.CertPath) - if err != nil { - return errors.Trace(err) - } - } - if len(conf.Security.KeyPath) > 0 { - conf.Security.SSLKEYBytes, err = os.ReadFile(conf.Security.KeyPath) - if err != nil { - return errors.Trace(err) - } + tlsConfig, err := util.NewTLSConfig( + util.WithCAPath(conf.Security.CAPath), + util.WithCertAndKeyPath(conf.Security.CertPath, conf.Security.KeyPath), + util.WithCAContent(conf.Security.SSLCABytes), + util.WithCertAndKeyContent(conf.Security.SSLCertBytes, conf.Security.SSLKeyBytes), + ) + if err != nil { + return errors.Trace(err) } - tlsConfig, err2 := util.NewTLSConfigWithVerifyCN(conf.Security.SSLCABytes, - conf.Security.SSLCertBytes, conf.Security.SSLKEYBytes, []string{}) - if err2 != nil { - return errors.Trace(err2) + if tlsConfig == nil { + return nil } conf.Security.DriveTLSName = "dumpling" + uuid.NewString() diff --git a/go.mod b/go.mod index 1929e002808c1..7e87032f90a78 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,7 @@ require ( github.com/pingcap/sysutil v0.0.0-20220114020952-ea68d2dbf5b4 github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e github.com/pingcap/tipb v0.0.0-20220824081009-0714a57aff1d + github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.37.0 @@ -190,7 +191,6 @@ require ( github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/procfs v0.8.0 // indirect diff --git a/util/security.go b/util/security.go index 75a287992f747..374c8666adbfb 100644 --- a/util/security.go +++ b/util/security.go @@ -20,12 +20,11 @@ import ( "io/ioutil" "net" "net/http" - "net/http/httptest" + "os" "strings" + "sync" "github.com/pingcap/errors" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) // TLS saves some information about tls @@ -107,84 +106,180 @@ func ToTLSConfigWithVerify(caPath, certPath, keyPath string, verifyCN []string) return tlsCfg, nil } -// NewTLSConfigWithVerifyCN constructs a `*tls.Config` from given data: -// - caData: represents the MySQL server's CA certificate. If not empty, the TLS connection only allows this CA. -// - certData, keyData: represents the client's certificate and key. If not empty, the TLS connection will use this -// certificate. Otherwise, it will use self-generated certificate. -// - verifyCN: represents the Common Name that the MySQL server's certificate is expected to have. If not empty, the TLS -// connection only allows the certificates with these CN. -func NewTLSConfigWithVerifyCN(caData, certData, keyData []byte, verifyCN []string) (*tls.Config, error) { - var ( - certificates []tls.Certificate - certPool *x509.CertPool - verifyFuncs []func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error - ) +type tlsConfigBuilder struct { + caPath, certPath, keyPath string + caContent, certContent, keyContent []byte + verifyCN []string +} - if len(certData) != 0 && len(keyData) != 0 { - // Generate a key pair from your pem-encoded cert and key ([]byte). - cert, err := tls.X509KeyPair(certData, keyData) - if err != nil { - return nil, errors.Wrap(err, "failed to generate cert") - } - certificates = []tls.Certificate{cert} +type TLSConfigOption func(*tlsConfigBuilder) + +// WithCAPath sets the CA path to build a tls.Config, and the peer should use the certificate which can be verified by +// this CA. It has higher priority than WithCAContent. +// empty `caPath` is no-op. +// WithCAPath also support rotation, which means if the CA file is changed, the new content will be used. +func WithCAPath(caPath string) TLSConfigOption { + return func(builder *tlsConfigBuilder) { + builder.caPath = caPath } +} + +// WithCertAndKeyPath sets the client certificate and primary key path to build a tls.Config. It has higher priority +// than WithCertAndKeyContent. +// empty `certPath`/`keyPath` is no-op. +// WithCertAndKeyPath also support rotation, which means if the client certificate or primary key file is changed, the +// new content will be used. +func WithCertAndKeyPath(certPath, keyPath string) TLSConfigOption { + return func(builder *tlsConfigBuilder) { + builder.certPath = certPath + builder.keyPath = keyPath + } +} + +// WithVerifyCommonName sets the Common Name the peer must provide before starting a TLS connection. +// empty `verifyCN` is no-op. +func WithVerifyCommonName(verifyCN []string) TLSConfigOption { + return func(builder *tlsConfigBuilder) { + builder.verifyCN = verifyCN + } +} + +// WithCAContent sets the CA content to build a tls.Config, and the peer should use the certificate which can be +// verified by this CA. It has lower priority than WithCAPath. +// empty `caContent` is no-op. +func WithCAContent(caContent []byte) TLSConfigOption { + return func(builder *tlsConfigBuilder) { + builder.caContent = caContent + } +} + +// WithCertAndKeyContent sets the client certificate and primary key content to build a tls.Config. It has lower +// priority than WithCertAndKeyPath. +// empty `certContent`/`keyContent` is no-op. +func WithCertAndKeyContent(certContent, keyContent []byte) TLSConfigOption { + return func(builder *tlsConfigBuilder) { + builder.certContent = certContent + builder.keyContent = keyContent + } +} + +// NewTLSConfig creates a tls.Config from the given options. If no certificate is provided, it will return (nil, nil). +func NewTLSConfig(opts ...TLSConfigOption) (*tls.Config, error) { + builder := &tlsConfigBuilder{} + for _, opt := range opts { + opt(builder) + } + + if builder.caPath == "" && len(builder.caContent) == 0 && + builder.certPath == "" && len(builder.certContent) == 0 && + builder.keyPath == "" && len(builder.keyContent) == 0 { + return nil, nil + } + + var ( + certPool *x509.CertPool + certPoolMu sync.RWMutex + verifyFuncs []func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error + ) /* #nosec G402 */ tlsCfg := &tls.Config{ MinVersion: tls.VersionTLS10, - Certificates: certificates, InsecureSkipVerify: true, NextProtos: []string{"h2", "http/1.2"}, // specify `h2` to let Go use HTTP/2. } - if len(caData) != 0 { - certPool = x509.NewCertPool() - if !certPool.AppendCertsFromPEM(caData) { - return nil, errors.New("failed to append ca certs") - } - tlsCfg.RootCAs = certPool - tlsCfg.ClientCAs = certPool + // 1. handle client certificates - // check the received MySQL server certificate during TLS handshake - // thanks to https://cloud.google.com/sql/docs/mysql/samples/cloud-sql-mysql-databasesql-sslcerts - verifyFuncs = append(verifyFuncs, func(rawCerts [][]byte, _ [][]*x509.Certificate) error { - if len(rawCerts) == 0 { - return errors.New("no certificates available to verify") + if builder.certPath != "" && builder.keyPath != "" { + // clear the content if path is provided + builder.certContent = nil + builder.keyContent = nil + loadCert := func() (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(builder.certPath, builder.keyPath) + if err != nil { + return nil, errors.Annotate(err, "could not load client key pair") } + return &cert, nil + } + tlsCfg.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return loadCert() + } + tlsCfg.GetCertificate = func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + return loadCert() + } + } + if len(builder.certContent) > 0 && len(builder.keyContent) > 0 { + cert, err := tls.X509KeyPair(builder.certContent, builder.keyContent) + if err != nil { + return nil, errors.Annotate(err, "could not load client key pair") + } + tlsCfg.Certificates = []tls.Certificate{cert} + } + + // 2. handle CA + + // need this closure to access certPool + // thanks to https://cloud.google.com/sql/docs/mysql/samples/cloud-sql-mysql-databasesql-sslcerts + verifyCA := func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + if len(rawCerts) == 0 { + return errors.New("no certificates available to verify") + } + + cert, err := x509.ParseCertificate(rawCerts[0]) + if err != nil { + return err + } - cert, err := x509.ParseCertificate(rawCerts[0]) + certPoolMu.RLock() + defer certPoolMu.RUnlock() + if _, err = cert.Verify(x509.VerifyOptions{Roots: certPool}); err != nil { + return errors.Wrap(err, "can't verify certificate, maybe different CA is used") + } + return nil + } + + if builder.caPath != "" { + // clear the content if path is provided + builder.caContent = nil + loadCA := func() (*tls.Config, error) { + certPoolMu.Lock() + defer certPoolMu.Unlock() + + certPool = x509.NewCertPool() + content, err := os.ReadFile(builder.caPath) if err != nil { - return err + return nil, errors.Annotate(err, "could not read ca certificate") } - - opts := x509.VerifyOptions{Roots: certPool} - if _, err = cert.Verify(opts); err != nil { - return errors.Wrap(err, "can't verify certificate, maybe different CA is used") + if !certPool.AppendCertsFromPEM(content) { + return nil, errors.New("failed to append ca certs in reading ca certs") } - return nil - }) + tlsCfg.RootCAs = certPool + tlsCfg.ClientCAs = certPool + return tlsCfg, nil + } + tlsCfg.GetConfigForClient = func(info *tls.ClientHelloInfo) (*tls.Config, error) { + return loadCA() + } + verifyFuncs = append(verifyFuncs, verifyCA) } + if len(builder.caContent) > 0 { + certPool = x509.NewCertPool() - if len(verifyCN) != 0 { - checkCN := make(map[string]struct{}) - for _, cn := range verifyCN { - cn = strings.TrimSpace(cn) - checkCN[cn] = struct{}{} + if !certPool.AppendCertsFromPEM(builder.caContent) { + return nil, errors.New("failed to append ca certs") } + tlsCfg.RootCAs = certPool + tlsCfg.ClientCAs = certPool + + verifyFuncs = append(verifyFuncs, verifyCA) + } + + // 3. handle verify Common Name + + if len(builder.verifyCN) > 0 { tlsCfg.ClientAuth = tls.RequireAndVerifyClientCert - // check the received CommonName of MySQL server certificate's verify chain during TLS handshake - verifyFuncs = append(verifyFuncs, func(_ [][]byte, verifiedChains [][]*x509.Certificate) error { - cns := make([]string, 0, len(verifiedChains)) - for _, chains := range verifiedChains { - for _, chain := range chains { - cns = append(cns, chain.Subject.CommonName) - if _, match := checkCN[chain.Subject.CommonName]; match { - return nil - } - } - } - return errors.Errorf("client certificate authentication failed. The Common Name from the client certificate %v was not found in the configuration cluster-verify-cn with value: %s", cns, verifyCN) - }) + verifyFuncs = append(verifyFuncs, verifyCommonName(builder.verifyCN)) } tlsCfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { @@ -198,6 +293,27 @@ func NewTLSConfigWithVerifyCN(caData, certData, keyData []byte, verifyCN []strin return tlsCfg, nil } +func verifyCommonName(verifyCN []string) func([][]byte, [][]*x509.Certificate) error { + checkCN := make(map[string]struct{}) + for _, cn := range verifyCN { + cn = strings.TrimSpace(cn) + checkCN[cn] = struct{}{} + } + + return func(_ [][]byte, verifiedChains [][]*x509.Certificate) error { + cns := make([]string, 0, len(verifiedChains)) + for _, chains := range verifiedChains { + for _, chain := range chains { + cns = append(cns, chain.Subject.CommonName) + if _, match := checkCN[chain.Subject.CommonName]; match { + return nil + } + } + } + return errors.Errorf("client certificate authentication failed. The Common Name from the client certificate %v was not found in the configuration cluster-verify-cn with value: %s", cns, verifyCN) + } +} + // NewTLS constructs a new HTTP client with TLS configured with the CA, // certificate and key paths. // @@ -231,53 +347,6 @@ func ClientWithTLS(tlsCfg *tls.Config) *http.Client { return &http.Client{Transport: transport} } -// NewTLSFromMockServer constructs a new TLS instance from the certificates of -// an *httptest.Server. -func NewTLSFromMockServer(server *httptest.Server) *TLS { - return &TLS{ - inner: server.TLS, - client: server.Client(), - url: server.URL, - } -} - -// WithHost creates a new TLS instance with the host replaced. -func (tc *TLS) WithHost(host string) *TLS { - var url string - if tc.inner != nil { - url = "https://" + host - } else { - url = "http://" + host - } - return &TLS{ - inner: tc.inner, - client: tc.client, - url: url, - } -} - -// TLSConfig returns tls config -func (tc *TLS) TLSConfig() *tls.Config { - return tc.inner -} - -// ToGRPCDialOption constructs a gRPC dial option. -func (tc *TLS) ToGRPCDialOption() grpc.DialOption { - if tc.inner != nil { - return grpc.WithTransportCredentials(credentials.NewTLS(tc.inner)) - } - return grpc.WithInsecure() -} - -// ToGRPCServerOption constructs a gRPC server option. -func (tc *TLS) ToGRPCServerOption() grpc.ServerOption { - if tc.inner != nil { - return grpc.Creds(credentials.NewTLS(tc.inner)) - } - - return grpc.Creds(nil) -} - // WrapListener places a TLS layer on top of the existing listener. func (tc *TLS) WrapListener(l net.Listener) net.Listener { if tc.inner == nil { @@ -285,8 +354,3 @@ func (tc *TLS) WrapListener(l net.Listener) net.Listener { } return tls.NewListener(l, tc.inner) } - -// GetJSON obtains JSON result with the HTTP GET method. -func (tc *TLS) GetJSON(path string, v interface{}) error { - return GetJSON(tc.client, tc.url+path, v) -} diff --git a/util/security_test.go b/util/security_test.go index 29a6edec459a6..98473f1f123a6 100644 --- a/util/security_test.go +++ b/util/security_test.go @@ -28,8 +28,6 @@ import ( "math/big" "net" "net/http" - "net/http/httptest" - "net/url" "os" "path/filepath" "testing" @@ -39,46 +37,6 @@ import ( "github.com/stretchr/testify/require" ) -func respondPathHandler(w http.ResponseWriter, req *http.Request) { - io.WriteString(w, `{"path":"`) - io.WriteString(w, req.URL.Path) - io.WriteString(w, `"}`) -} - -func TestGetJSONInsecure(t *testing.T) { - mockServer := httptest.NewServer(http.HandlerFunc(respondPathHandler)) - defer mockServer.Close() - - u, err := url.Parse(mockServer.URL) - require.NoError(t, err) - - tls, err := util.NewTLS("", "", "", u.Host, nil) - require.NoError(t, err) - - var result struct{ Path string } - err = tls.GetJSON("/aaa", &result) - require.NoError(t, err) - require.Equal(t, "/aaa", result.Path) - err = tls.GetJSON("/bbbb", &result) - require.NoError(t, err) - require.Equal(t, "/bbbb", result.Path) -} - -func TestGetJSONSecure(t *testing.T) { - mockServer := httptest.NewTLSServer(http.HandlerFunc(respondPathHandler)) - defer mockServer.Close() - - tls := util.NewTLSFromMockServer(mockServer) - - var result struct{ Path string } - err := tls.GetJSON("/ccc", &result) - require.NoError(t, err) - require.Equal(t, "/ccc", result.Path) - err = tls.GetJSON("/dddd", &result) - require.NoError(t, err) - require.Equal(t, "/dddd", result.Path) -} - func TestInvalidTLS(t *testing.T) { tempDir := t.TempDir() @@ -104,14 +62,18 @@ func TestInvalidTLS(t *testing.T) { require.Regexp(t, "could not load client key pair: tls.*", err.Error()) } -func TestVerifyCN(t *testing.T) { +func TestVerifyCommonNameAndRotate(t *testing.T) { caData, certs, keys := generateCerts(t, []string{"server", "client1", "client2"}) serverCert, serverKey := certs[0], keys[0] client1Cert, client1Key := certs[1], keys[1] client2Cert, client2Key := certs[2], keys[2] // only allow client1 to visit - serverTLS, err := util.NewTLSConfigWithVerifyCN(caData, serverCert, serverKey, []string{"client1"}) + serverTLS, err := util.NewTLSConfig( + util.WithCAContent(caData), + util.WithCertAndKeyContent(serverCert, serverKey), + util.WithVerifyCommonName([]string{"client1"}), + ) require.NoError(t, err) port := 9292 url := fmt.Sprintf("https://127.0.0.1:%d", port) @@ -122,7 +84,10 @@ func TestVerifyCN(t *testing.T) { server.Close() }() - clientTLS1, err := util.NewTLSConfigWithVerifyCN(caData, client1Cert, client1Key, []string{}) + clientTLS1, err := util.NewTLSConfig( + util.WithCAContent(caData), + util.WithCertAndKeyContent(client1Cert, client1Key), + ) require.NoError(t, err) resp, err := util.ClientWithTLS(clientTLS1).Get(url) require.NoError(t, err) @@ -132,23 +97,51 @@ func TestVerifyCN(t *testing.T) { require.NoError(t, resp.Body.Close()) // client2 can't visit server - clientTLS2, err := util.NewTLSConfigWithVerifyCN(caData, client2Cert, client2Key, []string{}) + dir := t.TempDir() + certPath := filepath.Join(dir, "client.pem") + keyPath := filepath.Join(dir, "client.key") + err = os.WriteFile(certPath, client2Cert, 0600) require.NoError(t, err) - resp, err = util.ClientWithTLS(clientTLS2).Get(url) + err = os.WriteFile(keyPath, client2Key, 0600) + require.NoError(t, err) + + clientTLS2, err := util.NewTLSConfig( + util.WithCAContent(caData), + util.WithCertAndKeyPath(certPath, keyPath), + ) + require.NoError(t, err) + client2 := util.ClientWithTLS(clientTLS2) + resp, err = client2.Get(url) require.ErrorContains(t, err, "tls: bad certificate") if resp != nil { require.NoError(t, resp.Body.Close()) } + + // test certificate rotation + err = os.WriteFile(certPath, client1Cert, 0600) + require.NoError(t, err) + err = os.WriteFile(keyPath, client1Key, 0600) + require.NoError(t, err) + + resp, err = client2.Get(url) + require.NoError(t, err) + body, err = io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, "This an example server", string(body)) + require.NoError(t, resp.Body.Close()) } -func TestWithOrWithoutCA(t *testing.T) { +func TestCA(t *testing.T) { caData, certs, keys := generateCerts(t, []string{"server", "client"}) serverCert, serverKey := certs[0], keys[0] clientCert, clientKey := certs[1], keys[1] caData2, _, _ := generateCerts(t, nil) - serverTLS, err := util.NewTLSConfigWithVerifyCN(caData, serverCert, serverKey, nil) + serverTLS, err := util.NewTLSConfig( + util.WithCAContent(caData), + util.WithCertAndKeyContent(serverCert, serverKey), + ) require.NoError(t, err) port := 9293 url := fmt.Sprintf("https://127.0.0.1:%d", port) @@ -160,7 +153,9 @@ func TestWithOrWithoutCA(t *testing.T) { }() // test only CA - clientTLS1, err := util.NewTLSConfigWithVerifyCN(caData, nil, nil, nil) + clientTLS1, err := util.NewTLSConfig( + util.WithCAContent(caData), + ) require.NoError(t, err) resp, err := util.ClientWithTLS(clientTLS1).Get(url) require.NoError(t, err) @@ -170,7 +165,9 @@ func TestWithOrWithoutCA(t *testing.T) { require.NoError(t, resp.Body.Close()) // test without CA - clientTLS2, err := util.NewTLSConfigWithVerifyCN(nil, clientCert, clientKey, nil) + clientTLS2, err := util.NewTLSConfig( + util.WithCertAndKeyContent(clientCert, clientKey), + ) require.NoError(t, err) // inject CA to imitate our generated CA is a trusted CA clientTLS2.RootCAs = clientTLS1.RootCAs @@ -182,7 +179,9 @@ func TestWithOrWithoutCA(t *testing.T) { require.NoError(t, resp.Body.Close()) // test wrong CA should fail - clientTLS3, err := util.NewTLSConfigWithVerifyCN(caData2, nil, nil, []string{}) + clientTLS3, err := util.NewTLSConfig( + util.WithCAContent(caData2), + ) require.NoError(t, err) // inject CA to imitate our generated CA is a trusted CA certPool := x509.NewCertPool() From 1cf8129b5b4176f09dd9d173483b87cfe2b1f934 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 7 Sep 2022 10:11:28 +0800 Subject: [PATCH 11/14] fix CI Signed-off-by: lance6716 --- br/pkg/lightning/common/security.go | 2 -- br/pkg/lightning/common/security_test.go | 2 +- util/security.go | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/br/pkg/lightning/common/security.go b/br/pkg/lightning/common/security.go index c7c25e2d9f046..a48abc48c2c54 100644 --- a/br/pkg/lightning/common/security.go +++ b/br/pkg/lightning/common/security.go @@ -44,8 +44,6 @@ type TLS struct { // NewTLS constructs a new HTTP client with TLS configured with the CA, // certificate and key paths. -// -// If the CA path is empty, returns an instance where TLS is disabled. func NewTLS(caPath, certPath, keyPath, host string, caBytes, certBytes, keyBytes []byte) (*TLS, error) { inner, err := util.NewTLSConfig( util.WithCAPath(caPath), diff --git a/br/pkg/lightning/common/security_test.go b/br/pkg/lightning/common/security_test.go index ec3e04a8fd8ca..e34ef3622500c 100644 --- a/br/pkg/lightning/common/security_test.go +++ b/br/pkg/lightning/common/security_test.go @@ -96,6 +96,6 @@ RtgQTNI= keyContent := []byte("invalid key content") err = os.WriteFile(keyPath, keyContent, 0o600) require.NoError(t, err) - _, err = common.NewTLS(caPath, certPath, keyPath, "localhost", caContent, certContent, keyContent) + _, err = common.NewTLS(caPath, "", "", "localhost", caContent, certContent, keyContent) require.ErrorContains(t, err, "tls: failed to find any PEM data in certificate input") } diff --git a/util/security.go b/util/security.go index 374c8666adbfb..06aac93fe71e8 100644 --- a/util/security.go +++ b/util/security.go @@ -112,6 +112,7 @@ type tlsConfigBuilder struct { verifyCN []string } +// TLSConfigOption is used to build a tls.Config in NewTLSConfig. type TLSConfigOption func(*tlsConfigBuilder) // WithCAPath sets the CA path to build a tls.Config, and the peer should use the certificate which can be verified by From 65210a105395b6ee7284c17f6b54094525d3f889 Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 7 Sep 2022 11:49:12 +0800 Subject: [PATCH 12/14] fix test Signed-off-by: lance6716 --- br/tests/_utils/run_services | 2 +- util/security.go | 36 ++++++++++++------------------------ 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/br/tests/_utils/run_services b/br/tests/_utils/run_services index a7449cb229bf2..c3063482b6e95 100644 --- a/br/tests/_utils/run_services +++ b/br/tests/_utils/run_services @@ -192,7 +192,7 @@ start_services_impl() { TIDB_CONFIG="tests/config/tidb.toml" TIKV_CONFIG="tests/config/tikv.toml" PD_CONFIG="tests/config/pd.toml" - RUN_TIFLASH=true + RUN_TIFLASH=false while [[ $# -gt 0 ]] do diff --git a/util/security.go b/util/security.go index 06aac93fe71e8..49391d4b3f5e8 100644 --- a/util/security.go +++ b/util/security.go @@ -240,34 +240,22 @@ func NewTLSConfig(opts ...TLSConfigOption) (*tls.Config, error) { return nil } + var ( + caContent []byte + err error + ) if builder.caPath != "" { - // clear the content if path is provided - builder.caContent = nil - loadCA := func() (*tls.Config, error) { - certPoolMu.Lock() - defer certPoolMu.Unlock() - - certPool = x509.NewCertPool() - content, err := os.ReadFile(builder.caPath) - if err != nil { - return nil, errors.Annotate(err, "could not read ca certificate") - } - if !certPool.AppendCertsFromPEM(content) { - return nil, errors.New("failed to append ca certs in reading ca certs") - } - tlsCfg.RootCAs = certPool - tlsCfg.ClientCAs = certPool - return tlsCfg, nil - } - tlsCfg.GetConfigForClient = func(info *tls.ClientHelloInfo) (*tls.Config, error) { - return loadCA() + caContent, err = os.ReadFile(builder.caPath) + if err != nil { + return nil, errors.Annotate(err, "could not read ca certificate") } - verifyFuncs = append(verifyFuncs, verifyCA) + } else { + caContent = builder.caContent } - if len(builder.caContent) > 0 { - certPool = x509.NewCertPool() - if !certPool.AppendCertsFromPEM(builder.caContent) { + if len(caContent) > 0 { + certPool = x509.NewCertPool() + if !certPool.AppendCertsFromPEM(caContent) { return nil, errors.New("failed to append ca certs") } tlsCfg.RootCAs = certPool From 02e53da015f8a1e0b60a6a3f1565b9c07f90290d Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 7 Sep 2022 13:04:00 +0800 Subject: [PATCH 13/14] update comment Signed-off-by: lance6716 --- util/security.go | 1 - 1 file changed, 1 deletion(-) diff --git a/util/security.go b/util/security.go index 49391d4b3f5e8..648c6c76fa435 100644 --- a/util/security.go +++ b/util/security.go @@ -118,7 +118,6 @@ type TLSConfigOption func(*tlsConfigBuilder) // WithCAPath sets the CA path to build a tls.Config, and the peer should use the certificate which can be verified by // this CA. It has higher priority than WithCAContent. // empty `caPath` is no-op. -// WithCAPath also support rotation, which means if the CA file is changed, the new content will be used. func WithCAPath(caPath string) TLSConfigOption { return func(builder *tlsConfigBuilder) { builder.caPath = caPath From 2b1bc9c82ff328af77382633d68b58d732327e8c Mon Sep 17 00:00:00 2001 From: lance6716 Date: Wed, 7 Sep 2022 13:04:50 +0800 Subject: [PATCH 14/14] fix debug Signed-off-by: lance6716 --- br/tests/_utils/run_services | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/tests/_utils/run_services b/br/tests/_utils/run_services index c3063482b6e95..a7449cb229bf2 100644 --- a/br/tests/_utils/run_services +++ b/br/tests/_utils/run_services @@ -192,7 +192,7 @@ start_services_impl() { TIDB_CONFIG="tests/config/tidb.toml" TIKV_CONFIG="tests/config/tikv.toml" PD_CONFIG="tests/config/pd.toml" - RUN_TIFLASH=false + RUN_TIFLASH=true while [[ $# -gt 0 ]] do