Skip to content

Commit fc3132c

Browse files
authored
expression: JSON_SCHEMA_VALID() (#52780)
close #52779
1 parent 2069651 commit fc3132c

File tree

16 files changed

+324
-7
lines changed

16 files changed

+324
-7
lines changed

DEPS.bzl

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6266,6 +6266,32 @@ def go_deps():
62666266
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.50.1.zip",
62676267
],
62686268
)
6269+
go_repository(
6270+
name = "com_github_qri_io_jsonpointer",
6271+
build_file_proto_mode = "disable_global",
6272+
importpath = "github.com/qri-io/jsonpointer",
6273+
sha256 = "6870d4b9fc5ac8efb9226447975fecfb07241133e23c7e661f5aac1a3088f338",
6274+
strip_prefix = "github.com/qri-io/[email protected]",
6275+
urls = [
6276+
"http://bazel-cache.pingcap.net:8080/gomod/github.com/qri-io/jsonpointer/com_github_qri_io_jsonpointer-v0.1.1.zip",
6277+
"http://ats.apps.svc/gomod/github.com/qri-io/jsonpointer/com_github_qri_io_jsonpointer-v0.1.1.zip",
6278+
"https://cache.hawkingrei.com/gomod/github.com/qri-io/jsonpointer/com_github_qri_io_jsonpointer-v0.1.1.zip",
6279+
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/qri-io/jsonpointer/com_github_qri_io_jsonpointer-v0.1.1.zip",
6280+
],
6281+
)
6282+
go_repository(
6283+
name = "com_github_qri_io_jsonschema",
6284+
build_file_proto_mode = "disable_global",
6285+
importpath = "github.com/qri-io/jsonschema",
6286+
sha256 = "51305cc45fd383b24de94e2eb421ffba8d83679520c18348842c4255025c5940",
6287+
strip_prefix = "github.com/qri-io/[email protected]",
6288+
urls = [
6289+
"http://bazel-cache.pingcap.net:8080/gomod/github.com/qri-io/jsonschema/com_github_qri_io_jsonschema-v0.2.1.zip",
6290+
"http://ats.apps.svc/gomod/github.com/qri-io/jsonschema/com_github_qri_io_jsonschema-v0.2.1.zip",
6291+
"https://cache.hawkingrei.com/gomod/github.com/qri-io/jsonschema/com_github_qri_io_jsonschema-v0.2.1.zip",
6292+
"https://storage.googleapis.com/pingcapmirror/gomod/github.com/qri-io/jsonschema/com_github_qri_io_jsonschema-v0.2.1.zip",
6293+
],
6294+
)
62696295
go_repository(
62706296
name = "com_github_quasilyte_go_ruleguard",
62716297
build_file_proto_mode = "disable_global",

go.mod

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ require (
9393
github.com/prometheus/client_model v0.6.1
9494
github.com/prometheus/common v0.53.0
9595
github.com/prometheus/prometheus v0.50.1
96+
github.com/qri-io/jsonschema v0.2.1
9697
github.com/robfig/cron/v3 v3.0.1
9798
github.com/sasha-s/go-deadlock v0.3.1
9899
github.com/shirou/gopsutil/v3 v3.24.4
@@ -157,6 +158,7 @@ require (
157158
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
158159
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
159160
github.com/pierrec/lz4/v4 v4.1.15 // indirect
161+
github.com/qri-io/jsonpointer v0.1.1 // indirect
160162
github.com/zeebo/xxh3 v1.0.2 // indirect
161163
)
162164

@@ -307,7 +309,7 @@ require (
307309
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
308310
google.golang.org/protobuf v1.33.0 // indirect
309311
gopkg.in/inf.v0 v0.9.1 // indirect
310-
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
312+
gopkg.in/natefinch/lumberjack.v2 v2.2.1
311313
gopkg.in/yaml.v3 v3.0.1 // indirect
312314
k8s.io/apimachinery v0.28.6 // indirect
313315
k8s.io/klog/v2 v2.120.1 // indirect

go.sum

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,10 @@ github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGK
740740
github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g=
741741
github.com/prometheus/prometheus v0.50.1 h1:N2L+DYrxqPh4WZStU+o1p/gQlBaqFbcLBTjlp3vpdXw=
742742
github.com/prometheus/prometheus v0.50.1/go.mod h1:FvE8dtQ1Ww63IlyKBn1V4s+zMwF9kHkVNkQBR1pM4CU=
743+
github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJiBA=
744+
github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64=
745+
github.com/qri-io/jsonschema v0.2.1 h1:NNFoKms+kut6ABPf6xiKNM5214jzxAhDBrPHCJ97Wg0=
746+
github.com/qri-io/jsonschema v0.2.1/go.mod h1:g7DPkiOsK1xv6T/Ao5scXRkd+yTFygcANPBaaqW+VrI=
743747
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
744748
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
745749
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
@@ -767,6 +771,9 @@ github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71e
767771
github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM=
768772
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
769773
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
774+
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
775+
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
776+
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
770777
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI=
771778
github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0=
772779
github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA=

pkg/executor/reload_expr_pushdown_blacklist.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,7 @@ var funcName2Alias = map[string]string{
347347
"json_merge_preserve": ast.JSONMergePreserve,
348348
"json_pretty": ast.JSONPretty,
349349
"json_quote": ast.JSONQuote,
350+
"json_schema_valid": ast.JSONSchemaValid,
350351
"json_search": ast.JSONSearch,
351352
"json_storage_size": ast.JSONStorageSize,
352353
"json_depth": ast.JSONDepth,

pkg/expression/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ go_library(
123123
"@com_github_pingcap_errors//:errors",
124124
"@com_github_pingcap_failpoint//:failpoint",
125125
"@com_github_pingcap_tipb//go-tipb",
126+
"@com_github_qri_io_jsonschema//:jsonschema",
126127
"@com_github_tikv_client_go_v2//oracle",
127128
"@org_uber_go_atomic//:atomic",
128129
"@org_uber_go_zap//:zap",

pkg/expression/builtin.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -902,6 +902,7 @@ var funcs = map[string]functionClass{
902902
ast.JSONMergePreserve: &jsonMergePreserveFunctionClass{baseFunctionClass{ast.JSONMergePreserve, 2, -1}},
903903
ast.JSONPretty: &jsonPrettyFunctionClass{baseFunctionClass{ast.JSONPretty, 1, 1}},
904904
ast.JSONQuote: &jsonQuoteFunctionClass{baseFunctionClass{ast.JSONQuote, 1, 1}},
905+
ast.JSONSchemaValid: &jsonSchemaValidFunctionClass{baseFunctionClass{ast.JSONSchemaValid, 2, 2}},
905906
ast.JSONSearch: &jsonSearchFunctionClass{baseFunctionClass{ast.JSONSearch, 3, -1}},
906907
ast.JSONStorageFree: &jsonStorageFreeFunctionClass{baseFunctionClass{ast.JSONStorageFree, 1, 1}},
907908
ast.JSONStorageSize: &jsonStorageSizeFunctionClass{baseFunctionClass{ast.JSONStorageSize, 1, 1}},

pkg/expression/builtin_json.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,21 @@ package expression
1616

1717
import (
1818
"bytes"
19+
"context"
1920
goJSON "encoding/json"
2021
"strconv"
2122
"strings"
2223

2324
"github.com/pingcap/errors"
25+
"github.com/pingcap/failpoint"
2426
"github.com/pingcap/tidb/pkg/parser/ast"
2527
"github.com/pingcap/tidb/pkg/parser/charset"
2628
"github.com/pingcap/tidb/pkg/parser/mysql"
2729
"github.com/pingcap/tidb/pkg/types"
2830
"github.com/pingcap/tidb/pkg/util/chunk"
2931
"github.com/pingcap/tidb/pkg/util/hack"
3032
"github.com/pingcap/tipb/go-tipb"
33+
"github.com/qri-io/jsonschema"
3134
)
3235

3336
var (
@@ -53,6 +56,7 @@ var (
5356
_ functionClass = &jsonMergePreserveFunctionClass{}
5457
_ functionClass = &jsonPrettyFunctionClass{}
5558
_ functionClass = &jsonQuoteFunctionClass{}
59+
_ functionClass = &jsonSchemaValidFunctionClass{}
5660
_ functionClass = &jsonSearchFunctionClass{}
5761
_ functionClass = &jsonStorageSizeFunctionClass{}
5862
_ functionClass = &jsonDepthFunctionClass{}
@@ -77,6 +81,7 @@ var (
7781
_ builtinFunc = &builtinJSONOverlapsSig{}
7882
_ builtinFunc = &builtinJSONStorageSizeSig{}
7983
_ builtinFunc = &builtinJSONDepthSig{}
84+
_ builtinFunc = &builtinJSONSchemaValidSig{}
8085
_ builtinFunc = &builtinJSONSearchSig{}
8186
_ builtinFunc = &builtinJSONKeysSig{}
8287
_ builtinFunc = &builtinJSONKeys2ArgsSig{}
@@ -1796,3 +1801,94 @@ func (b *builtinJSONLengthSig) evalInt(ctx EvalContext, row chunk.Row) (res int6
17961801
}
17971802
return int64(obj.GetElemCount()), false, nil
17981803
}
1804+
1805+
type jsonSchemaValidFunctionClass struct {
1806+
baseFunctionClass
1807+
}
1808+
1809+
func (c *jsonSchemaValidFunctionClass) getFunction(ctx BuildContext, args []Expression) (builtinFunc, error) {
1810+
if err := c.verifyArgs(args); err != nil {
1811+
return nil, err
1812+
}
1813+
bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETInt, types.ETJson, types.ETJson)
1814+
if err != nil {
1815+
return nil, err
1816+
}
1817+
1818+
sig := &builtinJSONSchemaValidSig{baseBuiltinFunc: bf}
1819+
return sig, nil
1820+
}
1821+
1822+
type builtinJSONSchemaValidSig struct {
1823+
baseBuiltinFunc
1824+
1825+
schemaCache builtinFuncCache[jsonschema.Schema]
1826+
}
1827+
1828+
func (b *builtinJSONSchemaValidSig) Clone() builtinFunc {
1829+
newSig := &builtinJSONSchemaValidSig{}
1830+
newSig.cloneFrom(&b.baseBuiltinFunc)
1831+
return newSig
1832+
}
1833+
1834+
func (b *builtinJSONSchemaValidSig) evalInt(ctx EvalContext, row chunk.Row) (res int64, isNull bool, err error) {
1835+
var schema jsonschema.Schema
1836+
1837+
// First argument is the schema
1838+
schemaData, schemaIsNull, err := b.args[0].EvalJSON(ctx, row)
1839+
if err != nil {
1840+
return res, false, err
1841+
}
1842+
if schemaIsNull {
1843+
return res, true, err
1844+
}
1845+
1846+
if b.args[0].ConstLevel() >= ConstOnlyInContext {
1847+
schema, err = b.schemaCache.getOrInitCache(ctx, func() (jsonschema.Schema, error) {
1848+
failpoint.Inject("jsonSchemaValidDisableCacheRefresh", func() {
1849+
failpoint.Return(jsonschema.Schema{}, errors.New("Cache refresh disabled by failpoint"))
1850+
})
1851+
dataBin, err := schemaData.MarshalJSON()
1852+
if err != nil {
1853+
return jsonschema.Schema{}, err
1854+
}
1855+
if err := goJSON.Unmarshal(dataBin, &schema); err != nil {
1856+
return jsonschema.Schema{}, err
1857+
}
1858+
return schema, nil
1859+
})
1860+
if err != nil {
1861+
return res, false, err
1862+
}
1863+
} else {
1864+
dataBin, err := schemaData.MarshalJSON()
1865+
if err != nil {
1866+
return res, false, err
1867+
}
1868+
if err := goJSON.Unmarshal(dataBin, &schema); err != nil {
1869+
return res, false, err
1870+
}
1871+
}
1872+
1873+
// Second argument is the JSON document
1874+
docData, docIsNull, err := b.args[1].EvalJSON(ctx, row)
1875+
if err != nil {
1876+
return res, false, err
1877+
}
1878+
if docIsNull {
1879+
return res, true, err
1880+
}
1881+
docDataBin, err := docData.MarshalJSON()
1882+
if err != nil {
1883+
return res, false, err
1884+
}
1885+
errs, err := schema.ValidateBytes(context.Background(), docDataBin)
1886+
if err != nil {
1887+
return res, false, err
1888+
}
1889+
if len(errs) > 0 {
1890+
return res, false, nil
1891+
}
1892+
res = 1
1893+
return res, false, nil
1894+
}

pkg/expression/builtin_json_test.go

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"fmt"
1919
"testing"
2020

21+
"github.com/pingcap/failpoint"
2122
"github.com/pingcap/tidb/pkg/parser/ast"
2223
"github.com/pingcap/tidb/pkg/parser/mysql"
2324
"github.com/pingcap/tidb/pkg/parser/terror"
@@ -1342,3 +1343,100 @@ func TestJSONMergePatch(t *testing.T) {
13421343
}
13431344
}
13441345
}
1346+
1347+
func TestJSONSchemaValid(t *testing.T) {
1348+
ctx := createContext(t)
1349+
fc := funcs[ast.JSONSchemaValid]
1350+
tbl := []struct {
1351+
Input any
1352+
Expected any
1353+
}{
1354+
// nulls
1355+
{[]any{nil, `{}`}, nil},
1356+
{[]any{`{}`, nil}, nil},
1357+
{[]any{nil, nil}, nil},
1358+
1359+
// empty
1360+
{[]any{`{}`, `{}`}, 1},
1361+
1362+
// required
1363+
{[]any{`{"required": ["a","b"]}`, `{"a": 5}`}, 0},
1364+
{[]any{`{"required": ["a","b"]}`, `{"a": 5, "b": 6}`}, 1},
1365+
1366+
// type
1367+
{[]any{`{"type": ["string"]}`, `{}`}, 0},
1368+
{[]any{`{"type": ["string"]}`, `"foobar"`}, 1},
1369+
{[]any{`{"type": ["object"]}`, `{}`}, 1},
1370+
{[]any{`{"type": ["object"]}`, `"foobar"`}, 0},
1371+
1372+
// properties, type
1373+
{[]any{`{"properties": {"a": {"type": "number"}}}`, `{}`}, 1},
1374+
{[]any{`{"properties": {"a": {"type": "number"}}}`, `{"a": "foobar"}`}, 0},
1375+
{[]any{`{"properties": {"a": {"type": "number"}}}`, `{"a": 5}`}, 1},
1376+
1377+
// properties, minimum
1378+
{[]any{`{"properties": {"a": {"type": "number", "minimum": 6}}}`, `{"a": 5}`}, 0},
1379+
1380+
// properties, pattern
1381+
{[]any{`{"properties": {"a": {"type": "string", "pattern": "^a"}}}`, `{"a": "abc"}`}, 1},
1382+
{[]any{`{"properties": {"a": {"type": "string", "pattern": "^a"}}}`, `{"a": "cba"}`}, 0},
1383+
}
1384+
dtbl := tblToDtbl(tbl)
1385+
for _, tt := range dtbl {
1386+
f, err := fc.getFunction(ctx, datumsToConstants(tt["Input"]))
1387+
require.NoError(t, err)
1388+
d, err := evalBuiltinFunc(f, ctx, chunk.Row{})
1389+
require.NoError(t, err)
1390+
if tt["Expected"][0].IsNull() {
1391+
require.True(t, d.IsNull())
1392+
} else {
1393+
testutil.DatumEqual(
1394+
t, tt["Expected"][0], d,
1395+
fmt.Sprintf("JSON_SCHEMA_VALID(%s,%s) = %d (expected: %d)",
1396+
tt["Input"][0].GetString(),
1397+
tt["Input"][1].GetString(),
1398+
d.GetInt64(),
1399+
tt["Expected"][0].GetInt64(),
1400+
),
1401+
)
1402+
}
1403+
}
1404+
}
1405+
1406+
// TestJSONSchemaValidCache is to test if the cached schema is used
1407+
func TestJSONSchemaValidCache(t *testing.T) {
1408+
ctx := createContext(t)
1409+
fc := funcs[ast.JSONSchemaValid]
1410+
tbl := []struct {
1411+
Input any
1412+
Expected any
1413+
}{
1414+
{[]any{`{}`, `{}`}, 1},
1415+
}
1416+
dtbl := tblToDtbl(tbl)
1417+
1418+
for _, tt := range dtbl {
1419+
// Get the function and eval once, ensuring it is cached
1420+
f, err := fc.getFunction(ctx, datumsToConstants(tt["Input"]))
1421+
require.NoError(t, err)
1422+
_, err = evalBuiltinFunc(f, ctx, chunk.Row{})
1423+
require.NoError(t, err)
1424+
1425+
// Disable the cache function
1426+
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/expression/jsonSchemaValidDisableCacheRefresh", `return(true)`))
1427+
1428+
// This eval should use the cache and not call the function.
1429+
_, err = evalBuiltinFunc(f, ctx, chunk.Row{})
1430+
require.NoError(t, err)
1431+
1432+
// Now get a new cache by getting the function again.
1433+
f, err = fc.getFunction(ctx, datumsToConstants(tt["Input"]))
1434+
require.NoError(t, err)
1435+
1436+
// Empty cache, we call the function. This should return an error.
1437+
_, err = evalBuiltinFunc(f, ctx, chunk.Row{})
1438+
require.Error(t, err)
1439+
}
1440+
1441+
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/expression/jsonSchemaValidDisableCacheRefresh"))
1442+
}

pkg/expression/function_traits.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ var booleanFunctions = map[string]struct{}{
279279
ast.IsIPv4Compat: {},
280280
ast.IsIPv4Mapped: {},
281281
ast.IsIPv6: {},
282+
ast.JSONSchemaValid: {},
282283
ast.JSONValid: {},
283284
ast.RegexpLike: {},
284285
}

pkg/parser/ast/functions.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,7 @@ const (
348348
JSONMergePreserve = "json_merge_preserve"
349349
JSONPretty = "json_pretty"
350350
JSONQuote = "json_quote"
351+
JSONSchemaValid = "json_schema_valid"
351352
JSONSearch = "json_search"
352353
JSONStorageFree = "json_storage_free"
353354
JSONStorageSize = "json_storage_size"

0 commit comments

Comments
 (0)