Skip to content

Commit 9e97f98

Browse files
This change allows the HTTP clients to consume and send gzip compressed response and request body. (#5696)
It is available for the following REST API endpoints: - GET & POST HTTP methods on /v0/data & /v1/data endpoints - POST HTTP method on /v1/compile endpoint HTTP clients can optionally: - send 'Accept-Encoding: gzip' header and expect a gzip compressed body and a Content-Encoding: gzip response header. The server will send the content encoded as gzip only after a threshold defined by server.encoding.gzip.min_length (default value is 1024). If the size is below the threshold, the body is not compressed - send 'Content-Encoding: gzip' header and a gzip compressed body and expect the server to correctly interpret the request Fixes #5310 Signed-off-by: aarnautu <[email protected]>
1 parent dc37446 commit 9e97f98

File tree

12 files changed

+1296
-21
lines changed

12 files changed

+1296
-21
lines changed

config/config.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ type Config struct {
3535
NDBuiltinCache bool `json:"nd_builtin_cache,omitempty"`
3636
PersistenceDirectory *string `json:"persistence_directory,omitempty"`
3737
DistributedTracing json.RawMessage `json:"distributed_tracing,omitempty"`
38-
Storage *struct {
38+
Server *struct {
39+
Encoding json.RawMessage `json:"encoding,omitempty"`
40+
} `json:"server,omitempty"`
41+
Storage *struct {
3942
Disk json.RawMessage `json:"disk,omitempty"`
4043
} `json:"storage,omitempty"`
4144
}

config/config_test.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,14 @@ func TestActiveConfig(t *testing.T) {
196196
"plugins": {
197197
"some-plugin": {}
198198
},
199+
"server": {
200+
"encoding": {
201+
"gzip": {
202+
"min_length": 1024,
203+
"compression_level": 1
204+
}
205+
}
206+
},
199207
"discovery": {"name": "config"}`
200208

201209
serviceObj := `"services": {
@@ -249,6 +257,14 @@ func TestActiveConfig(t *testing.T) {
249257
"plugins": {
250258
"some-plugin": {}
251259
},
260+
"server": {
261+
"encoding": {
262+
"gzip": {
263+
"min_length": 1024,
264+
"compression_level": 1
265+
}
266+
}
267+
},
252268
"default_authorization_decision": "/system/authz/allow",
253269
"default_decision": "/system/main",
254270
"discovery": {"name": "config"}`, version.Version)

docs/content/configuration.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ distributed_tracing:
7676
service_name: opa
7777
sample_percentage: 50
7878
encryption: "off"
79+
80+
server:
81+
encoding:
82+
gzip:
83+
min_length: 1024,
84+
compression_level: 9
7985
```
8086
8187
#### Environment Variable Substitution
@@ -925,3 +931,13 @@ with data put into the configured `directory`.
925931
| `storage.disk.badger` | `string` | No (default: empty) | "Superflags" passed to Badger allowing to modify advanced options. |
926932

927933
See [the docs on disk storage](../misc-disk/) for details about the settings.
934+
935+
### Server
936+
937+
The `server` configuration sets the gzip compression settings for `/v0/data`, `/v1/data` and `/v1/compile` HTTP `POST` endpoints
938+
The gzip compression settings are used when the client sends `Accept-Encoding: gzip`
939+
940+
| Field | Type | Required | Description |
941+
|------------------------------------------|-------|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
942+
| `server.encoding.gzip.min_length` | `int` | No, (default: 1024) | Specifies the minimum length of the response to compress |
943+
| `server.encoding.gzip.compression_level` | `int` | No, (default: 9) | Specifies the compression level. Accepted values: a value of either 0 (no compression), 1 (best speed, lowest compression) or 9 (slowest, best compression). See https://pkg.go.dev/compress/flate#pkg-constants |

docs/content/rest-api.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -732,6 +732,10 @@ The path separator is used to access values inside object and array documents. I
732732
- **instrument** - Instrument query evaluation and return a superset of performance metrics in addition to result. See [Performance Metrics](#performance-metrics) for more detail.
733733
- **strict-builtin-errors** - Treat built-in function call errors as fatal and return an error immediately.
734734

735+
#### Request Headers
736+
737+
- **Accept-Encoding: gzip**: Indicates the server should respond with a gzip encoded body. The server will send the compressed response only if its length is above `server.encoding.gzip.min_length` value. See the configuration section
738+
735739
#### Status Codes
736740

737741
- **200** - no error
@@ -820,6 +824,8 @@ The request body contains an object that specifies a value for [The input Docume
820824
#### Request Headers
821825

822826
- **Content-Type: application/x-yaml**: Indicates the request body is a YAML encoded object.
827+
- **Content-Encoding: gzip**: Indicates the request body is a gzip encoded object.
828+
- **Accept-Encoding: gzip**: Indicates the server should respond with a gzip encoded body. The server will send the compressed response only if its length is above `server.encoding.gzip.min_length` value. See the configuration section
823829

824830
#### Query Parameters
825831

@@ -939,6 +945,8 @@ array documents.
939945
#### Request Headers
940946

941947
- **Content-Type: application/x-yaml**: Indicates the request body is a YAML encoded object.
948+
- **Content-Encoding: gzip**: Indicates the request body is a gzip encoded object.
949+
- **Accept-Encoding: gzip**: Indicates the server should respond with a gzip encoded body. The server will send the compressed response only if its length is above `server.encoding.gzip.min_length` value. See the configuration section
942950

943951
#### Query Parameters
944952

@@ -1290,6 +1298,11 @@ Compile API requests contain the following fields:
12901298
| `options` | `object[string, any]` | No | Additional options to use during partial evaluation. Only `disableInlining` option is supported. (default: undefined). |
12911299
| `unknowns` | `array[string]` | No | The terms to treat as unknown during partial evaluation (default: `["input"]`]). |
12921300

1301+
### Request Headers
1302+
1303+
- **Content-Encoding: gzip**: Indicates the request body is a gzip encoded object.
1304+
- **Accept-Encoding: gzip**: Indicates the server should respond with a gzip encoded body. The server will send the compressed response only if its length is above `server.encoding.gzip.min_length` value
1305+
12931306
#### Query Parameters
12941307

12951308
- **pretty** - If parameter is `true`, response will formatted for humans.

plugins/server/encoding/config.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package encoding
2+
3+
import (
4+
"compress/gzip"
5+
"fmt"
6+
7+
"github.com/open-policy-agent/opa/util"
8+
)
9+
10+
var defaultGzipMinLength = 1024
11+
var defaultGzipCompressionLevel = gzip.BestCompression
12+
13+
// Config represents the configuration for the Server.Encoding settings
14+
type Config struct {
15+
Gzip *Gzip `json:"gzip,omitempty"`
16+
}
17+
18+
// Gzip represents the configuration for the Server.Encoding.Gzip settings
19+
type Gzip struct {
20+
MinLength *int `json:"min_length,omitempty"` // the minimum length of a response that will be gzipped
21+
CompressionLevel *int `json:"compression_level,omitempty"` // the compression level for gzip
22+
}
23+
24+
// ConfigBuilder assists in the construction of the plugin configuration.
25+
type ConfigBuilder struct {
26+
raw []byte
27+
}
28+
29+
// NewConfigBuilder returns a new ConfigBuilder to build and parse the server config
30+
func NewConfigBuilder() *ConfigBuilder {
31+
return &ConfigBuilder{}
32+
}
33+
34+
// WithBytes sets the raw server config
35+
func (b *ConfigBuilder) WithBytes(config []byte) *ConfigBuilder {
36+
b.raw = config
37+
return b
38+
}
39+
40+
// Parse returns a valid Config object with defaults injected.
41+
func (b *ConfigBuilder) Parse() (*Config, error) {
42+
if b.raw == nil {
43+
defaultConfig := &Config{
44+
Gzip: &Gzip{
45+
MinLength: &defaultGzipMinLength,
46+
CompressionLevel: &defaultGzipCompressionLevel,
47+
},
48+
}
49+
return defaultConfig, nil
50+
}
51+
52+
var result Config
53+
54+
if err := util.Unmarshal(b.raw, &result); err != nil {
55+
return nil, err
56+
}
57+
58+
return &result, result.validateAndInjectDefaults()
59+
}
60+
61+
func (c *Config) validateAndInjectDefaults() error {
62+
if c.Gzip == nil {
63+
c.Gzip = &Gzip{
64+
MinLength: &defaultGzipMinLength,
65+
CompressionLevel: &defaultGzipCompressionLevel,
66+
}
67+
}
68+
if c.Gzip.MinLength == nil {
69+
c.Gzip.MinLength = &defaultGzipMinLength
70+
}
71+
72+
if c.Gzip.CompressionLevel == nil {
73+
c.Gzip.CompressionLevel = &defaultGzipCompressionLevel
74+
}
75+
76+
if *c.Gzip.MinLength <= 0 {
77+
return fmt.Errorf("invalid value for server.encoding.gzip.min_length field, should be a positive number")
78+
}
79+
80+
acceptedCompressionLevels := map[int]bool{
81+
gzip.NoCompression: true,
82+
gzip.BestSpeed: true,
83+
gzip.BestCompression: true,
84+
}
85+
_, compressionLevelAccepted := acceptedCompressionLevels[*c.Gzip.CompressionLevel]
86+
if !compressionLevelAccepted {
87+
return fmt.Errorf("invalid value for server.encoding.gzip.compression_level field, accepted values are 0, 1 or 9")
88+
}
89+
90+
return nil
91+
}
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
package encoding
2+
3+
import (
4+
"fmt"
5+
"testing"
6+
)
7+
8+
func TestConfigValidation(t *testing.T) {
9+
tests := []struct {
10+
input string
11+
wantErr bool
12+
}{
13+
{
14+
input: `{}`,
15+
wantErr: false,
16+
},
17+
{
18+
input: `{"gzip": {"min_length": "not-a-number"}}`,
19+
wantErr: true,
20+
},
21+
{
22+
input: `{"gzip": {min_length": 42}}`,
23+
wantErr: false,
24+
},
25+
{
26+
input: `{"gzip":{"min_length": "42"}}`,
27+
wantErr: true,
28+
},
29+
{
30+
input: `{"gzip":{"min_length": 0}}`,
31+
wantErr: true,
32+
},
33+
{
34+
input: `{"gzip":{"min_length": -10}}`,
35+
wantErr: true,
36+
},
37+
{
38+
input: `{"gzip":{"random_key": 0}}`,
39+
wantErr: false,
40+
},
41+
{
42+
input: `{"gzip": {"min_length": -10, "compression_level": 13}}`,
43+
wantErr: true,
44+
},
45+
{
46+
input: `{"gzip":{"compression_level": "not-an-number"}}`,
47+
wantErr: true,
48+
},
49+
{
50+
input: `{"gzip":{"compression_level": 1}}`,
51+
wantErr: false,
52+
},
53+
{
54+
input: `{"gzip":{"compression_level": 13}}`,
55+
wantErr: true,
56+
},
57+
{
58+
input: `{"gzip":{"min_length": 42, "compression_level": 9}}`,
59+
wantErr: false,
60+
},
61+
}
62+
63+
for i, test := range tests {
64+
t.Run(fmt.Sprintf("TestConfigValidation_case_%d", i), func(t *testing.T) {
65+
_, err := NewConfigBuilder().WithBytes([]byte(test.input)).Parse()
66+
if err != nil && !test.wantErr {
67+
t.Fail()
68+
}
69+
if err == nil && test.wantErr {
70+
t.Fail()
71+
}
72+
})
73+
}
74+
}
75+
76+
func TestConfigValue(t *testing.T) {
77+
tests := []struct {
78+
input string
79+
minLengthExpectedValue int
80+
compressionLevelExpectedValue int
81+
}{
82+
{
83+
input: `{}`,
84+
minLengthExpectedValue: 1024,
85+
compressionLevelExpectedValue: 9,
86+
},
87+
{
88+
input: `{"gzip":{"min_length": 42, "compression_level": 1}}`,
89+
minLengthExpectedValue: 42,
90+
compressionLevelExpectedValue: 1,
91+
},
92+
}
93+
94+
for i, test := range tests {
95+
t.Run(fmt.Sprintf("TestConfigValue_case_%d", i), func(t *testing.T) {
96+
config, err := NewConfigBuilder().WithBytes([]byte(test.input)).Parse()
97+
if err != nil {
98+
t.Fail()
99+
}
100+
if *config.Gzip.MinLength != test.minLengthExpectedValue || *config.Gzip.CompressionLevel != test.compressionLevelExpectedValue {
101+
t.Fail()
102+
}
103+
})
104+
}
105+
}

0 commit comments

Comments
 (0)