httputil: rework request signing and request restriction

Signed-off-by: Hank Donnay <[email protected]>

Author: hdonnay

Documentation/reference/config.md

@@ -159,9 +159,9 @@ A key file for the TLS certificate. Encryption is not supported on the key.
 Indexer provides Clair Indexer node configuration.
 
 #### `$.indexer.airgap`
-Boolean.
-
-Disables scanners that have signaled they expect to talk to the Internet.
+Disables HTTP access to the Internet for indexers and fetchers.
+Private IPv4 and IPv6 addresses are allowed.
+Database connections are unaffected.
 
 #### `$.indexer.connstring`
 A Postgres connection string.

config/indexer.go

@@ -35,8 +35,16 @@ type Indexer struct {
 	//
 	// Whether Indexer nodes handle migrations to their database.
 	Migrations bool `yaml:"migrations,omitempty" json:"migrations,omitempty"`
-	// Airgap disables scanners that have signaled they expect to talk to the
-	// Internet.
+	// Airgap disables HTTP access to the Internet. This affects both indexers and
+	// the layer fetcher. Database connections are unaffected.
+	//
+	// "Airgap" is a bit of a misnomer, as [RFC 4193] and [RFC 1918] addresses
+	// are always allowed. This means that setting this flag and also
+	// configuring a proxy on a private network does not prevent contact with
+	// the Internet.
+	//
+	// [RFC 1918]: https://datatracker.ietf.org/doc/html/rfc1918
+	// [RFC 4193]: https://datatracker.ietf.org/doc/html/rfc4193
 	Airgap bool `yaml:"airgap,omitempty" json:"airgap,omitempty"`
 }
 

internal/httputil/client.go

@@ -1,97 +1,85 @@
 package httputil
 
 import (
+	"context"
+	"fmt"
+	"io"
+	"net"
 	"net/http"
 	"net/http/cookiejar"
-	"time"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
 
-	"github.com/quay/clair/config"
+	"github.com/quay/clair/v4/cmd"
 	"golang.org/x/net/publicsuffix"
-	"gopkg.in/square/go-jose.v2"
-	"gopkg.in/square/go-jose.v2/jwt"
 )
 
-// Client returns an http.Client configured according to the supplied
-// configuration.
+// NewClient constructs an [http.Client] that disallows access to public
+// networks, controlled by the localOnly flag.
 //
-// If nil is passed for a claim, the returned client does no signing.
-//
-// It returns an *http.Client and a boolean indicating whether the client is
-// configured for authentication, or an error that occurred during construction.
-func Client(next http.RoundTripper, cl *jwt.Claims, cfg *config.Config) (c *http.Client, authed bool, err error) {
-	if next == nil {
-		next = http.DefaultTransport.(*http.Transport).Clone()
+// If disallowed, the reported error will be a [*net.AddrError] with the "Err"
+// value of "disallowed by policy".
+func NewClient(ctx context.Context, localOnly bool) (*http.Client, error) {
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	dialer := &net.Dialer{}
+	// Set a control function if we're restricting subnets.
+	if localOnly {
+		dialer.Control = ctlLocalOnly
 	}
-	authed = false
+	tr.DialContext = dialer.DialContext
+
 	jar, err := cookiejar.New(&cookiejar.Options{
 		PublicSuffixList: publicsuffix.List,
 	})
 	if err != nil {
-		return nil, false, err
-	}
-	c = &http.Client{
-		Jar: jar,
+		return nil, err
 	}
+	return &http.Client{
+		Transport: tr,
+		Jar:       jar,
+	}, nil
+}
 
-	sk := jose.SigningKey{Algorithm: jose.HS256}
-	// Keep this organized from "best" to "worst". That way, we can add methods
-	// and keep everything working with some careful cluster rolling.
-	switch {
-	case cl == nil: // Skip signing
-	case cfg.Auth.Keyserver != nil:
-		sk.Key = []byte(cfg.Auth.Keyserver.Intraservice)
-	case cfg.Auth.PSK != nil:
-		sk.Key = []byte(cfg.Auth.PSK.Key)
-	default:
-	}
-	rt := &transport{
-		next: next,
+func ctlLocalOnly(network, address string, _ syscall.RawConn) error {
+	// Future-proof for QUIC by allowing UDP here.
+	if !strings.HasPrefix(network, "tcp") && !strings.HasPrefix(network, "udp") {
+		return &net.AddrError{
+			Addr: network + "!" + address,
+			Err:  "disallowed by policy",
+		}
 	}
-	// If we have a claim, make a copy into the transport.
-	if cl != nil {
-		rt.base = *cl
+	addr := net.ParseIP(address)
+	if addr == nil {
+		return &net.AddrError{
+			Addr: network + "!" + address,
+			Err:  "martian address",
+		}
 	}
-	c.Transport = rt
-
-	// Both of the JWT-based methods set the signing key.
-	if sk.Key != nil {
-		signer, err := jose.NewSigner(sk, nil)
-		if err != nil {
-			return nil, false, err
+	if !addr.IsPrivate() {
+		return &net.AddrError{
+			Addr: network + "!" + address,
+			Err:  "disallowed by policy",
 		}
-		rt.Signer = signer
-		authed = true
 	}
-	return c, authed, nil
+	return nil
 }
 
-var _ http.RoundTripper = (*transport)(nil)
-
-// Transport does request modification common to all requests.
-type transport struct {
-	jose.Signer
-	next http.RoundTripper
-	base jwt.Claims
-}
-
-func (cs *transport) RoundTrip(r *http.Request) (*http.Response, error) {
-	const (
-		userAgent = `clair/v4`
-	)
-	r.Header.Set("user-agent", userAgent)
-	if cs.Signer != nil {
-		// TODO(hank) Make this mint longer-lived tokens and re-use them, only
-		// refreshing when needed. Like a resettable sync.Once.
-		now := time.Now()
-		cl := cs.base
-		cl.IssuedAt = jwt.NewNumericDate(now)
-		cl.NotBefore = jwt.NewNumericDate(now.Add(-jwt.DefaultLeeway))
-		cl.Expiry = jwt.NewNumericDate(now.Add(jwt.DefaultLeeway))
-		h, err := jwt.Signed(cs).Claims(&cl).CompactSerialize()
-		if err != nil {
-			return nil, err
-		}
-		r.Header.Add("authorization", "Bearer "+h)
+// NewRequestWithContext is a wrapper around [http.NewRequestWithContext] that
+// sets some defaults in the returned request.
+func NewRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*http.Request, error) {
+	// The one OK use of the normal function.
+	req, err := http.NewRequestWithContext(ctx, method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	p, err := os.Executable()
+	if err != nil {
+		p = `clair?`
+	} else {
+		p = filepath.Base(p)
 	}
-	return cs.next.RoundTrip(r)
+	req.Header.Set("user-agent", fmt.Sprintf("%s/%s", p, cmd.Version))
+	return req, nil
 }

internal/httputil/ratelimiter.go

@@ -21,8 +21,8 @@ func RateLimiter(next http.RoundTripper) http.RoundTripper {
 // Ratelimiter implements the limiting by using a concurrent map and Limiter
 // structs.
 type ratelimiter struct {
-	rt http.RoundTripper
 	lm sync.Map
+	rt http.RoundTripper
 }
 
 const rateCap = 10

internal/httputil/signer.go

@@ -0,0 +1,114 @@
+package httputil
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/quay/clair/config"
+	"github.com/quay/zlog"
+	"gopkg.in/square/go-jose.v2"
+	"gopkg.in/square/go-jose.v2/jwt"
+)
+
+// NewSigner constructs a signer according to the provided Config and claim.
+//
+// The returned Signer only adds headers for the hosts specified in the
+// following spots:
+//
+//   - $.notifier.webhook.target
+//   - $.notifier.indexer_addr
+//   - $.notifier.matcher_addr
+//   - $.matcher.indexer_addr
+func NewSigner(ctx context.Context, cfg *config.Config, cl jwt.Claims) (*Signer, error) {
+	if cfg.Auth.PSK == nil {
+		zlog.Debug(ctx).
+			Str("component", "internal/httputil/NewSigner").
+			Msg("authentication disabled")
+		return new(Signer), nil
+	}
+	s := Signer{
+		use:   make(map[string]struct{}),
+		claim: cl,
+	}
+	if cfg.Notifier.Webhook != nil {
+		if err := s.Add(ctx, cfg.Notifier.Webhook.Target); err != nil {
+			return nil, err
+		}
+	}
+	if err := s.Add(ctx, cfg.Notifier.IndexerAddr); err != nil {
+		return nil, err
+	}
+	if err := s.Add(ctx, cfg.Notifier.MatcherAddr); err != nil {
+		return nil, err
+	}
+	if err := s.Add(ctx, cfg.Matcher.IndexerAddr); err != nil {
+		return nil, err
+	}
+
+	sk := jose.SigningKey{
+		Algorithm: jose.HS256,
+		Key:       []byte(cfg.Auth.PSK.Key),
+	}
+	signer, err := jose.NewSigner(sk, nil)
+	if err != nil {
+		return nil, err
+	}
+	s.signer = signer
+	if zlog.Debug(ctx).Enabled() {
+		as := make([]string, 0, len(s.use))
+		for a := range s.use {
+			as = append(as, a)
+		}
+		zlog.Debug(ctx).Strs("authorities", as).
+			Msg("enabling signing for authorities")
+	}
+	return &s, nil
+}
+
+// Add marks the authority in "uri" as one that expects signed requests.
+func (s *Signer) Add(ctx context.Context, uri string) error {
+	if uri == "" {
+		return nil
+	}
+	u, err := url.Parse(uri)
+	if err != nil {
+		return err
+	}
+	a := u.Host
+	s.use[a] = struct{}{}
+	return nil
+}
+
+// Signer signs requests.
+type Signer struct {
+	signer jose.Signer
+	use    map[string]struct{}
+	claim  jwt.Claims
+}
+
+// Sign modifies the passed [http.Request] as needed.
+func (s *Signer) Sign(ctx context.Context, req *http.Request) error {
+	if s == nil || s.signer == nil {
+		return nil
+	}
+	host := req.Host
+	if host == "" {
+		host = req.URL.Host
+	}
+	if _, ok := s.use[host]; !ok {
+		return nil
+	}
+	cl := s.claim
+	now := time.Now()
+	cl.IssuedAt = jwt.NewNumericDate(now)
+	cl.NotBefore = jwt.NewNumericDate(now.Add(-jwt.DefaultLeeway))
+	cl.Expiry = jwt.NewNumericDate(now.Add(jwt.DefaultLeeway))
+	h, err := jwt.Signed(s.signer).Claims(&cl).CompactSerialize()
+	if err != nil {
+		return err
+	}
+	req.Header.Add("authorization", "Bearer "+h)
+	return nil
+}