feat: support BigQuery dataset ACLs in absence of IAM endpoints (#78)

Author: onetwopunch

Makefile

@@ -51,5 +51,12 @@ fmtcheck:
 fmt:
 	gofmt -w $(GOFMT_FILES)
 
-
-.PHONY: bin default generate test vet bootstrap fmt fmtcheck
+update-resources:
+	pushd $(CURDIR)/plugin/iamutil && \
+	go build -o generate ./internal && \
+	./generate && \
+	rm generate && \
+	popd
+	
+
+.PHONY: bin default generate test vet bootstrap fmt fmtcheck update-resources

go.mod

@@ -8,11 +8,15 @@ require (
 	github.com/hashicorp/go-gcp-common v0.5.0
 	github.com/hashicorp/go-hclog v0.12.0
 	github.com/hashicorp/go-multierror v1.0.0
+	github.com/hashicorp/go-version v1.2.0 // indirect
 	github.com/hashicorp/hcl v1.0.0
 	github.com/hashicorp/vault-plugin-auth-gcp v0.5.1
 	github.com/hashicorp/vault/api v1.0.5-0.20200215224050-f6547fa8e820
 	github.com/hashicorp/vault/sdk v0.1.14-0.20200215224050-f6547fa8e820
 	github.com/mitchellh/mapstructure v1.1.2
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
+	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
+	golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 // indirect
 	google.golang.org/api v0.14.0
 )

go.sum

@@ -115,6 +115,8 @@ github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2I
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
 github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -175,6 +177,7 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -211,6 +214,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90Pveol
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 h1:O5YqonU5IWby+w98jVUG9h7zlCWCcH4RHyPVReBmhzk=
 golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -235,6 +240,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -260,10 +267,13 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU=
@@ -326,6 +336,7 @@ gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4
 gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

plugin/backend.go

@@ -33,7 +33,7 @@ type backend struct {
 	// cache directly.
 	cache *cache.Cache
 
-	iamResources iamutil.IamResourceParser
+	resources iamutil.ResourceParser
 
 	rolesetLock sync.Mutex
 }
@@ -49,8 +49,8 @@ func Factory(ctx context.Context, conf *logical.BackendConfig) (logical.Backend,
 
 func Backend() *backend {
 	var b = &backend{
-		cache:        cache.New(),
-		iamResources: iamutil.GetEnabledIamResources(),
+		cache:     cache.New(),
+		resources: iamutil.GetEnabledResources(),
 	}
 
 	b.Backend = &framework.Backend{

plugin/iamutil/api_handle.go

@@ -0,0 +1,120 @@
+package iamutil
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/hashicorp/errwrap"
+	"google.golang.org/api/googleapi"
+)
+
+type ApiHandle struct {
+	c         *http.Client
+	userAgent string
+}
+
+func GetApiHandle(client *http.Client, userAgent string) *ApiHandle {
+	return &ApiHandle{
+		c:         client,
+		userAgent: userAgent,
+	}
+}
+
+func (h *ApiHandle) DoGetRequest(ctx context.Context, r Resource, out interface{}) (err error) {
+	config := r.GetConfig()
+	req, err := constructRequest(r, &config.GetMethod, nil)
+	if err != nil {
+		return errwrap.Wrapf("Unable to construct Get request: {{err}}", err)
+	}
+	return h.doRequest(ctx, req, out)
+}
+
+func (h *ApiHandle) DoSetRequest(ctx context.Context, r Resource, data io.Reader, out interface{}) error {
+	config := r.GetConfig()
+	req, err := constructRequest(r, &config.SetMethod, data)
+	if err != nil {
+		return errwrap.Wrapf("Unable to construct Set request: {{err}}", err)
+	}
+	return h.doRequest(ctx, req, out)
+}
+
+func (h *ApiHandle) doRequest(ctx context.Context, req *http.Request, out interface{}) error {
+	if req.Header == nil {
+		req.Header = make(http.Header)
+	}
+	if h.userAgent != "" {
+		req.Header.Set("User-Agent", h.userAgent)
+	}
+
+	resp, err := h.c.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer googleapi.CloseBody(resp)
+
+	if err := googleapi.CheckResponse(resp); err != nil {
+		return err
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
+		return errwrap.Wrapf("unable to decode JSON resp to output interface: {{err}}", err)
+	}
+	return nil
+}
+
+func constructRequest(r Resource, restMethod *RestMethod, data io.Reader) (*http.Request, error) {
+	config := r.GetConfig()
+	if data == nil && config != nil && config.Service == "cloudresourcemanager" {
+		// In order to support Resource Manager policies with conditional bindings,
+		// we need to request the policy version of 3. This request parameter is backwards compatible
+		// and will return version 1 policies if they are not yet updated to version 3.
+		requestPolicyVersion3 := `{"options": {"requestedPolicyVersion": 3}}`
+		data = strings.NewReader(requestPolicyVersion3)
+	}
+	req, err := http.NewRequest(
+		restMethod.HttpMethod,
+		googleapi.ResolveRelative(restMethod.BaseURL, restMethod.Path),
+		data)
+	if err != nil {
+		return nil, err
+	}
+
+	if req.Header == nil {
+		req.Header = make(http.Header)
+	}
+	if data != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	relId := r.GetRelativeId()
+	replacementMap := make(map[string]string)
+
+	if strings.Contains(restMethod.Path, "{+resource}") {
+		// +resource is used to represent full relative resource name
+		if len(config.Parameters) == 1 && config.Parameters[0] == "resource" {
+			relName := ""
+			tkns := strings.Split(config.TypeKey, "/")
+			for _, colId := range tkns {
+				if colName, ok := relId.IdTuples[colId]; ok {
+					relName += fmt.Sprintf("%s/%s/", colId, colName)
+				}
+			}
+			replacementMap["resource"] = strings.Trim(relName, "/")
+		}
+	} else {
+		for colId, resId := range relId.IdTuples {
+			rId, ok := config.CollectionReplacementKeys[colId]
+			if !ok {
+				return nil, fmt.Errorf("expected value for collection id %s", colId)
+			}
+			replacementMap[rId] = resId
+		}
+	}
+
+	googleapi.Expand(req.URL, replacementMap)
+	return req, nil
+}

plugin/iamutil/iam_handle_test.go renamed to plugin/iamutil/api_handle_test.go

@@ -14,8 +14,8 @@ import (
 	"google.golang.org/api/option"
 )
 
-func TestIamHandle_ServiceAccount(t *testing.T) {
-	createServiceAccount := func(t *testing.T, httpC *http.Client) *parsedIamResource {
+func TestIamResource_ServiceAccount(t *testing.T) {
+	createServiceAccount := func(t *testing.T, httpC *http.Client) *IamResource {
 		iamAdmin, err := iam.NewService(context.Background(), option.WithHTTPClient(httpC))
 		if err != nil {
 			t.Fatal(err)
@@ -38,15 +38,15 @@ func TestIamHandle_ServiceAccount(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		rConfig := generatedIamResources["projects/serviceAccounts"]["iam"]["v1"]
+		rConfig := generatedResources["projects/serviceAccounts"]["iam"]["v1"]
 
-		return &parsedIamResource{
+		return &IamResource{
 			relativeId: relId,
 			config:     &rConfig,
 		}
 	}
 
-	deleteServiceAccount := func(t *testing.T, httpC *http.Client, r *parsedIamResource) {
+	deleteServiceAccount := func(t *testing.T, httpC *http.Client, r *IamResource) {
 		saName := fmt.Sprintf("projects/%s/serviceAccounts/%s",
 			r.relativeId.IdTuples["projects"],
 			r.relativeId.IdTuples["serviceAccounts"])
@@ -64,8 +64,8 @@ func TestIamHandle_ServiceAccount(t *testing.T) {
 }
 
 func verifyIamResource_GetSetPolicy(t *testing.T, resourceType string,
-	getF func(*testing.T, *http.Client) *parsedIamResource,
-	cleanupF func(*testing.T, *http.Client, *parsedIamResource)) {
+	getF func(*testing.T, *http.Client) *IamResource,
+	cleanupF func(*testing.T, *http.Client, *IamResource)) {
 
 	_, creds := util.GetTestCredentials(t)
 	httpC, err := gcputil.GetHttpClient(creds, iam.CloudPlatformScope)
@@ -76,9 +76,9 @@ func verifyIamResource_GetSetPolicy(t *testing.T, resourceType string,
 	r := getF(t, httpC)
 	defer cleanupF(t, httpC, r)
 
-	h := GetIamHandle(httpC, "")
+	h := GetApiHandle(httpC, "")
 
-	p, err := h.GetIamPolicy(context.Background(), r)
+	p, err := r.GetIamPolicy(context.Background(), h)
 	if err != nil {
 		t.Fatalf("could not get IAM Policy for resource type '%s': %v", resourceType, err)
 	}
@@ -92,12 +92,12 @@ func verifyIamResource_GetSetPolicy(t *testing.T, resourceType string,
 		t.Fatalf("could not get IAM Policy for resource type '%s': %v", resourceType, err)
 	}
 
-	changedP, err := h.SetIamPolicy(context.Background(), r, newP)
+	changedP, err := r.SetIamPolicy(context.Background(), h, newP)
 	if err != nil {
 		t.Fatalf("could not set IAM Policy for resource type '%s': %v", resourceType, err)
 	}
 
-	actualP, err := h.GetIamPolicy(context.Background(), r)
+	actualP, err := r.GetIamPolicy(context.Background(), h)
 	if err != nil {
 		t.Fatalf("could not get updated IAM Policy for resource type '%s': %v", resourceType, err)
 	}