-
Notifications
You must be signed in to change notification settings - Fork 44
GCP-429: feat: Add GCP Workload Identity Federation (WIF) credential support #206
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,13 +5,15 @@ import ( | |
| "fmt" | ||
| "net" | ||
| "net/url" | ||
| "os" | ||
| "strings" | ||
| "sync" | ||
|
|
||
| google "google.golang.org/api/compute/v1" | ||
| "google.golang.org/api/option" | ||
| corev1 "k8s.io/api/core/v1" | ||
| "k8s.io/apimachinery/pkg/util/sets" | ||
| "k8s.io/klog/v2" | ||
| "k8s.io/utils/ptr" | ||
| ) | ||
|
|
||
|
|
@@ -25,6 +27,9 @@ const ( | |
| // default universe domain | ||
| // https://github.com/openshift/cloud-network-config-controller/blob/dc255162b1442a1b85aa0b2ab37ed63245857476/vendor/golang.org/x/oauth2/google/default.go#L25 | ||
| defaultUniverseDomain = "googleapis.com" | ||
|
|
||
| wifCredentialsFile = "workload_identity_config.json" | ||
| serviceAccountFile = "service_account.json" | ||
| ) | ||
|
|
||
| // GCP implements the API wrapper for talking | ||
|
|
@@ -37,34 +42,18 @@ type GCP struct { | |
| } | ||
|
|
||
| func (g *GCP) initCredentials() (err error) { | ||
| secret, err := g.readSecretData("service_account.json") | ||
| credentialsJSON, err := g.readGCPCredentialsConfig() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| secretData := []byte(secret) | ||
|
|
||
| // If the UniverseDomain is not set, the client will try to retrieve it from the metadata server. | ||
| // https://github.com/openshift/cloud-network-config-controller/blob/dc255162b1442a1b85aa0b2ab37ed63245857476/vendor/golang.org/x/oauth2/google/default.go#L77 | ||
| // This won't work in OpenShift because the CNCC pod cannot access the metadata service IP address (we block | ||
| // the access to 169.254.169.254 from cluster-networked pods). | ||
| // Set the UniverseDomain to the default value explicitly. | ||
| if !strings.Contains(secret, "universe_domain") { | ||
| // Using option.WithUniverseDomain() doesn't work because the value is not passed to the client. | ||
| // Modify the credentials json directly instead | ||
| var jsonMap map[string]interface{} | ||
| err := json.Unmarshal(secretData, &jsonMap) | ||
| if err != nil { | ||
| return fmt.Errorf("error: cannot decode google client secret, err: %v", err) | ||
| } | ||
| jsonMap["universe_domain"] = defaultUniverseDomain | ||
| secretData, err = json.Marshal(&jsonMap) | ||
| if err != nil { | ||
| return fmt.Errorf("error: cannot encode google client secret, err: %v", err) | ||
| } | ||
|
|
||
| credentialsJSON, err = ensureUniverseDomain(credentialsJSON) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| opts := []option.ClientOption{ | ||
| option.WithCredentialsJSON(secretData), | ||
| option.WithCredentialsJSON(credentialsJSON), | ||
| option.WithUserAgent(UserAgent), | ||
| } | ||
| if g.cfg.APIOverride != "" { | ||
|
|
@@ -73,11 +62,70 @@ func (g *GCP) initCredentials() (err error) { | |
|
|
||
| g.client, err = google.NewService(g.ctx, opts...) | ||
| if err != nil { | ||
| return fmt.Errorf("error: cannot initialize google client, err: %v", err) | ||
| return fmt.Errorf("error: cannot initialize google client, err: %w", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // ensureUniverseDomain ensures the credentials JSON has a universe_domain field set. | ||
| // If universe_domain is not set, the client will try to retrieve it from the metadata server. | ||
| // This won't work in OpenShift because the CNCC pod cannot access 169.254.169.254. | ||
| // Set it to the default value explicitly. | ||
| func ensureUniverseDomain(credentialsJSON []byte) ([]byte, error) { | ||
| var jsonMap map[string]interface{} | ||
| if err := json.Unmarshal(credentialsJSON, &jsonMap); err != nil { | ||
| return nil, fmt.Errorf("cannot decode GCP credentials JSON: %w", err) | ||
| } | ||
| if jsonMap == nil { | ||
| return nil, fmt.Errorf("cannot decode GCP credentials JSON: top-level JSON object is required") | ||
| } | ||
| if _, has := jsonMap["universe_domain"]; !has { | ||
apahim marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| klog.Infof("universe_domain not found in credentials, setting default: %s", defaultUniverseDomain) | ||
| jsonMap["universe_domain"] = defaultUniverseDomain | ||
| credentialsJSON, err := json.Marshal(&jsonMap) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("cannot encode GCP credentials JSON: %w", err) | ||
| } | ||
| return credentialsJSON, nil | ||
coderabbitai[bot] marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
| return credentialsJSON, nil | ||
| } | ||
|
|
||
| // readGCPCredentialsConfig reads GCP credentials from configured sources. | ||
| // Priority: | ||
| // 1. WIF config from secret | ||
| // 2. service account JSON from secret (existing behavior) | ||
| // 3. GOOGLE_APPLICATION_CREDENTIALS env var (for HCP) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we add here that GOOGLE_APPLICATION_CREDENTIALS is also used for WIF? Otherwise I'm afraid it's not super clear for somebody who only looks at this repo. Thanks! |
||
| func (g *GCP) readGCPCredentialsConfig() ([]byte, error) { | ||
| // Priority 1: WIF config from secret | ||
| wifConfig, err := g.readSecretData(wifCredentialsFile) | ||
| if err == nil { | ||
| klog.Infof("Using GCP Workload Identity Federation credentials from secret") | ||
| return []byte(wifConfig), nil | ||
| } | ||
| klog.Infof("%s not found in secret: %v, trying service account", wifCredentialsFile, err) | ||
|
|
||
| // Priority 2: Service account JSON from secret (existing behavior) | ||
| saConfig, err := g.readSecretData(serviceAccountFile) | ||
| if err == nil { | ||
| klog.Infof("Using GCP service account JSON credentials from secret") | ||
| return []byte(saConfig), nil | ||
| } | ||
| klog.Infof("%s not found in secret: %v", serviceAccountFile, err) | ||
|
|
||
| // Priority 3: GOOGLE_APPLICATION_CREDENTIALS env var (for HCP deployments) | ||
| if credFile := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"); credFile != "" { | ||
| klog.Infof("Using GOOGLE_APPLICATION_CREDENTIALS from environment: %s", credFile) | ||
| data, err := os.ReadFile(credFile) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to read GOOGLE_APPLICATION_CREDENTIALS file %s: %w", credFile, err) | ||
| } | ||
| return data, nil | ||
| } | ||
|
|
||
| return nil, fmt.Errorf("no valid GCP credentials found (tried: %s, %s in %s, GOOGLE_APPLICATION_CREDENTIALS env)", wifCredentialsFile, serviceAccountFile, g.cfg.CredentialDir) | ||
| } | ||
|
|
||
| // AssignPrivateIP adds the IP to the associated instance's IP aliases. | ||
| // Important: GCP IP aliases can come in all forms, i.e: if you add 10.0.32.25 | ||
| // GCP can return 10.0.32.25/32 or 10.0.32.25 - we thus need to check for both | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,12 @@ | ||
| package cloudprovider | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
| "sync" | ||
| "testing" | ||
|
|
||
| corev1 "k8s.io/api/core/v1" | ||
|
|
@@ -49,3 +55,171 @@ func TestParseSubnet(t *testing.T) { | |
| t.Fatalf("did not expect err: %s", err) | ||
| } | ||
| } | ||
|
|
||
| func newTestGCP(t *testing.T) (*GCP, string) { | ||
| t.Helper() | ||
| dir := t.TempDir() | ||
| g := &GCP{ | ||
| CloudProvider: CloudProvider{ | ||
| cfg: CloudProviderConfig{CredentialDir: dir}, | ||
| ctx: context.Background(), | ||
| }, | ||
| nodeLockMap: make(map[string]*sync.Mutex), | ||
| } | ||
| return g, dir | ||
| } | ||
|
|
||
| func TestReadGCPCredentialsConfig_WIFPresent(t *testing.T) { | ||
| g, dir := newTestGCP(t) | ||
| wifData := `{"type":"external_account","audience":"//iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/pool/providers/provider"}` | ||
| if err := os.WriteFile(filepath.Join(dir, "workload_identity_config.json"), []byte(wifData), 0644); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
|
|
||
| data, err := g.readGCPCredentialsConfig() | ||
| if err != nil { | ||
| t.Fatalf("unexpected error: %v", err) | ||
| } | ||
| if string(data) != wifData { | ||
| t.Fatalf("expected WIF config, got: %s", string(data)) | ||
| } | ||
| } | ||
|
|
||
| func TestReadGCPCredentialsConfig_SAOnly(t *testing.T) { | ||
| g, dir := newTestGCP(t) | ||
| saData := `{"type":"service_account","project_id":"my-project"}` | ||
| if err := os.WriteFile(filepath.Join(dir, "service_account.json"), []byte(saData), 0644); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
|
|
||
| data, err := g.readGCPCredentialsConfig() | ||
| if err != nil { | ||
| t.Fatalf("unexpected error: %v", err) | ||
| } | ||
| if string(data) != saData { | ||
| t.Fatalf("expected SA config, got: %s", string(data)) | ||
| } | ||
| } | ||
|
|
||
| func TestReadGCPCredentialsConfig_EnvVarFallback(t *testing.T) { | ||
| g, _ := newTestGCP(t) | ||
| envData := `{"type":"external_account","audience":"test"}` | ||
| tmpFile := filepath.Join(t.TempDir(), "creds.json") | ||
| if err := os.WriteFile(tmpFile, []byte(envData), 0644); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", tmpFile) | ||
|
|
||
| data, err := g.readGCPCredentialsConfig() | ||
| if err != nil { | ||
| t.Fatalf("unexpected error: %v", err) | ||
| } | ||
| if string(data) != envData { | ||
| t.Fatalf("expected env var config, got: %s", string(data)) | ||
| } | ||
| } | ||
|
|
||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Right, for HCP, we always use the GOOGLE_APLICATION_CREDENTIALS. End-to-end flow:
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. OK, so let's add to gcp.go a line explaining that through the |
||
| func TestReadGCPCredentialsConfig_WIFTakesPriority(t *testing.T) { | ||
| g, dir := newTestGCP(t) | ||
| wifData := `{"type":"external_account","audience":"wif"}` | ||
| saData := `{"type":"service_account","project_id":"sa"}` | ||
| if err := os.WriteFile(filepath.Join(dir, "workload_identity_config.json"), []byte(wifData), 0644); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if err := os.WriteFile(filepath.Join(dir, "service_account.json"), []byte(saData), 0644); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
|
|
||
| data, err := g.readGCPCredentialsConfig() | ||
| if err != nil { | ||
| t.Fatalf("unexpected error: %v", err) | ||
| } | ||
| if string(data) != wifData { | ||
| t.Fatalf("expected WIF config to take priority, got: %s", string(data)) | ||
| } | ||
| } | ||
|
|
||
| func TestReadGCPCredentialsConfig_NothingPresent(t *testing.T) { | ||
| g, _ := newTestGCP(t) | ||
| t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "") | ||
|
|
||
| _, err := g.readGCPCredentialsConfig() | ||
| if err == nil { | ||
| t.Fatal("expected error when no credentials are present") | ||
| } | ||
| if !strings.Contains(err.Error(), "no valid GCP credentials found") { | ||
| t.Fatalf("unexpected error message: %v", err) | ||
| } | ||
| } | ||
|
|
||
| func TestReadGCPCredentialsConfig_EnvVarFileMissing(t *testing.T) { | ||
| g, _ := newTestGCP(t) | ||
| t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/nonexistent/path/creds.json") | ||
|
|
||
| _, err := g.readGCPCredentialsConfig() | ||
| if err == nil { | ||
| t.Fatal("expected error when GOOGLE_APPLICATION_CREDENTIALS file doesn't exist") | ||
| } | ||
| if !strings.Contains(err.Error(), "failed to read GOOGLE_APPLICATION_CREDENTIALS") { | ||
| t.Fatalf("unexpected error message: %v", err) | ||
| } | ||
| } | ||
|
|
||
| func TestEnsureUniverseDomain_Injected(t *testing.T) { | ||
| input := []byte(`{"type":"service_account","project_id":"test"}`) | ||
|
|
||
| result, err := ensureUniverseDomain(input) | ||
| if err != nil { | ||
| t.Fatalf("unexpected error: %v", err) | ||
| } | ||
|
|
||
| var resultMap map[string]interface{} | ||
| if err := json.Unmarshal(result, &resultMap); err != nil { | ||
| t.Fatalf("failed to unmarshal result: %v", err) | ||
| } | ||
| if resultMap["universe_domain"] != defaultUniverseDomain { | ||
| t.Fatalf("expected universe_domain %s, got %v", defaultUniverseDomain, resultMap["universe_domain"]) | ||
| } | ||
| } | ||
|
|
||
| func TestEnsureUniverseDomain_Preserved(t *testing.T) { | ||
| customDomain := "custom.googleapis.com" | ||
| input := []byte(`{"type":"service_account","project_id":"test","universe_domain":"` + customDomain + `"}`) | ||
|
|
||
| result, err := ensureUniverseDomain(input) | ||
| if err != nil { | ||
| t.Fatalf("unexpected error: %v", err) | ||
| } | ||
|
|
||
| var resultMap map[string]interface{} | ||
| if err := json.Unmarshal(result, &resultMap); err != nil { | ||
| t.Fatalf("failed to unmarshal result: %v", err) | ||
| } | ||
| if resultMap["universe_domain"] != customDomain { | ||
| t.Fatalf("expected universe_domain %s, got %v", customDomain, resultMap["universe_domain"]) | ||
| } | ||
| } | ||
|
|
||
| func TestEnsureUniverseDomain_InvalidJSON(t *testing.T) { | ||
| input := []byte(`{not valid json`) | ||
|
|
||
| _, err := ensureUniverseDomain(input) | ||
| if err == nil { | ||
| t.Fatal("expected error for invalid JSON") | ||
| } | ||
| if !strings.Contains(err.Error(), "cannot decode GCP credentials JSON") { | ||
| t.Fatalf("unexpected error message: %v", err) | ||
| } | ||
| } | ||
|
|
||
| func TestEnsureUniverseDomain_NullJSON(t *testing.T) { | ||
| input := []byte(`null`) | ||
|
|
||
| _, err := ensureUniverseDomain(input) | ||
| if err == nil { | ||
| t.Fatal("expected error for null JSON") | ||
| } | ||
| if !strings.Contains(err.Error(), "top-level JSON object is required") { | ||
| t.Fatalf("unexpected error message: %v", err) | ||
| } | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.