Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,54 @@ metadata:
type: Opaque
```

### GCP Workload Identity Federation (WIF)

In addition to service account key JSON files, the CNCC supports [GCP Workload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation) for keyless authentication. Credential detection is automatic — the controller checks for credentials in the following priority order:

1. `workload_identity_config.json` in the credentials secret (WIF)
2. `service_account.json` in the credentials secret (existing behavior)
3. `GOOGLE_APPLICATION_CREDENTIALS` environment variable (for HyperShift HCP deployments)

To use WIF, create a secret with a `workload_identity_config.json` key containing the external account credential configuration.

The raw JSON file contents should look like this:

```json
{
"type": "external_account",
"audience": "//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider",
"subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
"token_url": "https://sts.googleapis.com/v1/token",
"credential_source": {
"file": "/var/run/secrets/openshift/serviceaccount/token",
"format": {
"type": "text"
}
},
"service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/my-sa@my-project.iam.gserviceaccount.com:generateAccessToken"
}
```

Base64-encode the JSON and create the secret:

```yaml
apiVersion: v1
data:
workload_identity_config.json: <base64-encoded JSON from above>
kind: Secret
metadata:
name: cloud-credentials
namespace: openshift-cloud-network-config-controller
type: Opaque
```

**Migration:** No changes are required for existing deployments using `service_account.json`. The controller will continue to use service account credentials if `workload_identity_config.json` is not present.

**Troubleshooting:** Check the controller logs for messages indicating which credential source is being used:
- `"Using GCP Workload Identity Federation credentials from secret"` — WIF is active
- `"Using GCP service account JSON credentials from secret"` — service account key is active
- `"Using GOOGLE_APPLICATION_CREDENTIALS from environment"` — env var fallback is active

## AWS

### Secret
Expand Down
94 changes: 71 additions & 23 deletions pkg/cloudprovider/gcp.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ import (
"fmt"
"net"
"net/url"
"os"
"strings"
"sync"

google "google.golang.org/api/compute/v1"
"google.golang.org/api/option"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
"k8s.io/utils/ptr"
)

Expand All @@ -25,6 +27,9 @@ const (
// default universe domain
// https://github.com/openshift/cloud-network-config-controller/blob/dc255162b1442a1b85aa0b2ab37ed63245857476/vendor/golang.org/x/oauth2/google/default.go#L25
defaultUniverseDomain = "googleapis.com"

wifCredentialsFile = "workload_identity_config.json"
serviceAccountFile = "service_account.json"
)

// GCP implements the API wrapper for talking
Expand All @@ -37,34 +42,18 @@ type GCP struct {
}

func (g *GCP) initCredentials() (err error) {
secret, err := g.readSecretData("service_account.json")
credentialsJSON, err := g.readGCPCredentialsConfig()
if err != nil {
return err
}
secretData := []byte(secret)

// If the UniverseDomain is not set, the client will try to retrieve it from the metadata server.
// https://github.com/openshift/cloud-network-config-controller/blob/dc255162b1442a1b85aa0b2ab37ed63245857476/vendor/golang.org/x/oauth2/google/default.go#L77
// This won't work in OpenShift because the CNCC pod cannot access the metadata service IP address (we block
// the access to 169.254.169.254 from cluster-networked pods).
// Set the UniverseDomain to the default value explicitly.
if !strings.Contains(secret, "universe_domain") {
// Using option.WithUniverseDomain() doesn't work because the value is not passed to the client.
// Modify the credentials json directly instead
var jsonMap map[string]interface{}
err := json.Unmarshal(secretData, &jsonMap)
if err != nil {
return fmt.Errorf("error: cannot decode google client secret, err: %v", err)
}
jsonMap["universe_domain"] = defaultUniverseDomain
secretData, err = json.Marshal(&jsonMap)
if err != nil {
return fmt.Errorf("error: cannot encode google client secret, err: %v", err)
}

credentialsJSON, err = ensureUniverseDomain(credentialsJSON)
if err != nil {
return err
}

opts := []option.ClientOption{
option.WithCredentialsJSON(secretData),
option.WithCredentialsJSON(credentialsJSON),
option.WithUserAgent(UserAgent),
}
if g.cfg.APIOverride != "" {
Expand All @@ -73,11 +62,70 @@ func (g *GCP) initCredentials() (err error) {

g.client, err = google.NewService(g.ctx, opts...)
if err != nil {
return fmt.Errorf("error: cannot initialize google client, err: %v", err)
return fmt.Errorf("error: cannot initialize google client, err: %w", err)
}
return nil
}

// ensureUniverseDomain ensures the credentials JSON has a universe_domain field set.
// If universe_domain is not set, the client will try to retrieve it from the metadata server.
// This won't work in OpenShift because the CNCC pod cannot access 169.254.169.254.
// Set it to the default value explicitly.
func ensureUniverseDomain(credentialsJSON []byte) ([]byte, error) {
var jsonMap map[string]interface{}
if err := json.Unmarshal(credentialsJSON, &jsonMap); err != nil {
return nil, fmt.Errorf("cannot decode GCP credentials JSON: %w", err)
}
if jsonMap == nil {
return nil, fmt.Errorf("cannot decode GCP credentials JSON: top-level JSON object is required")
}
if _, has := jsonMap["universe_domain"]; !has {
klog.Infof("universe_domain not found in credentials, setting default: %s", defaultUniverseDomain)
jsonMap["universe_domain"] = defaultUniverseDomain
credentialsJSON, err := json.Marshal(&jsonMap)
if err != nil {
return nil, fmt.Errorf("cannot encode GCP credentials JSON: %w", err)
}
return credentialsJSON, nil
}
return credentialsJSON, nil
}

// readGCPCredentialsConfig reads GCP credentials from configured sources.
// Priority:
// 1. WIF config from secret
// 2. service account JSON from secret (existing behavior)
// 3. GOOGLE_APPLICATION_CREDENTIALS env var (for HCP)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add here that GOOGLE_APPLICATION_CREDENTIALS is also used for WIF? Otherwise I'm afraid it's not super clear for somebody who only looks at this repo. Thanks!

func (g *GCP) readGCPCredentialsConfig() ([]byte, error) {
// Priority 1: WIF config from secret
wifConfig, err := g.readSecretData(wifCredentialsFile)
if err == nil {
klog.Infof("Using GCP Workload Identity Federation credentials from secret")
return []byte(wifConfig), nil
}
klog.Infof("%s not found in secret: %v, trying service account", wifCredentialsFile, err)

// Priority 2: Service account JSON from secret (existing behavior)
saConfig, err := g.readSecretData(serviceAccountFile)
if err == nil {
klog.Infof("Using GCP service account JSON credentials from secret")
return []byte(saConfig), nil
}
klog.Infof("%s not found in secret: %v", serviceAccountFile, err)

// Priority 3: GOOGLE_APPLICATION_CREDENTIALS env var (for HCP deployments)
if credFile := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"); credFile != "" {
klog.Infof("Using GOOGLE_APPLICATION_CREDENTIALS from environment: %s", credFile)
data, err := os.ReadFile(credFile)
if err != nil {
return nil, fmt.Errorf("failed to read GOOGLE_APPLICATION_CREDENTIALS file %s: %w", credFile, err)
}
return data, nil
}

return nil, fmt.Errorf("no valid GCP credentials found (tried: %s, %s in %s, GOOGLE_APPLICATION_CREDENTIALS env)", wifCredentialsFile, serviceAccountFile, g.cfg.CredentialDir)
}

// AssignPrivateIP adds the IP to the associated instance's IP aliases.
// Important: GCP IP aliases can come in all forms, i.e: if you add 10.0.32.25
// GCP can return 10.0.32.25/32 or 10.0.32.25 - we thus need to check for both
Expand Down
174 changes: 174 additions & 0 deletions pkg/cloudprovider/gcp_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
package cloudprovider

import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"sync"
"testing"

corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -49,3 +55,171 @@ func TestParseSubnet(t *testing.T) {
t.Fatalf("did not expect err: %s", err)
}
}

func newTestGCP(t *testing.T) (*GCP, string) {
t.Helper()
dir := t.TempDir()
g := &GCP{
CloudProvider: CloudProvider{
cfg: CloudProviderConfig{CredentialDir: dir},
ctx: context.Background(),
},
nodeLockMap: make(map[string]*sync.Mutex),
}
return g, dir
}

func TestReadGCPCredentialsConfig_WIFPresent(t *testing.T) {
g, dir := newTestGCP(t)
wifData := `{"type":"external_account","audience":"//iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/pool/providers/provider"}`
if err := os.WriteFile(filepath.Join(dir, "workload_identity_config.json"), []byte(wifData), 0644); err != nil {
t.Fatal(err)
}

data, err := g.readGCPCredentialsConfig()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(data) != wifData {
t.Fatalf("expected WIF config, got: %s", string(data))
}
}

func TestReadGCPCredentialsConfig_SAOnly(t *testing.T) {
g, dir := newTestGCP(t)
saData := `{"type":"service_account","project_id":"my-project"}`
if err := os.WriteFile(filepath.Join(dir, "service_account.json"), []byte(saData), 0644); err != nil {
t.Fatal(err)
}

data, err := g.readGCPCredentialsConfig()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(data) != saData {
t.Fatalf("expected SA config, got: %s", string(data))
}
}

func TestReadGCPCredentialsConfig_EnvVarFallback(t *testing.T) {
g, _ := newTestGCP(t)
envData := `{"type":"external_account","audience":"test"}`
tmpFile := filepath.Join(t.TempDir(), "creds.json")
if err := os.WriteFile(tmpFile, []byte(envData), 0644); err != nil {
t.Fatal(err)
}
t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", tmpFile)

data, err := g.readGCPCredentialsConfig()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(data) != envData {
t.Fatalf("expected env var config, got: %s", string(data))
}
}

Copy link
Contributor

@ricky-rav ricky-rav Mar 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would also add a test where you have service_account.json and theGOOGLE_APPLICATION_CREDENTIALS env var, in order to show which one gets picked.
EDIT: Actually, looking at openshift-online/gcp-hcp#7, in the case of HCP I see that GOOGLE_APPLICATION_CREDENTIALS contains the WIF credentials and there's no fallback. Right?

Copy link
Author

@apahim apahim Mar 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, for HCP, we always use the GOOGLE_APLICATION_CREDENTIALS.

End-to-end flow:

  1. NEW - HO PR #7824: HyperShift Operator (ReconcileCredentials in hostedcluster controller)
  • Creates secret "cloud-network-config-controller-creds" containing: application_default_credentials.json (WIF external_account JSON)
  1. NEW - HO PR #7824: HyperShift CPO (control-plane-operator, CNO deployment component)
  • Sets GCP_CNCC_CREDENTIALS_FILE=application_default_credentials.json as env var on the CNO deployment in the control plane namespace
  1. NEW - CNO PR #2915: CNO (cluster-network-operator, renderCloudNetworkConfigController)
  • Reads GCP_CNCC_CREDENTIALS_FILE
  • Renders managed/controller.yaml template with GOOGLE_APPLICATION_CREDENTIALS=/etc/secret/cloudprovider/application_default_credentials.json on the CNCC container
  1. EXISTING - already in managed/controller.yaml template: kubelet (on management cluster node)
  • Mounts the secret as a volume at /etc/secret/cloudprovider (configured in the template as volume "cloud-provider-secret")
  1. EXISTING - cloud-token minter
  • Sidecar container in CNCC pod
  • Mints a projected SA token and writes it to /var/run/secrets/openshift/serviceaccount/token
  1. NEW - CNCC PR GCP-429: feat: Add GCP Workload Identity Federation (WIF) credential support #206: CNCC (cloud-network-config-controller, readGCPCredentialsConfig)
  • Reads GOOGLE_APPLICATION_CREDENTIALS
  • Loads WIF config
  • WIF config's credential_source.file references the minted token
  1. EXISTING - CNCC GCP client exchanges token via STS for access credentials

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, so let's add to gcp.go a line explaining that through the GOOGLE_APPLICATION_CREDENTIALS env var we load the WIF config. :)

func TestReadGCPCredentialsConfig_WIFTakesPriority(t *testing.T) {
g, dir := newTestGCP(t)
wifData := `{"type":"external_account","audience":"wif"}`
saData := `{"type":"service_account","project_id":"sa"}`
if err := os.WriteFile(filepath.Join(dir, "workload_identity_config.json"), []byte(wifData), 0644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "service_account.json"), []byte(saData), 0644); err != nil {
t.Fatal(err)
}

data, err := g.readGCPCredentialsConfig()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(data) != wifData {
t.Fatalf("expected WIF config to take priority, got: %s", string(data))
}
}

func TestReadGCPCredentialsConfig_NothingPresent(t *testing.T) {
g, _ := newTestGCP(t)
t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "")

_, err := g.readGCPCredentialsConfig()
if err == nil {
t.Fatal("expected error when no credentials are present")
}
if !strings.Contains(err.Error(), "no valid GCP credentials found") {
t.Fatalf("unexpected error message: %v", err)
}
}

func TestReadGCPCredentialsConfig_EnvVarFileMissing(t *testing.T) {
g, _ := newTestGCP(t)
t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/nonexistent/path/creds.json")

_, err := g.readGCPCredentialsConfig()
if err == nil {
t.Fatal("expected error when GOOGLE_APPLICATION_CREDENTIALS file doesn't exist")
}
if !strings.Contains(err.Error(), "failed to read GOOGLE_APPLICATION_CREDENTIALS") {
t.Fatalf("unexpected error message: %v", err)
}
}

func TestEnsureUniverseDomain_Injected(t *testing.T) {
input := []byte(`{"type":"service_account","project_id":"test"}`)

result, err := ensureUniverseDomain(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

var resultMap map[string]interface{}
if err := json.Unmarshal(result, &resultMap); err != nil {
t.Fatalf("failed to unmarshal result: %v", err)
}
if resultMap["universe_domain"] != defaultUniverseDomain {
t.Fatalf("expected universe_domain %s, got %v", defaultUniverseDomain, resultMap["universe_domain"])
}
}

func TestEnsureUniverseDomain_Preserved(t *testing.T) {
customDomain := "custom.googleapis.com"
input := []byte(`{"type":"service_account","project_id":"test","universe_domain":"` + customDomain + `"}`)

result, err := ensureUniverseDomain(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

var resultMap map[string]interface{}
if err := json.Unmarshal(result, &resultMap); err != nil {
t.Fatalf("failed to unmarshal result: %v", err)
}
if resultMap["universe_domain"] != customDomain {
t.Fatalf("expected universe_domain %s, got %v", customDomain, resultMap["universe_domain"])
}
}

func TestEnsureUniverseDomain_InvalidJSON(t *testing.T) {
input := []byte(`{not valid json`)

_, err := ensureUniverseDomain(input)
if err == nil {
t.Fatal("expected error for invalid JSON")
}
if !strings.Contains(err.Error(), "cannot decode GCP credentials JSON") {
t.Fatalf("unexpected error message: %v", err)
}
}

func TestEnsureUniverseDomain_NullJSON(t *testing.T) {
input := []byte(`null`)

_, err := ensureUniverseDomain(input)
if err == nil {
t.Fatal("expected error for null JSON")
}
if !strings.Contains(err.Error(), "top-level JSON object is required") {
t.Fatalf("unexpected error message: %v", err)
}
}