Added PSC Private Service Connect for GCP CloudSQL (#27889)

* Added PSC Private Service Connect for GCP CloudSQL
Added PrivateIP support for GCP MySQL

* Added changelog

* Update changelog

* Value need to be exported or will be false

* Exported variablee for MySQL as well

* Add test cases

* Add go doc test comments

---------

Co-authored-by: robmonte <17119716+robmonte@users.noreply.github.com>
This commit is contained in:
Etourneau Gwenn 2025-07-16 01:29:47 +09:00 committed by GitHub
parent 07854e7be6
commit 06eaa6d500
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 207 additions and 20 deletions

6
changelog/27889.txt Normal file
View File

@ -0,0 +1,6 @@
```release-note:improvement
secrets/database: Add PSC support for GCP CloudSQL MySQL and Postgresql
```
```release-note:improvement
secrets/database: Add PrivateIP support for MySQL
```

View File

@ -37,6 +37,8 @@ type mySQLConnectionProducer struct {
Password string `json:"password" mapstructure:"password" structs:"password"`
AuthType string `json:"auth_type" mapstructure:"auth_type" structs:"auth_type"`
ServiceAccountJSON string `json:"service_account_json" mapstructure:"service_account_json" structs:"service_account_json"`
UsePrivateIP bool `json:"use_private_ip" mapstructure:"use_private_ip" structs:"use_private_ip"`
UsePSC bool `json:"use_psc" mapstructure:"use_psc" structs:"use_psc"`
TLSCertificateKeyData []byte `json:"tls_certificate_key" mapstructure:"tls_certificate_key" structs:"-"`
TLSCAData []byte `json:"tls_ca" mapstructure:"tls_ca" structs:"-"`
@ -123,8 +125,11 @@ func (c *mySQLConnectionProducer) Init(ctx context.Context, conf map[string]inte
}
// validate auth_type if provided
if ok := connutil.ValidateAuthType(c.AuthType); !ok {
return nil, fmt.Errorf("invalid auth_type: %s", c.AuthType)
authType := c.AuthType
if authType != "" {
if ok := connutil.ValidateAuthType(authType); !ok {
return nil, fmt.Errorf("invalid auth_type %s provided", authType)
}
}
if c.AuthType == connutil.AuthTypeGCPIAM {
@ -137,7 +142,7 @@ func (c *mySQLConnectionProducer) Init(ctx context.Context, conf map[string]inte
// however, the driver might store a credentials file, in which case the state stored by the driver is in
// fact critical to the proper function of the connection. So it needs to be registered here inside the
// ConnectionProducer init.
dialerCleanup, err := registerDriverMySQL(c.cloudDriverName, c.ServiceAccountJSON)
dialerCleanup, err := registerDriverMySQL(c.cloudDriverName, c.ServiceAccountJSON, c.UsePrivateIP, c.UsePSC)
if err != nil {
return nil, err
}
@ -318,8 +323,8 @@ func (c *mySQLConnectionProducer) rewriteProtocolForGCP(inDSN string) (string, e
return config.FormatDSN(), nil
}
func registerDriverMySQL(driverName, credentials string) (cleanup func() error, err error) {
opts, err := connutil.GetCloudSQLAuthOptions(credentials, false)
func registerDriverMySQL(driverName, credentials string, usePrivateIP bool, usePSC bool) (cleanup func() error, err error) {
opts, err := connutil.GetCloudSQLAuthOptions(credentials, usePrivateIP, usePSC)
if err != nil {
return nil, err
}

View File

@ -46,13 +46,16 @@ func TestMySQL_Initialize(t *testing.T) {
}
}
// TestMySQL_Initialize_CloudGCP validates the proper initialization of a MySQL backend pointing
// to a GCP CloudSQL MySQL instance. This expects some external setup (exact TBD)
func TestMySQL_Initialize_CloudGCP(t *testing.T) {
envConnURL := "CONNECTION_URL"
connURL := os.Getenv(envConnURL)
if connURL == "" {
t.Skipf("env var %s not set, skipping test", envConnURL)
// TestMySQL_Initialize_CloudGCP_Normal - test initialize in GCP with a normal connection.
// For the CloudGCP Normal, PrivateIP, and PSC tests, follow the instructions within the
// README at https://github.com/shinji62/vault-tf-psc-test for environment setup.
// The Terraform takes care of most steps.
func TestMySQL_Initialize_CloudGCP_Normal(t *testing.T) {
envNormalConnURL := "CLOUDGCP_NORMAL_CONNECTION_URL"
normalConnURL := os.Getenv(envNormalConnURL)
if normalConnURL == "" {
t.Skipf("env var %s not set, skipping test", envNormalConnURL)
}
credStr := dbtesting.GetGCPTestCredentials(t)
@ -65,7 +68,7 @@ func TestMySQL_Initialize_CloudGCP(t *testing.T) {
"empty auth type": {
req: dbplugin.InitializeRequest{
Config: map[string]interface{}{
"connection_url": connURL,
"connection_url": normalConnURL,
"auth_type": "",
},
},
@ -73,7 +76,7 @@ func TestMySQL_Initialize_CloudGCP(t *testing.T) {
"invalid auth type": {
req: dbplugin.InitializeRequest{
Config: map[string]interface{}{
"connection_url": connURL,
"connection_url": normalConnURL,
"auth_type": "invalid",
},
},
@ -83,7 +86,7 @@ func TestMySQL_Initialize_CloudGCP(t *testing.T) {
"JSON credentials": {
req: dbplugin.InitializeRequest{
Config: map[string]interface{}{
"connection_url": connURL,
"connection_url": normalConnURL,
"auth_type": connutil.AuthTypeGCPIAM,
"service_account_json": credStr,
},
@ -119,6 +122,159 @@ func TestMySQL_Initialize_CloudGCP(t *testing.T) {
}
}
// TestMySQL_Initialize_CloudGCP_PrivateIP - test initialize in GCP with a private IP.
// For the CloudGCP Normal, PrivateIP, and PSC tests, follow the instructions within the
// README at https://github.com/shinji62/vault-tf-psc-test for environment setup.
// The Terraform takes care of most steps.
func TestMySQL_Initialize_CloudGCP_PrivateIP(t *testing.T) {
envPrivateIPConnURL := "CLOUDGCP_PRIVATEIP_CONNECTION_URL"
privateIPConnURL := os.Getenv(envPrivateIPConnURL)
if privateIPConnURL == "" {
t.Skipf("env var %s not set, skipping test", envPrivateIPConnURL)
}
credStr := dbtesting.GetGCPTestCredentials(t)
tests := map[string]struct {
req dbplugin.InitializeRequest
wantErr bool
expectedError string
}{
"empty auth type": {
req: dbplugin.InitializeRequest{
Config: map[string]interface{}{
"connection_url": privateIPConnURL,
"auth_type": "",
},
},
},
"invalid auth type": {
req: dbplugin.InitializeRequest{
Config: map[string]interface{}{
"connection_url": privateIPConnURL,
"auth_type": "invalid",
},
},
wantErr: true,
expectedError: "invalid auth_type",
},
"JSON credentials Private IP connection": {
req: dbplugin.InitializeRequest{
Config: map[string]interface{}{
"connection_url": privateIPConnURL,
"auth_type": connutil.AuthTypeGCPIAM,
"service_account_json": credStr,
"use_private_ip": true,
},
VerifyConnection: true,
},
},
}
for n, tc := range tests {
t.Run(n, func(t *testing.T) {
db := newMySQL(DefaultUserNameTemplate)
defer dbtesting.AssertClose(t, db)
_, err := db.Initialize(context.Background(), tc.req)
if tc.wantErr {
if err == nil {
t.Fatalf("expected error but received nil")
}
if !strings.Contains(err.Error(), tc.expectedError) {
t.Fatalf("expected error %s, got %s", tc.expectedError, err.Error())
}
} else {
if err != nil {
t.Fatalf("expected no error, received %s", err)
}
if !db.Initialized {
t.Fatal("Database should be initialized")
}
}
})
}
}
// TestMySQL_Initialize_CloudGCP_PSC - test initialize in GCP with private service connect.
// For the CloudGCP Normal, PrivateIP, and PSC tests, follow the instructions within the
// README at https://github.com/shinji62/vault-tf-psc-test for environment setup.
// The Terraform takes care of most steps.
func TestMySQL_Initialize_CloudGCP_PSC(t *testing.T) {
envPSCConnURL := "CLOUDGCP_PSC_CONNECTION_URL"
pscConnURL := os.Getenv(envPSCConnURL)
if pscConnURL == "" {
t.Skipf("env var %s not set, skipping test", pscConnURL)
}
credStr := dbtesting.GetGCPTestCredentials(t)
tests := map[string]struct {
req dbplugin.InitializeRequest
wantErr bool
expectedError string
}{
"empty auth type": {
req: dbplugin.InitializeRequest{
Config: map[string]interface{}{
"connection_url": pscConnURL,
"auth_type": "",
},
},
},
"invalid auth type": {
req: dbplugin.InitializeRequest{
Config: map[string]interface{}{
"connection_url": pscConnURL,
"auth_type": "invalid",
},
},
wantErr: true,
expectedError: "invalid auth_type",
},
"JSON credentials Private Service Connect connection": {
req: dbplugin.InitializeRequest{
Config: map[string]interface{}{
"connection_url": pscConnURL,
"auth_type": connutil.AuthTypeGCPIAM,
"service_account_json": credStr,
"use_psc": true,
},
VerifyConnection: true,
},
},
}
for n, tc := range tests {
t.Run(n, func(t *testing.T) {
db := newMySQL(DefaultUserNameTemplate)
defer dbtesting.AssertClose(t, db)
_, err := db.Initialize(context.Background(), tc.req)
if tc.wantErr {
if err == nil {
t.Fatalf("expected error but received nil")
}
if !strings.Contains(err.Error(), tc.expectedError) {
t.Fatalf("expected error %s, got %s", tc.expectedError, err.Error())
}
} else {
if err != nil {
t.Fatalf("expected no error, received %s", err)
}
if !db.Initialized {
t.Fatal("Database should be initialized")
}
}
})
}
}
func testInitialize(t *testing.T, rootPassword string) {
cleanup, connURL := mysqlhelper.PrepareTestContainer(t, false, rootPassword)
defer cleanup()

View File

@ -23,13 +23,13 @@ func (c *SQLConnectionProducer) getCloudSQLDriverType() (string, error) {
return driverType, nil
}
func (c *SQLConnectionProducer) registerDrivers(driverName string, credentials string, usePrivateIP bool) (func() error, error) {
func (c *SQLConnectionProducer) registerDrivers(driverName string, credentials string, usePrivateIP bool, usePSC bool) (func() error, error) {
typ, err := c.getCloudSQLDriverType()
if err != nil {
return nil, err
}
opts, err := GetCloudSQLAuthOptions(credentials, usePrivateIP)
opts, err := GetCloudSQLAuthOptions(credentials, usePrivateIP, usePSC)
if err != nil {
return nil, err
}
@ -45,7 +45,7 @@ func (c *SQLConnectionProducer) registerDrivers(driverName string, credentials s
// GetCloudSQLAuthOptions takes a credentials JSON and returns
// a set of GCP CloudSQL options - always WithIAMAUthN, and then the appropriate file/JSON option.
func GetCloudSQLAuthOptions(credentials string, usePrivateIP bool) ([]cloudsqlconn.Option, error) {
func GetCloudSQLAuthOptions(credentials string, usePrivateIP bool, usePSC bool) ([]cloudsqlconn.Option, error) {
opts := []cloudsqlconn.Option{cloudsqlconn.WithIAMAuthN()}
if credentials != "" {
@ -56,5 +56,9 @@ func GetCloudSQLAuthOptions(credentials string, usePrivateIP bool) ([]cloudsqlco
opts = append(opts, cloudsqlconn.WithDefaultDialOptions(cloudsqlconn.WithPrivateIP()))
}
if usePSC {
opts = append(opts, cloudsqlconn.WithDefaultDialOptions(cloudsqlconn.WithPSC()))
}
return opts, nil
}

View File

@ -54,7 +54,8 @@ type SQLConnectionProducer struct {
MaxIdleConnections int `json:"max_idle_connections" mapstructure:"max_idle_connections" structs:"max_idle_connections"`
MaxConnectionLifetimeRaw interface{} `json:"max_connection_lifetime" mapstructure:"max_connection_lifetime" structs:"max_connection_lifetime"`
DisableEscaping bool `json:"disable_escaping" mapstructure:"disable_escaping" structs:"disable_escaping"`
usePrivateIP bool `json:"use_private_ip" mapstructure:"use_private_ip" structs:"use_private_ip"`
UsePrivateIP bool `json:"use_private_ip" mapstructure:"use_private_ip" structs:"use_private_ip"`
UsePSC bool `json:"use_psc" mapstructure:"use_psc" structs:"use_psc"`
SelfManaged bool `json:"self_managed" mapstructure:"self_managed" structs:"self_managed"`
// Username/Password is the default auth type when AuthType is not set
@ -204,7 +205,7 @@ func (c *SQLConnectionProducer) Init(ctx context.Context, conf map[string]interf
// however, the driver might store a credentials file, in which case the state stored by the driver is in
// fact critical to the proper function of the connection. So it needs to be registered here inside the
// ConnectionProducer init.
dialerCleanup, err := c.registerDrivers(c.cloudDriverName, c.ServiceAccountJSON, c.usePrivateIP)
dialerCleanup, err := c.registerDrivers(c.cloudDriverName, c.ServiceAccountJSON, c.UsePrivateIP, c.UsePSC)
if err != nil {
return nil, err
}

View File

@ -53,6 +53,12 @@ has a number of parameters to further configure a connection.
- `service_account_json` `(string: "")` - JSON encoded credentials for a GCP Service Account to use
for IAM authentication. Requires `auth_type` to be `gcp_iam`.
- `use_private_ip` `(boolean: false)` - Enables the option to connect to CloudSQL Instances with Private IP.
Requires `auth_type` to be `gcp_iam`.
- `use_psc` `(boolean: false)` - Enables the option to connect to CloudSQL Instances with Private Service Connect.
Requires `auth_type` to be `gcp_iam`.
- `tls_certificate_key` `(string: "")` - x509 certificate for connecting to the database.
This must be a PEM encoded version of the private key and the certificate combined.

View File

@ -74,6 +74,9 @@ has a number of parameters to further configure a connection.
- `use_private_ip` `(boolean: false)` - Enables the option to connect to CloudSQL Instances with Private IP.
Requires `auth_type` to be `gcp_iam`.
- `use_psc` `(boolean: false)` - Enables the option to connect to CloudSQL Instances with Private Service Connect.
Requires `auth_type` to be `gcp_iam`.
- `username_template` `(string)` - [Template](/vault/docs/concepts/username-templating) describing how
dynamic usernames are generated.

View File

@ -202,6 +202,8 @@ GRANT SELECT, CREATE, CREATE USER ON <database>.<object> TO "test-user"@"%" WITH
plugin_name="mysql-database-plugin" \
allowed_roles="my-role" \
connection_url="user@cloudsql-mysql(project:region:instance)/mysql" \
use_private_ip="false" \
use_psc="false" \
auth_type="gcp_iam"
```
@ -214,6 +216,8 @@ GRANT SELECT, CREATE, CREATE USER ON <database>.<object> TO "test-user"@"%" WITH
allowed_roles="my-role" \
connection_url="user@cloudsql-mysql(project:region:instance)/mysql" \
auth_type="gcp_iam" \
use_private_ip="false" \
use_psc="false" \
service_account_json="@my_credentials.json"
```

View File

@ -238,6 +238,7 @@ ALTER USER "<YOUR DB USERNAME>" WITH CREATEROLE;
allowed_roles="my-role" \
connection_url="host=project:us-west1:mydb user=test-user@project.iam dbname=postgres sslmode=disable" \
use_private_ip="false" \
use_psc="false" \
auth_type="gcp_iam"
```
@ -250,6 +251,7 @@ ALTER USER "<YOUR DB USERNAME>" WITH CREATEROLE;
allowed_roles="my-role" \
connection_url="host=project:region:instance user=test-user@project.iam dbname=postgres sslmode=disable" \
use_private_ip="false" \
use_psc="false" \
auth_type="gcp_iam" \
service_account_json="@my_credentials.json"
```