diff --git a/changelog/27889.txt b/changelog/27889.txt new file mode 100644 index 0000000000..2b4959a5d7 --- /dev/null +++ b/changelog/27889.txt @@ -0,0 +1,6 @@ +```release-note:improvement +secrets/database: Add PSC support for GCP CloudSQL MySQL and Postgresql +``` +```release-note:improvement +secrets/database: Add PrivateIP support for MySQL +``` diff --git a/plugins/database/mysql/connection_producer.go b/plugins/database/mysql/connection_producer.go index f35bfaf522..0c5ca5996c 100644 --- a/plugins/database/mysql/connection_producer.go +++ b/plugins/database/mysql/connection_producer.go @@ -37,6 +37,8 @@ type mySQLConnectionProducer struct { Password string `json:"password" mapstructure:"password" structs:"password"` AuthType string `json:"auth_type" mapstructure:"auth_type" structs:"auth_type"` ServiceAccountJSON string `json:"service_account_json" mapstructure:"service_account_json" structs:"service_account_json"` + UsePrivateIP bool `json:"use_private_ip" mapstructure:"use_private_ip" structs:"use_private_ip"` + UsePSC bool `json:"use_psc" mapstructure:"use_psc" structs:"use_psc"` TLSCertificateKeyData []byte `json:"tls_certificate_key" mapstructure:"tls_certificate_key" structs:"-"` TLSCAData []byte `json:"tls_ca" mapstructure:"tls_ca" structs:"-"` @@ -123,8 +125,11 @@ func (c *mySQLConnectionProducer) Init(ctx context.Context, conf map[string]inte } // validate auth_type if provided - if ok := connutil.ValidateAuthType(c.AuthType); !ok { - return nil, fmt.Errorf("invalid auth_type: %s", c.AuthType) + authType := c.AuthType + if authType != "" { + if ok := connutil.ValidateAuthType(authType); !ok { + return nil, fmt.Errorf("invalid auth_type %s provided", authType) + } } if c.AuthType == connutil.AuthTypeGCPIAM { @@ -137,7 +142,7 @@ func (c *mySQLConnectionProducer) Init(ctx context.Context, conf map[string]inte // however, the driver might store a credentials file, in which case the state stored by the driver is in // fact critical to the proper function of the connection. So it needs to be registered here inside the // ConnectionProducer init. - dialerCleanup, err := registerDriverMySQL(c.cloudDriverName, c.ServiceAccountJSON) + dialerCleanup, err := registerDriverMySQL(c.cloudDriverName, c.ServiceAccountJSON, c.UsePrivateIP, c.UsePSC) if err != nil { return nil, err } @@ -318,8 +323,8 @@ func (c *mySQLConnectionProducer) rewriteProtocolForGCP(inDSN string) (string, e return config.FormatDSN(), nil } -func registerDriverMySQL(driverName, credentials string) (cleanup func() error, err error) { - opts, err := connutil.GetCloudSQLAuthOptions(credentials, false) +func registerDriverMySQL(driverName, credentials string, usePrivateIP bool, usePSC bool) (cleanup func() error, err error) { + opts, err := connutil.GetCloudSQLAuthOptions(credentials, usePrivateIP, usePSC) if err != nil { return nil, err } diff --git a/plugins/database/mysql/mysql_test.go b/plugins/database/mysql/mysql_test.go index 72c2a125a5..df58d765a7 100644 --- a/plugins/database/mysql/mysql_test.go +++ b/plugins/database/mysql/mysql_test.go @@ -46,13 +46,16 @@ func TestMySQL_Initialize(t *testing.T) { } } -// TestMySQL_Initialize_CloudGCP validates the proper initialization of a MySQL backend pointing -// to a GCP CloudSQL MySQL instance. This expects some external setup (exact TBD) -func TestMySQL_Initialize_CloudGCP(t *testing.T) { - envConnURL := "CONNECTION_URL" - connURL := os.Getenv(envConnURL) - if connURL == "" { - t.Skipf("env var %s not set, skipping test", envConnURL) +// TestMySQL_Initialize_CloudGCP_Normal - test initialize in GCP with a normal connection. +// For the CloudGCP Normal, PrivateIP, and PSC tests, follow the instructions within the +// README at https://github.com/shinji62/vault-tf-psc-test for environment setup. +// The Terraform takes care of most steps. +func TestMySQL_Initialize_CloudGCP_Normal(t *testing.T) { + envNormalConnURL := "CLOUDGCP_NORMAL_CONNECTION_URL" + normalConnURL := os.Getenv(envNormalConnURL) + + if normalConnURL == "" { + t.Skipf("env var %s not set, skipping test", envNormalConnURL) } credStr := dbtesting.GetGCPTestCredentials(t) @@ -65,7 +68,7 @@ func TestMySQL_Initialize_CloudGCP(t *testing.T) { "empty auth type": { req: dbplugin.InitializeRequest{ Config: map[string]interface{}{ - "connection_url": connURL, + "connection_url": normalConnURL, "auth_type": "", }, }, @@ -73,7 +76,7 @@ func TestMySQL_Initialize_CloudGCP(t *testing.T) { "invalid auth type": { req: dbplugin.InitializeRequest{ Config: map[string]interface{}{ - "connection_url": connURL, + "connection_url": normalConnURL, "auth_type": "invalid", }, }, @@ -83,7 +86,7 @@ func TestMySQL_Initialize_CloudGCP(t *testing.T) { "JSON credentials": { req: dbplugin.InitializeRequest{ Config: map[string]interface{}{ - "connection_url": connURL, + "connection_url": normalConnURL, "auth_type": connutil.AuthTypeGCPIAM, "service_account_json": credStr, }, @@ -119,6 +122,159 @@ func TestMySQL_Initialize_CloudGCP(t *testing.T) { } } +// TestMySQL_Initialize_CloudGCP_PrivateIP - test initialize in GCP with a private IP. +// For the CloudGCP Normal, PrivateIP, and PSC tests, follow the instructions within the +// README at https://github.com/shinji62/vault-tf-psc-test for environment setup. +// The Terraform takes care of most steps. +func TestMySQL_Initialize_CloudGCP_PrivateIP(t *testing.T) { + envPrivateIPConnURL := "CLOUDGCP_PRIVATEIP_CONNECTION_URL" + privateIPConnURL := os.Getenv(envPrivateIPConnURL) + + if privateIPConnURL == "" { + t.Skipf("env var %s not set, skipping test", envPrivateIPConnURL) + } + credStr := dbtesting.GetGCPTestCredentials(t) + + tests := map[string]struct { + req dbplugin.InitializeRequest + wantErr bool + expectedError string + }{ + "empty auth type": { + req: dbplugin.InitializeRequest{ + Config: map[string]interface{}{ + "connection_url": privateIPConnURL, + "auth_type": "", + }, + }, + }, + "invalid auth type": { + req: dbplugin.InitializeRequest{ + Config: map[string]interface{}{ + "connection_url": privateIPConnURL, + "auth_type": "invalid", + }, + }, + wantErr: true, + expectedError: "invalid auth_type", + }, + "JSON credentials Private IP connection": { + req: dbplugin.InitializeRequest{ + Config: map[string]interface{}{ + "connection_url": privateIPConnURL, + "auth_type": connutil.AuthTypeGCPIAM, + "service_account_json": credStr, + "use_private_ip": true, + }, + VerifyConnection: true, + }, + }, + } + + for n, tc := range tests { + t.Run(n, func(t *testing.T) { + db := newMySQL(DefaultUserNameTemplate) + defer dbtesting.AssertClose(t, db) + _, err := db.Initialize(context.Background(), tc.req) + + if tc.wantErr { + if err == nil { + t.Fatalf("expected error but received nil") + } + + if !strings.Contains(err.Error(), tc.expectedError) { + t.Fatalf("expected error %s, got %s", tc.expectedError, err.Error()) + } + } else { + if err != nil { + t.Fatalf("expected no error, received %s", err) + } + + if !db.Initialized { + t.Fatal("Database should be initialized") + } + } + }) + } +} + +// TestMySQL_Initialize_CloudGCP_PSC - test initialize in GCP with private service connect. +// For the CloudGCP Normal, PrivateIP, and PSC tests, follow the instructions within the +// README at https://github.com/shinji62/vault-tf-psc-test for environment setup. +// The Terraform takes care of most steps. +func TestMySQL_Initialize_CloudGCP_PSC(t *testing.T) { + envPSCConnURL := "CLOUDGCP_PSC_CONNECTION_URL" + pscConnURL := os.Getenv(envPSCConnURL) + + if pscConnURL == "" { + t.Skipf("env var %s not set, skipping test", pscConnURL) + } + + credStr := dbtesting.GetGCPTestCredentials(t) + + tests := map[string]struct { + req dbplugin.InitializeRequest + wantErr bool + expectedError string + }{ + "empty auth type": { + req: dbplugin.InitializeRequest{ + Config: map[string]interface{}{ + "connection_url": pscConnURL, + "auth_type": "", + }, + }, + }, + "invalid auth type": { + req: dbplugin.InitializeRequest{ + Config: map[string]interface{}{ + "connection_url": pscConnURL, + "auth_type": "invalid", + }, + }, + wantErr: true, + expectedError: "invalid auth_type", + }, + "JSON credentials Private Service Connect connection": { + req: dbplugin.InitializeRequest{ + Config: map[string]interface{}{ + "connection_url": pscConnURL, + "auth_type": connutil.AuthTypeGCPIAM, + "service_account_json": credStr, + "use_psc": true, + }, + VerifyConnection: true, + }, + }, + } + + for n, tc := range tests { + t.Run(n, func(t *testing.T) { + db := newMySQL(DefaultUserNameTemplate) + defer dbtesting.AssertClose(t, db) + _, err := db.Initialize(context.Background(), tc.req) + + if tc.wantErr { + if err == nil { + t.Fatalf("expected error but received nil") + } + + if !strings.Contains(err.Error(), tc.expectedError) { + t.Fatalf("expected error %s, got %s", tc.expectedError, err.Error()) + } + } else { + if err != nil { + t.Fatalf("expected no error, received %s", err) + } + + if !db.Initialized { + t.Fatal("Database should be initialized") + } + } + }) + } +} + func testInitialize(t *testing.T, rootPassword string) { cleanup, connURL := mysqlhelper.PrepareTestContainer(t, false, rootPassword) defer cleanup() diff --git a/sdk/database/helper/connutil/cloudsql.go b/sdk/database/helper/connutil/cloudsql.go index f6cbba1d24..6398d3c479 100644 --- a/sdk/database/helper/connutil/cloudsql.go +++ b/sdk/database/helper/connutil/cloudsql.go @@ -23,13 +23,13 @@ func (c *SQLConnectionProducer) getCloudSQLDriverType() (string, error) { return driverType, nil } -func (c *SQLConnectionProducer) registerDrivers(driverName string, credentials string, usePrivateIP bool) (func() error, error) { +func (c *SQLConnectionProducer) registerDrivers(driverName string, credentials string, usePrivateIP bool, usePSC bool) (func() error, error) { typ, err := c.getCloudSQLDriverType() if err != nil { return nil, err } - opts, err := GetCloudSQLAuthOptions(credentials, usePrivateIP) + opts, err := GetCloudSQLAuthOptions(credentials, usePrivateIP, usePSC) if err != nil { return nil, err } @@ -45,7 +45,7 @@ func (c *SQLConnectionProducer) registerDrivers(driverName string, credentials s // GetCloudSQLAuthOptions takes a credentials JSON and returns // a set of GCP CloudSQL options - always WithIAMAUthN, and then the appropriate file/JSON option. -func GetCloudSQLAuthOptions(credentials string, usePrivateIP bool) ([]cloudsqlconn.Option, error) { +func GetCloudSQLAuthOptions(credentials string, usePrivateIP bool, usePSC bool) ([]cloudsqlconn.Option, error) { opts := []cloudsqlconn.Option{cloudsqlconn.WithIAMAuthN()} if credentials != "" { @@ -56,5 +56,9 @@ func GetCloudSQLAuthOptions(credentials string, usePrivateIP bool) ([]cloudsqlco opts = append(opts, cloudsqlconn.WithDefaultDialOptions(cloudsqlconn.WithPrivateIP())) } + if usePSC { + opts = append(opts, cloudsqlconn.WithDefaultDialOptions(cloudsqlconn.WithPSC())) + } + return opts, nil } diff --git a/sdk/database/helper/connutil/sql.go b/sdk/database/helper/connutil/sql.go index 3df692dcec..131b3b7fd8 100644 --- a/sdk/database/helper/connutil/sql.go +++ b/sdk/database/helper/connutil/sql.go @@ -54,7 +54,8 @@ type SQLConnectionProducer struct { MaxIdleConnections int `json:"max_idle_connections" mapstructure:"max_idle_connections" structs:"max_idle_connections"` MaxConnectionLifetimeRaw interface{} `json:"max_connection_lifetime" mapstructure:"max_connection_lifetime" structs:"max_connection_lifetime"` DisableEscaping bool `json:"disable_escaping" mapstructure:"disable_escaping" structs:"disable_escaping"` - usePrivateIP bool `json:"use_private_ip" mapstructure:"use_private_ip" structs:"use_private_ip"` + UsePrivateIP bool `json:"use_private_ip" mapstructure:"use_private_ip" structs:"use_private_ip"` + UsePSC bool `json:"use_psc" mapstructure:"use_psc" structs:"use_psc"` SelfManaged bool `json:"self_managed" mapstructure:"self_managed" structs:"self_managed"` // Username/Password is the default auth type when AuthType is not set @@ -204,7 +205,7 @@ func (c *SQLConnectionProducer) Init(ctx context.Context, conf map[string]interf // however, the driver might store a credentials file, in which case the state stored by the driver is in // fact critical to the proper function of the connection. So it needs to be registered here inside the // ConnectionProducer init. - dialerCleanup, err := c.registerDrivers(c.cloudDriverName, c.ServiceAccountJSON, c.usePrivateIP) + dialerCleanup, err := c.registerDrivers(c.cloudDriverName, c.ServiceAccountJSON, c.UsePrivateIP, c.UsePSC) if err != nil { return nil, err } diff --git a/website/content/api-docs/secret/databases/mysql-maria.mdx b/website/content/api-docs/secret/databases/mysql-maria.mdx index bdce2662bd..eae1b568db 100644 --- a/website/content/api-docs/secret/databases/mysql-maria.mdx +++ b/website/content/api-docs/secret/databases/mysql-maria.mdx @@ -53,6 +53,12 @@ has a number of parameters to further configure a connection. - `service_account_json` `(string: "")` - JSON encoded credentials for a GCP Service Account to use for IAM authentication. Requires `auth_type` to be `gcp_iam`. +- `use_private_ip` `(boolean: false)` - Enables the option to connect to CloudSQL Instances with Private IP. + Requires `auth_type` to be `gcp_iam`. + +- `use_psc` `(boolean: false)` - Enables the option to connect to CloudSQL Instances with Private Service Connect. + Requires `auth_type` to be `gcp_iam`. + - `tls_certificate_key` `(string: "")` - x509 certificate for connecting to the database. This must be a PEM encoded version of the private key and the certificate combined. diff --git a/website/content/api-docs/secret/databases/postgresql.mdx b/website/content/api-docs/secret/databases/postgresql.mdx index 9ddfece16c..b03742a462 100644 --- a/website/content/api-docs/secret/databases/postgresql.mdx +++ b/website/content/api-docs/secret/databases/postgresql.mdx @@ -74,6 +74,9 @@ has a number of parameters to further configure a connection. - `use_private_ip` `(boolean: false)` - Enables the option to connect to CloudSQL Instances with Private IP. Requires `auth_type` to be `gcp_iam`. +- `use_psc` `(boolean: false)` - Enables the option to connect to CloudSQL Instances with Private Service Connect. + Requires `auth_type` to be `gcp_iam`. + - `username_template` `(string)` - [Template](/vault/docs/concepts/username-templating) describing how dynamic usernames are generated. diff --git a/website/content/docs/secrets/databases/mysql-maria.mdx b/website/content/docs/secrets/databases/mysql-maria.mdx index d1dc3c5c7d..4d3107293b 100644 --- a/website/content/docs/secrets/databases/mysql-maria.mdx +++ b/website/content/docs/secrets/databases/mysql-maria.mdx @@ -202,6 +202,8 @@ GRANT SELECT, CREATE, CREATE USER ON . TO "test-user"@"%" WITH plugin_name="mysql-database-plugin" \ allowed_roles="my-role" \ connection_url="user@cloudsql-mysql(project:region:instance)/mysql" \ + use_private_ip="false" \ + use_psc="false" \ auth_type="gcp_iam" ``` @@ -214,6 +216,8 @@ GRANT SELECT, CREATE, CREATE USER ON . TO "test-user"@"%" WITH allowed_roles="my-role" \ connection_url="user@cloudsql-mysql(project:region:instance)/mysql" \ auth_type="gcp_iam" \ + use_private_ip="false" \ + use_psc="false" \ service_account_json="@my_credentials.json" ``` diff --git a/website/content/docs/secrets/databases/postgresql.mdx b/website/content/docs/secrets/databases/postgresql.mdx index 579d32b1c5..8752a35cf4 100644 --- a/website/content/docs/secrets/databases/postgresql.mdx +++ b/website/content/docs/secrets/databases/postgresql.mdx @@ -238,6 +238,7 @@ ALTER USER "" WITH CREATEROLE; allowed_roles="my-role" \ connection_url="host=project:us-west1:mydb user=test-user@project.iam dbname=postgres sslmode=disable" \ use_private_ip="false" \ + use_psc="false" \ auth_type="gcp_iam" ``` @@ -250,6 +251,7 @@ ALTER USER "" WITH CREATEROLE; allowed_roles="my-role" \ connection_url="host=project:region:instance user=test-user@project.iam dbname=postgres sslmode=disable" \ use_private_ip="false" \ + use_psc="false" \ auth_type="gcp_iam" \ service_account_json="@my_credentials.json" ```