CE changes for plugin download (#30927)

* ce changes for https://github.com/hashicorp/vault-enterprise/pull/8193

* lower case enterprise only errors

---------

Co-authored-by: Ben Ash <bash@hashicorp.com>

api/sys_plugins.go

@@ -215,6 +215,9 @@ type RegisterPluginInput struct {
 	// Env specifies a list of key=value pairs to add to the plugin's environment
 	// variables.
 	Env []string `json:"env,omitempty"`
+
+	// Download the plugin when set to true. This is only applicable for external plugins.
+	Download bool `json:"download,omitempty"`
 }
 
 // RegisterPlugin wraps RegisterPluginWithContext using context.Background.

builtin/logical/database/version_wrapper.go

@@ -35,6 +35,7 @@ func newDatabaseWrapper(ctx context.Context, pluginName string, pluginVersion st
 	if versions.IsBuiltinVersion(pluginVersion) {
 		pluginVersion = ""
 	}
+
 	newDB, err := v5.PluginFactoryVersion(ctx, pluginName, pluginVersion, sys, logger)
 	if err == nil {
 		dbw = databaseVersionWrapper{

sdk/database/dbplugin/v5/plugin_client_test.go

@@ -160,3 +160,7 @@ func (m *mockRunnerUtil) MlockEnabled() bool {
 func (m *mockRunnerUtil) ClusterID(ctx context.Context) (string, error) {
 	return "clusterid", nil
 }
+
+func (m *mockRunnerUtil) DownloadExtractVerifyPlugin(_ context.Context, _ *pluginutil.PluginRunner) error {
+	return nil
+}

sdk/database/dbplugin/v5/plugin_factory.go

@@ -49,6 +49,13 @@ func PluginFactoryVersion(ctx context.Context, pluginName string, pluginVersion
 		transport = "builtin"
 
 	} else {
+		if pluginRunner.Download {
+			if err = sys.DownloadExtractVerifyPlugin(ctx, pluginRunner); err != nil {
+				return nil, fmt.Errorf("failed to extract and verify plugin=%q version=%q: %w",
+					pluginRunner.Name, pluginRunner.Version, err)
+			}
+		}
+
 		config := pluginutil.PluginClientConfig{
 			Name:            pluginName,
 			PluginType:      consts.PluginTypeDatabase,

sdk/helper/pluginutil/run_config_test.go

@@ -447,6 +447,10 @@ func (m *mockRunnerUtil) ClusterID(ctx context.Context) (string, error) {
 	return "1234", nil
 }
 
+func (m *mockRunnerUtil) DownloadExtractVerifyPlugin(ctx context.Context, _ *PluginRunner) error {
+	return nil
+}
+
 func TestContainerConfig(t *testing.T) {
 	dummySHA, err := hex.DecodeString("abc123")
 	if err != nil {

sdk/helper/pluginutil/runner.go

@@ -45,6 +45,7 @@ type RunnerUtil interface {
 	MlockEnabled() bool
 	VaultVersion(ctx context.Context) (string, error)
 	ClusterID(ctx context.Context) (string, error)
+	DownloadExtractVerifyPlugin(ctx context.Context, pr *PluginRunner) error
 }
 
 // LookRunnerUtil defines the functions for both Looker and Wrapper
@@ -75,6 +76,7 @@ type PluginRunner struct {
 	Sha256         []byte                      `json:"sha256" structs:"sha256"`
 	Builtin        bool                        `json:"builtin" structs:"builtin"`
 	Tier           consts.PluginTier           `json:"tier" structs:"tier"`
+	Download       bool                        `json:"download" structs:"download"`
 	BuiltinFactory func() (interface{}, error) `json:"-" structs:"-"`
 	RuntimeConfig  *prutil.PluginRuntimeConfig `json:"-" structs:"-"`
 	Tmpdir         string                      `json:"-" structs:"-"`
@@ -111,6 +113,8 @@ type SetPluginInput struct {
 	Args     []string
 	Env      []string
 	Sha256   []byte
+	Tier     consts.PluginTier
+	Download bool
 }
 
 // Run takes a wrapper RunnerUtil instance along with the go-plugin parameters and

sdk/logical/system_view.go

@@ -111,6 +111,9 @@ type SystemView interface {
 	// credential from the Rotation Manager.
 	// NOTE: This method is intended for use only by HashiCorp Vault Enterprise plugins.
 	DeregisterRotationJob(ctx context.Context, req *rotation.RotationJobDeregisterRequest) error
+
+	// ExtractVerifyPlugin extracts and verifies the plugin artifact
+	DownloadExtractVerifyPlugin(ctx context.Context, plugin *pluginutil.PluginRunner) error
 }
 
 type PasswordPolicy interface {
@@ -305,3 +308,7 @@ func (d StaticSystemView) RegisterRotationJob(_ context.Context, _ *rotation.Rot
 func (d StaticSystemView) DeregisterRotationJob(_ context.Context, _ *rotation.RotationJobDeregisterRequest) (err error) {
 	return errors.New("DeregisterRotationJob is not implemented in StaticSystemView")
 }
+
+func (d StaticSystemView) DownloadExtractVerifyPlugin(_ context.Context, _ *pluginutil.PluginRunner) error {
+	return errors.New("DownloadExtractVerifyPlugin is not implemented in StaticSystemView")
+}

sdk/plugin/grpc_system.go

@@ -508,3 +508,7 @@ func (s *gRPCSystemViewServer) DeregisterRotationJob(ctx context.Context, req *p
 
 	return &pb.Empty{}, nil
 }
+
+func (s *gRPCSystemViewClient) DownloadExtractVerifyPlugin(_ context.Context, _ *pluginutil.PluginRunner) error {
+	return fmt.Errorf("cannot call DownloadExtractVerifyPlugin from a plugin backend")
+}

vault/auth.go

@@ -1006,6 +1006,13 @@ func (c *Core) newCredentialBackend(ctx context.Context, entry *MountEntry, sysV
 			runningSha = hex.EncodeToString(plug.Sha256)
 		}
 
+		if plug.Download {
+			if err = sysView.DownloadExtractVerifyPlugin(ctx, plug); err != nil {
+				return nil, fmt.Errorf("failed to extract and verify plugin=%q version=%q before mounting: %w",
+					plug.Name, plug.Version, err)
+			}
+		}
+
 		factory = plugin.Factory
 		if !plug.Builtin {
 			factory = wrapFactoryCheckPerms(c, plugin.Factory)

vault/core.go

@@ -561,11 +561,13 @@ type Core struct {
 	introspectionEnabled     bool
 	introspectionEnabledLock sync.Mutex
 
-	// pluginDirectory is the location vault will look for plugin binaries
+	// pluginDirectory is the location vault will look for old style plugins, it is
+	// the root for all plugin artifacts.
 	pluginDirectory string
-	// pluginTmpdir is the location vault will use for containerized plugin
+
+	// containerPluginTmpdir is the location vault will use for containerized plugin
 	// temporary files
-	pluginTmpdir string
+	containerPluginTmpdir string
 
 	// pluginFileUid is the uid of the plugin files and directory
 	pluginFileUid int
@@ -1274,7 +1276,7 @@ func NewCore(conf *CoreConfig) (*Core, error) {
 		}
 	}
 	if conf.PluginTmpdir != "" {
-		c.pluginTmpdir, err = filepath.Abs(conf.PluginTmpdir)
+		c.containerPluginTmpdir, err = filepath.Abs(conf.PluginTmpdir)
 		if err != nil {
 			return nil, fmt.Errorf("core setup failed, could not verify plugin tmpdir: %w", err)
 		}
@@ -2616,7 +2618,7 @@ func (c *Core) setupPluginCatalog(ctx context.Context) error {
 		BuiltinRegistry:      c.builtinRegistry,
 		CatalogView:          NewBarrierView(c.barrier, pluginCatalogPath),
 		PluginDirectory:      c.pluginDirectory,
-		Tmpdir:               c.pluginTmpdir,
+		Tmpdir:               c.containerPluginTmpdir,
 		EnableMlock:          c.enableMlock,
 		PluginRuntimeCatalog: c.pluginRuntimeCatalog,
 	})

vault/core_stubs_oss.go

@@ -7,6 +7,7 @@ package vault
 
 import (
 	"context"
+	"fmt"
 )
 
 //go:generate go run github.com/hashicorp/vault/tools/stubmaker
@@ -120,3 +121,21 @@ func (c *Core) GetReplicationLagMillisIgnoreErrs() int64 { return 0 }
 func (c *Core) ReloadOverloadController() {}
 
 func (c *Core) EntSetupUIDefaultAuth(ctx context.Context) error { return nil }
+
+// entGetPluginCacheDir returns empty string and an error indicating that this is an
+// enterprise-only feature. This is used to prevent the use of the plugin cache
+func (c *Core) entGetPluginCacheDir() (string, error) {
+	return "", fmt.Errorf("enterprise only feature")
+}
+
+// entGetPluginRuntimeDir returns empty string and an error indicating that this is an
+// enterprise-only feature
+func (c *Core) entGetPluginRuntimeDir() (string, error) {
+	return "", fmt.Errorf("enterprise only feature")
+}
+
+// entJoinPluginDir returns empty string and an error indicating that this is an
+// enterprise-only feature
+func (c *Core) entJoinPluginDir(_ string) (string, error) {
+	return "", fmt.Errorf("enterprise only feature")
+}

vault/dynamic_system_view_stubs_oss.go

@@ -0,0 +1,18 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+//go:build !enterprise
+
+package vault
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/hashicorp/vault/sdk/helper/pluginutil"
+)
+
+// DownloadExtractVerifyPlugin returns an error as this is an enterprise only feature
+func (d dynamicSystemView) DownloadExtractVerifyPlugin(_ context.Context, _ *pluginutil.PluginRunner) error {
+	return fmt.Errorf("enterprise only feature")
+}

vault/extended_system_view.go

@@ -142,3 +142,7 @@ func (e extendedSystemViewImpl) DeregisterWellKnownRedirect(ctx context.Context,
 func (e extendedSystemViewImpl) GetPinnedPluginVersion(ctx context.Context, pluginType consts.PluginType, pluginName string) (*pluginutil.PinnedVersion, error) {
 	return e.core.pluginCatalog.GetPinnedVersion(ctx, pluginType, pluginName)
 }
+
+func (e extendedSystemViewImpl) DownloadExtractVerifyPlugin(_ context.Context, _ *pluginutil.PluginRunner) error {
+	return fmt.Errorf("cannot call DownloadExtractVerifyPlugin from a plugin backend")
+}

vault/mount.go

@@ -1742,6 +1742,13 @@ func (c *Core) newLogicalBackend(ctx context.Context, entry *MountEntry, sysView
 			runningSha = hex.EncodeToString(plug.Sha256)
 		}
 
+		if plug.Download {
+			if err = sysView.DownloadExtractVerifyPlugin(ctx, plug); err != nil {
+				return nil, fmt.Errorf("failed to extract and verify plugin=%q version=%q before mounting: %w",
+					plug.Name, plug.Version, err)
+			}
+		}
+
 		factory = plugin.Factory
 		if !plug.Builtin {
 			factory = wrapFactoryCheckPerms(c, factory)

vault/mount_stubs_oss.go

@@ -0,0 +1,17 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+// go:build !enterprise
+
+package vault
+
+import (
+	"context"
+
+	"github.com/hashicorp/vault/sdk/helper/pluginutil"
+)
+
+func entExtractVerifyPlugin(context.Context, *pluginutil.PluginRunner) error {
+	// Do nothing in OSS
+	return nil
+}

vault/plugincatalog/plugin_artifact.go

@@ -5,20 +5,45 @@ package plugincatalog
 
 import (
 	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
-	"io"
 	"os"
 	"path"
 	"runtime"
 	"strings"
+	"sync"
 
 	"github.com/ProtonMail/gopenpgp/v3/crypto"
 )
 
-// hashiCorpPGPPubKey is HashiCorp's PGP public key at https://www.hashicorp.com/.well-known/pgp-key.txt.
-// This key is used to verify the authenticity of HashiCorp plugins.
-const hashiCorpPGPPubKey = `
+const (
+	extractedArtifactDirFmt = "%s_%s_%s_%s" // vault-plugin-database-oracle_1.2.3+ent_linux_amd64
+	metadataFile            = "metadata.json"
+	metadataSig             = "metadata.json.sig"
+)
+
+var (
+	once     sync.Once
+	verifier crypto.PGPVerify
+
+	errExtractedArtifactDirNotFound = errors.New("extracted artifact directory not found")
+	errReadMetadata                 = errors.New("failed to read metadata")
+	errReadMetadataSig              = errors.New("failed to read metadata signature")
+	errReadPlugin                   = errors.New("failed to read plugin binary")
+	errReadPluginMetadata           = errors.New("failed to read plugin metadata")
+	errReadPluginPGPSig             = errors.New("failed to read plugin binary PGP signature")
+	errVerifyMetadataSig            = errors.New("failed to verify metadata detached signature")
+	errVerifyPluginSig              = errors.New("failed to verify plugin binary PGP signature")
+)
+
+func load() error {
+	var errs error
+	once.Do(func() {
+		// hashiCorpPGPPubKey is HashiCorp's PGP public key
+		// at https://www.hashicorp.com/.well-known/pgp-key.txt.
+		// This key is used to verify the authenticity of HashiCorp plugins.
+		const hashiCorpPGPPubKey = `
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 
 mQINBGB9+xkBEACabYZOWKmgZsHTdRDiyPJxhbuUiKX65GUWkyRMJKi/1dviVxOX
@@ -143,26 +168,26 @@ ZF5q4h4I33PSGDdSvGXn9UMY5Isjpg==
 -----END PGP PUBLIC KEY BLOCK-----
 `
 
-var defaultPGPPubKey = hashiCorpPGPPubKey
+		pgp := crypto.PGP()
+		key, err := crypto.NewKeyFromArmored(hashiCorpPGPPubKey)
+		if err != nil {
+			errs = errors.Join(errs, err)
+		}
 
-const (
-	extractedArtifactDirFmt = "%s_%s_%s_%s" // vault-plugin-database-oracle_1.2.3+ent_linux_amd64
-	metadataFile            = "metadata.json"
-	metadataSig             = "metadata.json.sig"
-)
+		verifier, err = pgp.Verify().VerificationKey(key).New()
+		if err != nil {
+			errs = errors.Join(errs, err)
+		}
+	})
 
-var (
-	errExtractedArtifactDirNotFound = errors.New("extracted artifact directory not found")
-	errReadMetadata                 = errors.New("failed to read metadata")
-	errReadMetadataSig              = errors.New("failed to read metadata signature")
-	errReadPlugin                   = errors.New("failed to read plugin binary")
-	errReadPluginMetadata           = errors.New("failed to read plugin metadata")
-	errReadPluginPGPSig             = errors.New("failed to read plugin binary PGP signature")
-	errVerifyMetadataSig            = errors.New("failed to verify metadata detached signature")
-	errVerifyPluginSig              = errors.New("failed to verify plugin binary PGP signature")
-)
+	if verifier == nil {
+		errs = errors.Join(errs, errors.New("verifier is nil after initialization"))
+	}
 
-func getExtractedArtifactDir(pluginName, pluginVersion string) string {
+	return errs
+}
+
+func GetExtractedArtifactDir(pluginName, pluginVersion string) string {
 	if strings.HasPrefix(pluginVersion, "v") {
 		pluginVersion = pluginVersion[1:]
 	}
@@ -170,28 +195,32 @@ func getExtractedArtifactDir(pluginName, pluginVersion string) string {
 	return fmt.Sprintf(extractedArtifactDirFmt, pluginName, pluginVersion, runtime.GOOS, runtime.GOARCH)
 }
 
-func verifyPlugin(pluginDir, pluginName, pubKey string) (*pluginMetadata, error) {
-	// verify the extracted plugin artifact directory structure and each file inside
-	// vault-plugin-secrets-example_1.2.3+ent_darwin_arm64/EULA.txt (optional)
-	// vault-plugin-secrets-example_1.2.3+ent_darwin_arm64/TermsOfEvaluation.txt (optional)
-	// vault-plugin-secrets-example_1.2.3+ent_darwin_arm64/LICENSE (optional)
-	// vault-plugin-secrets-example_1.2.3+ent_darwin_arm64/vault-plugin-secrets-example
-	// vault-plugin-secrets-example_1.2.3+ent_darwin_arm64/metadata.json
-	// vault-plugin-secrets-example_1.2.3+ent_darwin_arm64/metadata.json.sig
+type verifyFunc func(data, sig []byte) error
 
-	if _, err := os.Stat(pluginDir); os.IsNotExist(err) {
-		return nil, fmt.Errorf("%w: %w", errExtractedArtifactDirNotFound, err)
+func verifyPGPSignatureDetached(data, sig []byte) error {
+	if err := load(); err != nil {
+		return fmt.Errorf("failed to load verifier: %w", err)
 	}
 
-	pgp := crypto.PGP()
-	key, err := crypto.NewKeyFromArmored(pubKey)
+	verifyResult, err := verifier.VerifyDetached(data, sig, crypto.Armor)
 	if err != nil {
-		return nil, err
+		return fmt.Errorf("failed to verify data: %w", err)
+	}
+	if sigErr := verifyResult.SignatureError(); sigErr != nil {
+		return fmt.Errorf("unexpected signature error: %w", sigErr)
 	}
 
-	verifier, err := pgp.Verify().VerificationKey(key).New()
-	if err != nil {
-		return nil, err
+	return nil
+}
+
+func VerifyPlugin(pluginDir string, pluginName string, verifyFunc verifyFunc) (*PluginMetadata, error) {
+	if st, err := os.Stat(pluginDir); err != nil {
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("%w: %s", errExtractedArtifactDirNotFound, pluginDir)
+		}
+		return nil, fmt.Errorf("unexpected err %w", err)
+	} else if !st.IsDir() {
+		return nil, fmt.Errorf("%w: %s is not a directory", errExtractedArtifactDirNotFound, pluginDir)
 	}
 
 	// verify metadata.json is untampered
@@ -206,15 +235,12 @@ func verifyPlugin(pluginDir, pluginName, pubKey string) (*pluginMetadata, error)
 		return nil, fmt.Errorf("%w: %w", errReadMetadataSig, err)
 	}
 
-	verifyResult, err := verifier.VerifyDetached(metadataBytes, metadataSigBytes, crypto.Armor)
-	if err != nil {
+	if err := verifyFunc(metadataBytes, metadataSigBytes); err != nil {
 		return nil, fmt.Errorf("%w: %w", errVerifyMetadataSig, err)
 	}
-	if sigErr := verifyResult.SignatureError(); sigErr != nil {
-		return nil, fmt.Errorf("%w: %w", errVerifyMetadataSig, sigErr)
-	}
 
-	// verify plugin binary is untampred
+	// verify plugin binary is untampered
+	// TODO: We should update this to not read in the whole file into memory to be more efficient. Reference the secure-plugin-api's HashFile function.
 	pluginBytes, err := os.ReadFile(path.Join(pluginDir, pluginName))
 	if err != nil {
 		return nil, fmt.Errorf("%w: %w", errReadPlugin, err)
@@ -225,28 +251,17 @@ func verifyPlugin(pluginDir, pluginName, pubKey string) (*pluginMetadata, error)
 		return nil, fmt.Errorf("%w: %w", errReadPluginMetadata, err)
 	}
 
-	verifyResult, err = verifier.VerifyDetached(pluginBytes, []byte(metadata.Plugin.PGPSig), crypto.Armor)
-	if err != nil {
+	if err := verifyFunc(pluginBytes, []byte(metadata.Plugin.PGPSig)); err != nil {
 		return nil, fmt.Errorf("%w: %w", errVerifyPluginSig, err)
 	}
-	if sigErr := verifyResult.SignatureError(); sigErr != nil {
-		return nil, fmt.Errorf("%w: %w", errVerifyPluginSig, sigErr)
+
+	// TODO: Once we set sha256 on the metadata file from releases.hashicorp.com,
+	// we should compare the file hash with the metadata file's sha256.
+	hasher := sha256.New()
+	if _, err := hasher.Write(pluginBytes); err != nil {
+		return nil, fmt.Errorf("failed to hash plugin binary: %w", err)
 	}
+	metadata.Plugin.Sha256 = hex.EncodeToString(hasher.Sum(nil))
 
 	return metadata, nil
 }
-
-func pluginSHA256Sum(path string) ([]byte, error) {
-	file, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	hash := sha256.New()
-	if _, err = io.Copy(hash, file); err != nil {
-		return nil, err
-	}
-
-	return hash.Sum(nil), nil
-}

vault/plugincatalog/plugin_artifact_test.go

@@ -4,149 +4,13 @@
 package plugincatalog
 
 import (
-	"errors"
 	"fmt"
-	"os"
-	"path"
-	"path/filepath"
 	"runtime"
 	"testing"
 
-	"github.com/ProtonMail/gopenpgp/v3/crypto"
-	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/stretchr/testify/assert"
 )
 
-// Test_verifyPlugin tests the verifyPlugin function.
-func Test_verifyPlugin(t *testing.T) {
-	t.Parallel()
-
-	type args struct {
-		pluginName    string
-		pluginVersion string
-		pluginType    consts.PluginType
-	}
-	tests := []struct {
-		name        string
-		args        args
-		expectedErr error
-	}{
-		{
-			name: "success",
-			args: args{
-				pluginName:    "vault-plugin-auth-example",
-				pluginVersion: "0.1.1+ent",
-				pluginType:    consts.PluginTypeCredential,
-			},
-			expectedErr: nil,
-		},
-		{
-			name: "missing metadata",
-			args: args{
-				pluginName:    "vault-plugin-auth-example",
-				pluginVersion: "0.1.2+ent",
-				pluginType:    consts.PluginTypeCredential,
-			},
-			expectedErr: errReadMetadata,
-		},
-		{
-			name: "missing metadata signature",
-			args: args{
-				pluginName:    "vault-plugin-auth-example",
-				pluginVersion: "0.1.3+ent",
-				pluginType:    consts.PluginTypeCredential,
-			},
-			expectedErr: errReadMetadataSig,
-		},
-		{
-			name: "bad metadata signature verify",
-			args: args{
-				pluginName:    "vault-plugin-secret-example",
-				pluginVersion: "0.1.4+ent",
-				pluginType:    consts.PluginTypeSecrets,
-			},
-			expectedErr: errVerifyMetadataSig,
-		},
-		{
-			name: "missing plugin binary",
-			args: args{
-				pluginName:    "vault-plugin-database-example",
-				pluginVersion: "0.1.5+ent",
-				pluginType:    consts.PluginTypeDatabase,
-			},
-			expectedErr: errReadPlugin,
-		},
-		{
-			name: "bad plugin binary signature verify",
-			args: args{
-				pluginName:    "vault-plugin-database-example",
-				pluginVersion: "0.1.6+ent",
-				pluginType:    consts.PluginTypeDatabase,
-			},
-			expectedErr: errVerifyPluginSig,
-		},
-		{
-			name: "bad extracted artifact directory",
-			args: args{
-				pluginName:    "vault-plugin-database-example",
-				pluginVersion: "0.1.6+ent",
-				pluginType:    consts.PluginTypeDatabase,
-			},
-			expectedErr: errExtractedArtifactDirNotFound,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			privKey, pubKeyArmored := generatePGPKeyPair(t)
-
-			contents := generatePluginArtifactContents(t, tt.args.pluginName,
-				tt.args.pluginVersion, tt.args.pluginType, !errors.Is(tt.expectedErr, errReadPluginPGPSig), privKey)
-
-			actualExtractedArtifactDir := getExtractedArtifactDir(tt.args.pluginName, tt.args.pluginVersion)
-			switch {
-			case tt.expectedErr == nil:
-			case errors.Is(tt.expectedErr, errReadPluginPGPSig):
-				// no-op
-			case errors.Is(tt.expectedErr, errReadMetadata):
-				delete(contents, metadataFile)
-			case errors.Is(tt.expectedErr, errReadMetadataSig):
-				delete(contents, metadataSig)
-			case errors.Is(tt.expectedErr, errVerifyMetadataSig):
-				contents[metadataFile] = []byte(`{"will not" : "match signature"}`)
-			case errors.Is(tt.expectedErr, errReadPlugin):
-				delete(contents, tt.args.pluginName)
-			case errors.Is(tt.expectedErr, errVerifyPluginSig):
-				contents[tt.args.pluginName] = []byte("will not match signature")
-			case errors.Is(tt.expectedErr, errExtractedArtifactDirNotFound):
-				actualExtractedArtifactDir += "not_found"
-			default:
-				t.Fatalf("unexpected error: %v", tt.expectedErr)
-			}
-
-			tempDir := t.TempDir()
-			actualPluginDir := filepath.Join(tempDir, actualExtractedArtifactDir)
-			err := os.MkdirAll(actualPluginDir, 0o755)
-			assert.NoError(t, err, "expected successful create extracted plugin directory")
-
-			// Write the files to the extracted plugin directory
-			for name, content := range contents {
-				err = os.WriteFile(filepath.Join(actualPluginDir, name), content, 0o644)
-				assert.NoError(t, err, "expected successful file write")
-			}
-			var metadata *pluginMetadata
-			metadata, err = verifyPlugin(path.Join(tempDir, getExtractedArtifactDir(tt.args.pluginName, tt.args.pluginVersion)),
-				tt.args.pluginName, pubKeyArmored)
-			assert.ErrorIs(t, err, tt.expectedErr, "expected verify plugin error to match")
-
-			if tt.expectedErr == nil {
-				assert.NotNil(t, metadata)
-				assert.Equal(t, tt.args.pluginName, metadata.Plugin.Name)
-				assert.Equal(t, tt.args.pluginVersion, metadata.Plugin.Version)
-			}
-		})
-	}
-}
-
 // Test_getExtractedArtifactDir tests the getExtractedArtifactDir function.
 func Test_getExtractedArtifactDir(t *testing.T) {
 	t.Parallel()
@@ -173,18 +37,13 @@ func Test_getExtractedArtifactDir(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			assert.Equal(t, tt.want, getExtractedArtifactDir(tt.args.command, tt.args.version))
+			assert.Equal(t, tt.want, GetExtractedArtifactDir(tt.args.command, tt.args.version))
 		})
 	}
 }
 
-// TestPluginCatalog_hashiCorpPubPGPKey tests hashiCorpPubPGPKey read
-// and verification key creation.
-func TestPluginCatalog_hashiCorpPubPGPKey(t *testing.T) {
-	pgp := crypto.PGP()
-	key, err := crypto.NewKeyFromArmored(hashiCorpPGPPubKey)
-	assert.NoError(t, err)
-
-	_, err = pgp.Verify().VerificationKey(key).New()
-	assert.NoError(t, err)
+// TestPluginCatalog_load tests that we can successfully load the HashiCorp PGP public key into our global verifier.
+func TestPluginCatalog_load(t *testing.T) {
+	err := load()
+	assert.NoError(t, err, "expected successful load of PGP public key")
 }

vault/plugincatalog/plugin_catalog.go

@@ -834,7 +834,7 @@ func (c *PluginCatalog) upgradePlugins(ctx context.Context, logger log.Logger) e
 		}
 
 		// Upgrade the storage. At this point we don't know what type of plugin this is so pass in the unknown type.
-		runner, err := c.setInternal(ctx, pluginutil.SetPluginInput{
+		runner, err := c.set(ctx, pluginutil.SetPluginInput{
 			Name:    pluginName,
 			Type:    consts.PluginTypeUnknown,
 			Version: plugin.Version,
@@ -871,15 +871,27 @@ func (c *PluginCatalog) verifyOfficialPlugins(ctx context.Context) error {
 		return err
 	}
 
+	// // TODO: In the future, add retries.
+	for _, plugin := range plugins {
+		if plugin.Download {
+			// TODO: In the future, we may want to have some sort of deadline here so we don't block Vault from starting or impose any extended delays.
+			if err := c.entDownloadExtractVerifyPlugin(ctx, plugin); err != nil {
+				c.logger.Warn("failed to download, extract, and verify plugin",
+					"plugin", plugin.Name, "version", plugin.Version, "error", err)
+			}
+		}
+	}
+
+	// The below verification block only applies to official plugins that were manually downloaded and extracted to the plugin directory.
 	var hasOfficialPlugins bool
 	for _, plugin := range plugins {
-		if plugin.Tier != consts.PluginTierOfficial {
+		if plugin.Tier != consts.PluginTierOfficial || plugin.Download {
 			continue
 		}
 
 		hasOfficialPlugins = true
 		pluginDir := path.Join(c.directory, path.Dir(plugin.Command))
-		if _, err = verifyPlugin(pluginDir, plugin.Name, hashiCorpPGPPubKey); err != nil {
+		if _, err = VerifyPlugin(pluginDir, plugin.Name, verifyPGPSignatureDetached); err != nil {
 			return fmt.Errorf("failed to verify plugin %q version %q: %w", plugin.Name, plugin.Version, err)
 		}
 	}
@@ -989,54 +1001,69 @@ func (c *PluginCatalog) Set(ctx context.Context, plugin pluginutil.SetPluginInpu
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	_, err := c.setInternal(ctx, plugin)
+	_, err := c.set(ctx, plugin)
 	return err
 }
 
-// setInternal sets a plugin entry in the catalog. In addition to its CE functionality,
-// it will attempt to verify the plugin in its extracted artifact directory
-// if the sha256 is not specified.
-func (c *PluginCatalog) setInternal(ctx context.Context, plugin pluginutil.SetPluginInput) (*pluginutil.PluginRunner, error) {
-	command := plugin.Command
+// set sets a plugin entry in the catalog.
+// For manually extracted plugin artifacts when sha256 is unset, it additionally attempts to verify the plugin.
+func (c *PluginCatalog) set(ctx context.Context, plugin pluginutil.SetPluginInput) (*pluginutil.PluginRunner, error) {
+	absCommand := plugin.Command
 	var pluginTier consts.PluginTier
 
 	if plugin.OCIImage == "" {
 		// When OCIImage is empty, then we want to register with the binary either directly (if Sha256 is set) or via the extracted artifact directory (if Sha256 is unset).
 
+		// Validate name and command are set as we rely on these to be specified to populate the plugin runner.
+		if plugin.Name == "" {
+			return nil, fmt.Errorf("must specify name to register plugin")
+		}
+		if plugin.Command == "" {
+			return nil, fmt.Errorf("must specify command to register plugin")
+		}
+
 		var expectedPluginDir string
-		if len(plugin.Sha256) > 0 {
-			// When Sha256 is set, we can assume the binary is already available.
+		var err error
+		if plugin.Download {
+			expectedPluginDir, plugin.Command, pluginTier, err = c.entPrepareDownloadedPlugin(ctx, plugin)
+		} else if len(plugin.Sha256) > 0 {
+			// When Download is false and Sha256 is set, we can assume the binary is already available.
 			expectedPluginDir = c.directory
-			command = filepath.Join(c.directory, plugin.Command)
 		} else {
-			// When Sha256 is unset, ensure Version is set then attempt to verify the plugin
-			// in its extracted artifact directory.
+			// When Download is false and Sha256 is unset, ensure Version is set then attempt to verify
+			// the plugin in its extracted artifact directory.
+			//
+			// TODO: We should move this verification of manually extracted artifacts into handlePluginCatalogUpdate
+			// and pass tier and sha256 from the pluginutil.SetPluginInput instead of setting them here.
 
 			if len(plugin.Version) == 0 {
 				return nil, fmt.Errorf("must specify sha256 to register plugin with binary or version to register plugin with extracted artifact directory")
 			}
 
 			var err error
-			extractedArtifactDir := getExtractedArtifactDir(plugin.Name, plugin.Version)
+			extractedArtifactDir := GetExtractedArtifactDir(plugin.Name, plugin.Version)
 			expectedPluginDir = path.Join(c.directory, extractedArtifactDir)
+
+			// plugin.Command is the command relative to the plugin directory,
+			// e.g., plugins/vault-plugin-secrets-kv_0.24.0_linux_amd64/vault-plugin-secrets-kv
 			plugin.Command = path.Join(extractedArtifactDir, plugin.Name)
 
-			metadata, err := verifyPlugin(expectedPluginDir, plugin.Name, defaultPGPPubKey)
+			metadata, err := VerifyPlugin(expectedPluginDir, plugin.Name, verifyPGPSignatureDetached)
 			if err != nil {
 				return nil, fmt.Errorf("failed to verify plugin plugin %q version %q: %w",
 					plugin.Name, plugin.Version, err)
 			}
 			pluginTier = metadata.Plugin.Tier
 
-			plugin.Sha256, err = pluginSHA256Sum(path.Join(c.directory, plugin.Command))
+			plugin.Sha256, err = hex.DecodeString(metadata.Plugin.Sha256)
 			if err != nil {
-				return nil, fmt.Errorf("failed to calculate SHA256 of plugin: %w", err)
+				return nil, fmt.Errorf("failed to decode plugin metadata sha256: %w", err)
 			}
-
-			command = filepath.Join(c.directory, plugin.Command)
 		}
 
-		sym, err := filepath.EvalSymlinks(command)
+		absCommand = filepath.Join(c.directory, plugin.Command)
+
+		sym, err := filepath.EvalSymlinks(absCommand)
 		if err != nil {
 			return nil, fmt.Errorf("error while validating the command path: %w", err)
 		}
@@ -1059,7 +1086,7 @@ func (c *PluginCatalog) setInternal(ctx context.Context, plugin pluginutil.SetPl
 	// the plugin directory to the command, but we can't use get() here.
 	entryTmp := &pluginutil.PluginRunner{
 		Name:     plugin.Name,
-		Command:  command,
+		Command:  absCommand,
 		OCIImage: plugin.OCIImage,
 		Runtime:  plugin.Runtime,
 		Args:     plugin.Args,
@@ -1068,6 +1095,7 @@ func (c *PluginCatalog) setInternal(ctx context.Context, plugin pluginutil.SetPl
 		Builtin:  false,
 		Tmpdir:   c.tmpdir,
 		Tier:     pluginTier,
+		Download: plugin.Download,
 	}
 
 	if entryTmp.OCIImage != "" && entryTmp.Runtime != "" {
@@ -1131,6 +1159,7 @@ func (c *PluginCatalog) setInternal(ctx context.Context, plugin pluginutil.SetPl
 		Builtin:  false,
 		Tmpdir:   c.tmpdir,
 		Tier:     pluginTier,
+		Download: plugin.Download,
 	}
 
 	buf, err := json.Marshal(entry)

vault/plugincatalog/plugin_catalog_stubs_oss.go

@@ -8,9 +8,14 @@ package plugincatalog
 import (
 	"context"
 
+	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/helper/pluginutil"
 )
 
-func (c *PluginCatalog) entPrepareDownloadedPlugin(ctx context.Context, plugin pluginutil.SetPluginInput) (string, string, error) {
-	return "", "", nil
+func (c *PluginCatalog) entPrepareDownloadedPlugin(ctx context.Context, plugin pluginutil.SetPluginInput) (string, string, consts.PluginTier, error) {
+	return "", "", consts.PluginTierUnknown, nil
+}
+
+func (c *PluginCatalog) entDownloadExtractVerifyPlugin(ctx context.Context, pr *pluginutil.PluginRunner) error {
+	return nil
 }

vault/plugincatalog/plugin_metadata.go

@@ -10,11 +10,14 @@ import (
 	"github.com/hashicorp/vault/sdk/helper/consts"
 )
 
-type pluginMetadata struct {
+// PluginMetadata represents metadata.json in the plugin artifact
+type PluginMetadata struct {
 	Version string `json:"version"`
 	Plugin  Plugin `json:"plugin"`
 }
 
+// Plugin represents the metadata of a plugin as defined
+// under "plugin" object in the metadata.json
 type Plugin struct {
 	Name string            `json:"name"`
 	Type consts.PluginType `json:"type"`
@@ -26,15 +29,18 @@ type Plugin struct {
 	Arch     string `json:"arch"`
 	// PGPSig is PGP ASCII armored detached signature
 	PGPSig string `json:"pgp_sig"`
+	// Sha256 is the SHA256 checksum of the plugin binary.
+	// TODO: Not currently present in today's plugin release metadata, but will be added in the future.
+	Sha256 string `json:"sha256"`
 }
 
-func readPluginMetadata(metadataPath string) (*pluginMetadata, error) {
+func readPluginMetadata(metadataPath string) (*PluginMetadata, error) {
 	metadataBytes, err := os.ReadFile(metadataPath)
 	if err != nil {
 		return nil, err
 	}
 
-	metadata := pluginMetadata{}
+	metadata := PluginMetadata{}
 	if err = json.Unmarshal(metadataBytes, &metadata); err != nil {
 		return nil, err
 	}

vault/plugincatalog/testing_util.go

@@ -38,7 +38,7 @@ func generatePluginArtifactContents(t *testing.T, pluginName, pluginVersion stri
 ) map[string][]byte {
 	t.Helper()
 
-	metadata := pluginMetadata{
+	metadata := PluginMetadata{
 		Version: "v0",
 		Plugin: Plugin{
 			Name:     pluginName,