mirrors/ghorg
1
0
Fork 0
mirror of https://github.com/gabrie30/ghorg.git synced 2026-05-05 03:46:09 +02:00
Code Issues Projects Releases Wiki Activity

Ensure credentials are stripped from git remotes after fetch (#559)

Browse Source
This commit is contained in:
gabrie30 2025-08-24 15:56:25 -07:00 committed by GitHub
parent 7d71967489
commit f561d77357
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 132 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/clone.go
View File

@ -297,6 +297,8 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
os.Exit(1)
}
if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" {
updateAbsolutePathToCloneToWithHostname()
}
@ -1236,3 +1238,5 @@ func filterByGhorgignore(cloneTargets []scm.Repo) []scm.Repo {
return cloneTargets
}

90
cmd/repository_processor.go
View File

@ -211,33 +211,30 @@ func (rp *RepositoryProcessor) handleExistingRepository(repo *scm.Repo, action *
return false
}
var success bool
if os.Getenv("GHORG_BACKUP") == "true" {
*action = "updating remote"
success := rp.handleBackupMode(repo)
if !success {
return false
}
success = rp.handleBackupMode(repo)
} else if os.Getenv("GHORG_NO_CLEAN") == "true" {
*action = "fetching"
success := rp.handleNoCleanMode(repo)
if !success {
return false
}
success = rp.handleNoCleanMode(repo)
} else {
// Standard pull mode
success := rp.handleStandardPull(repo)
if !success {
return false
}
success = rp.handleStandardPull(repo)
}
// Reset origin
// Always reset origin to remove credentials, even if processing failed
err = rp.git.SetOrigin(*repo)
if err != nil {
rp.addError(fmt.Sprintf("Problem resetting remote: %s Error: %v", repo.Name, err))
return false
}
// Return success after ensuring tokens are stripped
if !success {
return false
}
rp.mutex.Lock()
rp.stats.PulledCount++
rp.mutex.Unlock()
@ -284,9 +281,26 @@ func (rp *RepositoryProcessor) handleNewRepository(repo *scm.Repo, action *strin
// Fetch all if enabled
if os.Getenv("GHORG_FETCH_ALL") == "true" {
err = rp.git.FetchAll(*repo)
// Temporarily restore credentials for fetch-all to work with private repos
err = rp.git.SetOriginWithCredentials(*repo)
if err != nil {
rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err))
rp.addError(fmt.Sprintf("Problem trying to set remote with credentials: %s Error: %v", repo.URL, err))
return false
}
err = rp.git.FetchAll(*repo)
fetchErr := err // Store fetch error for later reporting
// Always strip credentials again for security, even if fetch failed
err = rp.git.SetOrigin(*repo)
if err != nil {
rp.addError(fmt.Sprintf("Problem trying to reset remote after fetch: %s Error: %v", repo.URL, err))
return false
}
// Report fetch error if it occurred
if fetchErr != nil {
rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, fetchErr))
return false
}
}
@ -317,15 +331,30 @@ func (rp *RepositoryProcessor) handleBackupMode(repo *scm.Repo) bool {
// handleNoCleanMode processes repositories in no-clean mode
func (rp *RepositoryProcessor) handleNoCleanMode(repo *scm.Repo) bool {
err := rp.git.FetchAll(*repo)
if err != nil && repo.IsWiki {
rp.addInfo(fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, err))
// Temporarily restore credentials for fetch-all to work with private repos
err := rp.git.SetOriginWithCredentials(*repo)
if err != nil {
rp.addError(fmt.Sprintf("Problem trying to set remote with credentials: %s Error: %v", repo.URL, err))
return false
}
err = rp.git.FetchAll(*repo)
fetchErr := err // Store fetch error for later reporting
// Always strip credentials again for security, even if fetch failed
err = rp.git.SetOrigin(*repo)
if err != nil {
rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err))
rp.addError(fmt.Sprintf("Problem trying to reset remote after fetch: %s Error: %v", repo.URL, err))
return false
}
if fetchErr != nil && repo.IsWiki {
rp.addInfo(fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, fetchErr))
return false
}
if fetchErr != nil {
rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, fetchErr))
return false
}
@ -336,9 +365,26 @@ func (rp *RepositoryProcessor) handleNoCleanMode(repo *scm.Repo) bool {
func (rp *RepositoryProcessor) handleStandardPull(repo *scm.Repo) bool {
// Fetch all if enabled
if os.Getenv("GHORG_FETCH_ALL") == "true" {
err := rp.git.FetchAll(*repo)
// Temporarily restore credentials for fetch-all to work with private repos
err := rp.git.SetOriginWithCredentials(*repo)
if err != nil {
rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err))
rp.addError(fmt.Sprintf("Problem trying to set remote with credentials: %s Error: %v", repo.URL, err))
return false
}
err = rp.git.FetchAll(*repo)
fetchErr := err // Store fetch error for later reporting
// Always strip credentials again for security, even if fetch failed
err = rp.git.SetOrigin(*repo)
if err != nil {
rp.addError(fmt.Sprintf("Problem trying to reset remote after fetch: %s Error: %v", repo.URL, err))
return false
}
// Report fetch error if it occurred
if fetchErr != nil {
rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, fetchErr))
return false
}
}

12
scripts/local-gitlab/configs/test-scenarios.json
View File

@ -605,6 +605,18 @@
"all-users-snippets-preserve-dir-test/testuser1/testuser1-repo.snippets"
]
},
{
"name": "fetch-all-https-token-compatibility-test",
"description": "Test that FETCH_ALL + HTTPS token authentication works correctly by temporarily restoring credentials and tokens are stripped from remotes",
"command": "ghorg clone local-gitlab-group1 --scm=gitlab --base-url={{.BaseURL}} --token={{.Token}} --fetch-all --output-dir=fetch-all-https-token-test",
"run_twice": false,
"expected_structure": [
"fetch-all-https-token-test/baz0",
"fetch-all-https-token-test/baz1",
"fetch-all-https-token-test/baz2",
"fetch-all-https-token-test/baz3"
]
},
{
"name": "gitlab-group-exclude-regex-subgroup-a",
"description": "Test GitLab group exclude regex excludes all repos from subgroup-a",

56
scripts/local-gitlab/test-runner/main.go
View File

@ -13,14 +13,15 @@ import (
)
type TestScenario struct {
Name string `json:"name"`
Description string `json:"description"`
Command string `json:"command"`
RunTwice bool `json:"run_twice"`
SetupCommands []string `json:"setup_commands,omitempty"`
VerifyCommands []string `json:"verify_commands,omitempty"`
ExpectedStructure []string `json:"expected_structure"`
Disabled bool `json:"disabled,omitempty"`
Name string `json:"name"`
Description string `json:"description"`
Command string `json:"command"`
RunTwice bool `json:"run_twice"`
SetupCommands []string `json:"setup_commands,omitempty"`
VerifyCommands []string `json:"verify_commands,omitempty"`
ExpectedStructure []string `json:"expected_structure"`
Disabled bool `json:"disabled,omitempty"`
SkipTokenVerification bool `json:"skip_token_verification,omitempty"`
}
type TestConfig struct {
@ -144,6 +145,13 @@ func (tr *TestRunner) runTest(scenario *TestScenario) error {
return fmt.Errorf("structure verification failed: %w", err)
}
// Verify no tokens in git remotes by default (unless explicitly skipped)
if len(scenario.ExpectedStructure) > 0 && !scenario.SkipTokenVerification {
if err := tr.verifyNoTokensInRemotes(scenario.ExpectedStructure, tr.context.Token); err != nil {
return fmt.Errorf("token verification failed: %w", err)
}
}
// Execute verification commands if any
for _, verifyCmd := range scenario.VerifyCommands {
renderedCmd, err := tr.renderTemplate(verifyCmd)
@ -211,6 +219,38 @@ func (tr *TestRunner) verifyExpectedStructure(expectedPaths []string) error {
return nil
}
func (tr *TestRunner) verifyNoTokensInRemotes(expectedPaths []string, token string) error {
for _, expectedPath := range expectedPaths {
fullPath := filepath.Join(tr.context.GhorgDir, expectedPath)
// Check if this is a git repository directory
if _, err := os.Stat(filepath.Join(fullPath, ".git")); err != nil {
if os.IsNotExist(err) {
// Not a git repository, skip
continue
}
return fmt.Errorf("failed to check .git directory in %s: %w", expectedPath, err)
}
// Run git remote -v to get all remotes
cmd := exec.Command("git", "remote", "-v")
cmd.Dir = fullPath
output, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("failed to get git remotes for %s: %w\nOutput: %s", expectedPath, err, string(output))
}
// Check if the token appears in any remote URL
remoteOutput := string(output)
if strings.Contains(remoteOutput, token) {
return fmt.Errorf("token found in git remote URLs for %s:\n%s", expectedPath, remoteOutput)
}
}
return nil
}
func (tr *TestRunner) ensureGhorgDirectoryExists() error {
log.Printf("Ensuring ghorg directory exists: %s", tr.context.GhorgDir)