diff --git a/cmd/clone.go b/cmd/clone.go index 608d5351..176be745 100644 --- a/cmd/clone.go +++ b/cmd/clone.go @@ -297,6 +297,8 @@ func cloneFunc(cmd *cobra.Command, argz []string) { os.Exit(1) } + + if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" { updateAbsolutePathToCloneToWithHostname() } @@ -1236,3 +1238,5 @@ func filterByGhorgignore(cloneTargets []scm.Repo) []scm.Repo { return cloneTargets } + + diff --git a/cmd/repository_processor.go b/cmd/repository_processor.go index d07f60b5..9c5ee5f2 100644 --- a/cmd/repository_processor.go +++ b/cmd/repository_processor.go @@ -211,33 +211,30 @@ func (rp *RepositoryProcessor) handleExistingRepository(repo *scm.Repo, action * return false } + var success bool if os.Getenv("GHORG_BACKUP") == "true" { *action = "updating remote" - success := rp.handleBackupMode(repo) - if !success { - return false - } + success = rp.handleBackupMode(repo) } else if os.Getenv("GHORG_NO_CLEAN") == "true" { *action = "fetching" - success := rp.handleNoCleanMode(repo) - if !success { - return false - } + success = rp.handleNoCleanMode(repo) } else { // Standard pull mode - success := rp.handleStandardPull(repo) - if !success { - return false - } + success = rp.handleStandardPull(repo) } - // Reset origin + // Always reset origin to remove credentials, even if processing failed err = rp.git.SetOrigin(*repo) if err != nil { rp.addError(fmt.Sprintf("Problem resetting remote: %s Error: %v", repo.Name, err)) return false } + // Return success after ensuring tokens are stripped + if !success { + return false + } + rp.mutex.Lock() rp.stats.PulledCount++ rp.mutex.Unlock() @@ -284,9 +281,26 @@ func (rp *RepositoryProcessor) handleNewRepository(repo *scm.Repo, action *strin // Fetch all if enabled if os.Getenv("GHORG_FETCH_ALL") == "true" { - err = rp.git.FetchAll(*repo) + // Temporarily restore credentials for fetch-all to work with private repos + err = rp.git.SetOriginWithCredentials(*repo) if err != nil { - rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err)) + rp.addError(fmt.Sprintf("Problem trying to set remote with credentials: %s Error: %v", repo.URL, err)) + return false + } + + err = rp.git.FetchAll(*repo) + fetchErr := err // Store fetch error for later reporting + + // Always strip credentials again for security, even if fetch failed + err = rp.git.SetOrigin(*repo) + if err != nil { + rp.addError(fmt.Sprintf("Problem trying to reset remote after fetch: %s Error: %v", repo.URL, err)) + return false + } + + // Report fetch error if it occurred + if fetchErr != nil { + rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, fetchErr)) return false } } @@ -317,15 +331,30 @@ func (rp *RepositoryProcessor) handleBackupMode(repo *scm.Repo) bool { // handleNoCleanMode processes repositories in no-clean mode func (rp *RepositoryProcessor) handleNoCleanMode(repo *scm.Repo) bool { - err := rp.git.FetchAll(*repo) - - if err != nil && repo.IsWiki { - rp.addInfo(fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, err)) + // Temporarily restore credentials for fetch-all to work with private repos + err := rp.git.SetOriginWithCredentials(*repo) + if err != nil { + rp.addError(fmt.Sprintf("Problem trying to set remote with credentials: %s Error: %v", repo.URL, err)) return false } + err = rp.git.FetchAll(*repo) + fetchErr := err // Store fetch error for later reporting + + // Always strip credentials again for security, even if fetch failed + err = rp.git.SetOrigin(*repo) if err != nil { - rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err)) + rp.addError(fmt.Sprintf("Problem trying to reset remote after fetch: %s Error: %v", repo.URL, err)) + return false + } + + if fetchErr != nil && repo.IsWiki { + rp.addInfo(fmt.Sprintf("Wiki may be enabled but there was no content to clone on: %s Error: %v", repo.URL, fetchErr)) + return false + } + + if fetchErr != nil { + rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, fetchErr)) return false } @@ -336,9 +365,26 @@ func (rp *RepositoryProcessor) handleNoCleanMode(repo *scm.Repo) bool { func (rp *RepositoryProcessor) handleStandardPull(repo *scm.Repo) bool { // Fetch all if enabled if os.Getenv("GHORG_FETCH_ALL") == "true" { - err := rp.git.FetchAll(*repo) + // Temporarily restore credentials for fetch-all to work with private repos + err := rp.git.SetOriginWithCredentials(*repo) if err != nil { - rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, err)) + rp.addError(fmt.Sprintf("Problem trying to set remote with credentials: %s Error: %v", repo.URL, err)) + return false + } + + err = rp.git.FetchAll(*repo) + fetchErr := err // Store fetch error for later reporting + + // Always strip credentials again for security, even if fetch failed + err = rp.git.SetOrigin(*repo) + if err != nil { + rp.addError(fmt.Sprintf("Problem trying to reset remote after fetch: %s Error: %v", repo.URL, err)) + return false + } + + // Report fetch error if it occurred + if fetchErr != nil { + rp.addError(fmt.Sprintf("Could not fetch remotes: %s Error: %v", repo.URL, fetchErr)) return false } } diff --git a/scripts/local-gitlab/configs/test-scenarios.json b/scripts/local-gitlab/configs/test-scenarios.json index 98a47730..0a87284b 100644 --- a/scripts/local-gitlab/configs/test-scenarios.json +++ b/scripts/local-gitlab/configs/test-scenarios.json @@ -605,6 +605,18 @@ "all-users-snippets-preserve-dir-test/testuser1/testuser1-repo.snippets" ] }, + { + "name": "fetch-all-https-token-compatibility-test", + "description": "Test that FETCH_ALL + HTTPS token authentication works correctly by temporarily restoring credentials and tokens are stripped from remotes", + "command": "ghorg clone local-gitlab-group1 --scm=gitlab --base-url={{.BaseURL}} --token={{.Token}} --fetch-all --output-dir=fetch-all-https-token-test", + "run_twice": false, + "expected_structure": [ + "fetch-all-https-token-test/baz0", + "fetch-all-https-token-test/baz1", + "fetch-all-https-token-test/baz2", + "fetch-all-https-token-test/baz3" + ] + }, { "name": "gitlab-group-exclude-regex-subgroup-a", "description": "Test GitLab group exclude regex excludes all repos from subgroup-a", diff --git a/scripts/local-gitlab/test-runner/main.go b/scripts/local-gitlab/test-runner/main.go index 20a69547..5e888a15 100644 --- a/scripts/local-gitlab/test-runner/main.go +++ b/scripts/local-gitlab/test-runner/main.go @@ -13,14 +13,15 @@ import ( ) type TestScenario struct { - Name string `json:"name"` - Description string `json:"description"` - Command string `json:"command"` - RunTwice bool `json:"run_twice"` - SetupCommands []string `json:"setup_commands,omitempty"` - VerifyCommands []string `json:"verify_commands,omitempty"` - ExpectedStructure []string `json:"expected_structure"` - Disabled bool `json:"disabled,omitempty"` + Name string `json:"name"` + Description string `json:"description"` + Command string `json:"command"` + RunTwice bool `json:"run_twice"` + SetupCommands []string `json:"setup_commands,omitempty"` + VerifyCommands []string `json:"verify_commands,omitempty"` + ExpectedStructure []string `json:"expected_structure"` + Disabled bool `json:"disabled,omitempty"` + SkipTokenVerification bool `json:"skip_token_verification,omitempty"` } type TestConfig struct { @@ -144,6 +145,13 @@ func (tr *TestRunner) runTest(scenario *TestScenario) error { return fmt.Errorf("structure verification failed: %w", err) } + // Verify no tokens in git remotes by default (unless explicitly skipped) + if len(scenario.ExpectedStructure) > 0 && !scenario.SkipTokenVerification { + if err := tr.verifyNoTokensInRemotes(scenario.ExpectedStructure, tr.context.Token); err != nil { + return fmt.Errorf("token verification failed: %w", err) + } + } + // Execute verification commands if any for _, verifyCmd := range scenario.VerifyCommands { renderedCmd, err := tr.renderTemplate(verifyCmd) @@ -211,6 +219,38 @@ func (tr *TestRunner) verifyExpectedStructure(expectedPaths []string) error { return nil } +func (tr *TestRunner) verifyNoTokensInRemotes(expectedPaths []string, token string) error { + for _, expectedPath := range expectedPaths { + fullPath := filepath.Join(tr.context.GhorgDir, expectedPath) + + // Check if this is a git repository directory + if _, err := os.Stat(filepath.Join(fullPath, ".git")); err != nil { + if os.IsNotExist(err) { + // Not a git repository, skip + continue + } + return fmt.Errorf("failed to check .git directory in %s: %w", expectedPath, err) + } + + // Run git remote -v to get all remotes + cmd := exec.Command("git", "remote", "-v") + cmd.Dir = fullPath + + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to get git remotes for %s: %w\nOutput: %s", expectedPath, err, string(output)) + } + + // Check if the token appears in any remote URL + remoteOutput := string(output) + if strings.Contains(remoteOutput, token) { + return fmt.Errorf("token found in git remote URLs for %s:\n%s", expectedPath, remoteOutput) + } + } + + return nil +} + func (tr *TestRunner) ensureGhorgDirectoryExists() error { log.Printf("Ensuring ghorg directory exists: %s", tr.context.GhorgDir)