feat: allow usage of --prune with --preserve-dir (#526)

cmd/clone.go

@@ -5,6 +5,7 @@ import (
 	"bufio"
 	"crypto/sha256"
 	"fmt"
+	"io/fs"
 	"log"
 	"net/url"
 	"os"
@@ -556,18 +557,18 @@ func printDryRun(repos []scm.Repo) {
 			// to do.
 			colorlog.PrintInfo("\nScanning for local clones that have been removed on remote...")
 
-			files, err := os.ReadDir(outputDirAbsolutePath)
+			repositories, err := getRelativePathRepositories(outputDirAbsolutePath)
 			if err != nil {
 				log.Fatal(err)
 			}
 
 			eligibleForPrune := 0
-			for _, f := range files {
+			for _, repository := range repositories {
 				// for each item in the org's clone directory, let's make sure we found a
 				// corresponding repo on the remote.
-				if !sliceContainsNamedRepo(repos, f.Name()) {
+				if !sliceContainsNamedRepo(repos, repository) {
 					eligibleForPrune++
-					colorlog.PrintSubtleInfo(fmt.Sprintf("%s not found in remote.", f.Name()))
+					colorlog.PrintSubtleInfo(fmt.Sprintf("%s not found in remote.", repository))
 				}
 			}
 			colorlog.PrintSuccess(fmt.Sprintf("Local clones eligible for pruning: %d", eligibleForPrune))
@@ -599,6 +600,29 @@ func getCloneableInventory(allRepos []scm.Repo) (int, int, int, int) {
 	return total, repos, snippets, wikis
 }
 
+func isGitRepository(path string) bool {
+	stat, err := os.Stat(filepath.Join(path, ".git"))
+	return err == nil && stat.IsDir()
+}
+
+func getRelativePathRepositories(root string) ([]string, error) {
+	var relativePaths []string
+	err := filepath.WalkDir(root, func(path string, file fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if path != outputDirAbsolutePath && file.IsDir() && isGitRepository(path) {
+			rel, err := filepath.Rel(outputDirAbsolutePath, path)
+			if err != nil {
+				return err
+			}
+			relativePaths = append(relativePaths, rel)
+		}
+		return nil
+	})
+	return relativePaths, err
+}
+
 // CloneAllRepos clones all repos
 func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 	// Filter repos that have attributes that don't need specific scm api calls
@@ -1294,7 +1318,7 @@ func pruneRepos(cloneTargets []scm.Repo) int {
 	count := 0
 	colorlog.PrintInfo("\nScanning for local clones that have been removed on remote...")
 
-	files, err := os.ReadDir(outputDirAbsolutePath)
+	repositories, err := getRelativePathRepositories(outputDirAbsolutePath)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -1303,18 +1327,18 @@ func pruneRepos(cloneTargets []scm.Repo) int {
 	// break out of the loop.
 	userAgreesToDelete := true
 	pruneNoConfirm := os.Getenv("GHORG_PRUNE_NO_CONFIRM") == "true"
-	for _, f := range files {
+	for _, repository := range repositories {
 		// For each item in the org's clone directory, let's make sure we found a corresponding
 		// repo on the remote.  We check userAgreesToDelete here too, so that if the user says
 		// "No" at any time, we stop trying to prune things altogether.
-		if userAgreesToDelete && !sliceContainsNamedRepo(cloneTargets, f.Name()) {
+		if userAgreesToDelete && !sliceContainsNamedRepo(cloneTargets, repository) {
 			// If the user specified --prune-no-confirm, we needn't prompt interactively.
 			userAgreesToDelete = pruneNoConfirm || interactiveYesNoPrompt(
-				fmt.Sprintf("%s was not found in remote.  Do you want to prune it?", f.Name()))
+				fmt.Sprintf("%s was not found in remote.  Do you want to prune it?", repository))
 			if userAgreesToDelete {
 				colorlog.PrintSubtleInfo(
-					fmt.Sprintf("Deleting %s", filepath.Join(outputDirAbsolutePath, f.Name())))
-				err = os.RemoveAll(filepath.Join(outputDirAbsolutePath, f.Name()))
+					fmt.Sprintf("Deleting %s", repository))
+				err = os.RemoveAll(filepath.Join(outputDirAbsolutePath, repository))
 				count++
 				if err != nil {
 					log.Fatal(err)
@@ -1365,18 +1389,9 @@ func interactiveYesNoPrompt(prompt string) bool {
 }
 
 // There's probably a nicer way of finding whether any scm.Repo in the slice matches a given name.
-// TODO, currently this does not work if user sets --preserve-dir see https://github.com/gabrie30/ghorg/issues/210 for more info
 func sliceContainsNamedRepo(haystack []scm.Repo, needle string) bool {
-
-	if os.Getenv("GHORG_PRESERVE_DIRECTORY_STRUCTURE") == "true" {
-		colorlog.PrintError("GHORG_PRUNE (--prune) does not currently work in combination with GHORG_PRESERVE_DIRECTORY_STRUCTURE (--preserve-dir), this will come in later versions")
-		os.Exit(1)
-	}
-
 	for _, repo := range haystack {
-		basepath := filepath.Base(repo.Path)
-
-		if basepath == needle {
+		if repo.Path == fmt.Sprintf("/%s", needle) {
 			return true
 		}
 	}

cmd/clone_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"log"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"testing"
@@ -525,3 +526,154 @@ func Test_filterDownReposIfTargetReposPathEnabled(t *testing.T) {
 		})
 	}
 }
+
+func TestRelativePathRepositories(t *testing.T) {
+	testing, err := os.MkdirTemp("", "testing")
+	if err != nil {
+		t.Fatalf("Failed to create temp directory: %v", err)
+	}
+	defer os.RemoveAll(testing)
+
+	outputDirAbsolutePath = testing
+
+	repository := filepath.Join(testing, "repository", ".git")
+	if err := os.MkdirAll(repository, 0o755); err != nil {
+		t.Fatalf("Failed to create directory: %v", err)
+	}
+
+	files, err := getRelativePathRepositories(testing)
+	if err != nil {
+		t.Fatalf("getRelativePathRepositories returned an error: %v", err)
+	}
+
+	if len(files) != 1 {
+		t.Errorf("Expected 1 directory, got %d", len(files))
+	}
+
+	if len(files) > 0 && files[0] != "repository" {
+		t.Errorf("Expected 'repository', got '%s'", files[0])
+	}
+}
+
+func TestRelativePathRepositoriesNoGitDir(t *testing.T) {
+	testing, err := os.MkdirTemp("", "testing")
+	if err != nil {
+		t.Fatalf("Failed to create temp directory: %v", err)
+	}
+	defer os.RemoveAll(testing)
+
+	outputDirAbsolutePath = testing
+
+	directory := filepath.Join(testing, "directory")
+	if err := os.MkdirAll(directory, 0o755); err != nil {
+		t.Fatalf("Failed to create directory: %v", err)
+	}
+
+	files, err := getRelativePathRepositories(testing)
+	if err != nil {
+		t.Fatalf("getRelativePathRepositories returned an error: %v", err)
+	}
+
+	if len(files) != 0 {
+		t.Errorf("Expected 0 directories, got %d", len(files))
+	}
+}
+
+func TestRelativePathRepositoriesWithGitSubmodule(t *testing.T) {
+	testing, err := os.MkdirTemp("", "testing")
+	if err != nil {
+		t.Fatalf("Failed to create temp directory: %v", err)
+	}
+	defer os.RemoveAll(testing)
+
+	outputDirAbsolutePath = testing
+
+	repository := filepath.Join(testing, "repository", ".git")
+	submodule := filepath.Join(testing, "repository", "submodule", ".git")
+
+	if err := os.MkdirAll(repository, 0o755); err != nil {
+		t.Fatalf("Failed to create directory: %v", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(submodule), 0o755); err != nil {
+		t.Fatalf("Failed to create directory: %v", err)
+	}
+	if _, err := os.Create(submodule); err != nil {
+		t.Fatalf("Failed to create .git file: %v", err)
+	}
+
+	files, err := getRelativePathRepositories(testing)
+	if err != nil {
+		t.Fatalf("getRelativePathRepositories returned an error: %v", err)
+	}
+
+	if len(files) != 1 {
+		t.Errorf("Expected 1 directory, got %d", len(files))
+	}
+
+	if len(files) > 0 && files[0] != "repository" {
+		t.Errorf("Expected 'repository', got '%s'", files[0])
+	}
+}
+
+func TestRelativePathRepositoriesDeeplyNested(t *testing.T) {
+	testing, err := os.MkdirTemp("", "testing")
+	if err != nil {
+		t.Fatalf("Failed to create directory: %v", err)
+	}
+	defer os.RemoveAll(testing)
+
+	outputDirAbsolutePath = testing
+
+	repository := filepath.Join(testing, "deeply", "nested", "repository", ".git")
+	if err := os.MkdirAll(repository, 0o755); err != nil {
+		t.Fatalf("Failed to create repository: %v", err)
+	}
+
+	files, err := getRelativePathRepositories(testing)
+	if err != nil {
+		t.Fatalf("getRelativePathRepositories returned an error: %v", err)
+	}
+
+	if len(files) != 1 {
+		t.Errorf("Expected 1 directory, got %d", len(files))
+	}
+
+	expected := filepath.Join("deeply", "nested", "repository")
+	if len(files) > 0 && files[0] != expected {
+		t.Errorf("Expected '%s', got '%s'", expected, files[0])
+	}
+}
+
+func TestPruneRepos(t *testing.T) {
+	os.Setenv("GHORG_PRUNE_NO_CONFIRM", "true")
+
+	cloneTargets := []scm.Repo{{Path: "/repository"}}
+
+	testing, err := os.MkdirTemp("", "testing")
+	if err != nil {
+		t.Fatalf("Failed to create directory: %v", err)
+	}
+	defer os.RemoveAll(testing)
+
+	outputDirAbsolutePath = testing
+
+	repository := filepath.Join(testing, "repository", ".git")
+	if err := os.MkdirAll(repository, 0o755); err != nil {
+		t.Fatalf("Failed to create repository: %v", err)
+	}
+
+	prunable := filepath.Join(testing, "prunnable", ".git")
+	if err := os.MkdirAll(prunable, 0o755); err != nil {
+		t.Fatalf("Failed to create directory: %v", err)
+	}
+
+	pruneRepos(cloneTargets)
+
+	if _, err := os.Stat(repository); os.IsNotExist(err) {
+		t.Errorf("Expected '%s' to exist, but it was deleted", repository)
+	}
+
+	if _, err := os.Stat(prunable); !os.IsNotExist(err) {
+		t.Errorf("Expected '%s' to be deleted, but it exists", prunable)
+	}
+}

scm/structs.go

@@ -8,7 +8,7 @@ type Repo struct {
 	Name string
 	// HostPath is the path on the users machine that the repo will be cloned to. Its used in all the git commands to locate the directory of the repo. HostPath is updated for wikis and snippets because the folder for the clone is appended with .wiki and .snippet
 	HostPath string
-	// Path where the repo is located within the scm provider. Its mostly used with gitlab repos when the directory structure is preserved. In this case the path becomes where to locate the repo in relation to gitlab.com/group/group/group/repo.git => group/group/group/repo
+	// Path where the repo is located within the scm provider. Its mostly used with gitlab repos when the directory structure is preserved. In this case the path becomes where to locate the repo in relation to gitlab.com/group/group/group/repo.git => /group/group/group/repo
 	Path string
 	// URL is the web address of the repo
 	URL string

scripts/gitlab_cloud_integration_tests.sh

@@ -87,6 +87,21 @@ else
     exit 1
 fi
 
+# PRUNE AND PRESERVE DIR
+ghorg clone $GITLAB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --prune --prune-no-confirm --preserve-dir
+git init "${HOME}"/ghorg/"${GITLAB_GROUP}"/wayne-enterprises/wayne-industries/prunable
+ghorg clone $GITLAB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --prune --prune-no-confirm --preserve-dir
+
+if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP}"/wayne-enterprises/wayne-industries/microservice ] && \
+    [ ! -e "${HOME}"/ghorg/"${GITLAB_GROUP}"/wayne-enterprises/wayne-industries/prunable ]
+then
+    echo "Pass: gitlab org clone preserve dir, prune"
+    rm -rf "${HOME}/ghorg/${GITLAB_GROUP}"
+else
+    echo "Fail: gitlab org clone preserve dir, prune"
+    exit 1
+fi
+
 # REPO NAME COLLISION
 ghorg clone $GITLAB_GROUP_2 --token="${GITLAB_TOKEN}" --scm=gitlab