From c4a30ac9bf3c6b9e09cb0015dcdc7604268010df Mon Sep 17 00:00:00 2001 From: gabrie30 Date: Sun, 24 Aug 2025 14:10:33 -0700 Subject: [PATCH] Improve GitLab snippet and wiki clone consistency (#557) --- cmd/examples-copy/gitlab.md | 16 ++- scm/gitlab.go | 8 +- scripts/bitbucket_cloud_integration_tests.sh | 1 + scripts/github_cloud_integration_tests.sh | 1 + scripts/gitlab_cloud_integration_tests.sh | 1 + .../local-gitlab/configs/test-scenarios.json | 110 ++++++++++++++++++ scripts/local-gitlab/integration-tests.sh | 11 ++ scripts/local-gitlab/test-runner/main.go | 3 +- scripts/windows_gitlab_integration_tests.bat | 1 + 9 files changed, 143 insertions(+), 9 deletions(-) diff --git a/cmd/examples-copy/gitlab.md b/cmd/examples-copy/gitlab.md index 484977ed..eb635ce1 100644 --- a/cmd/examples-copy/gitlab.md +++ b/cmd/examples-copy/gitlab.md @@ -135,15 +135,17 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri ghorg clone group3/subgroup1 --base-url=https:// --scm=gitlab --token=XXXXXX ``` - This would produce a directory structure like, where `projectX` is a project in a subgroup nested inside `subgroup1` + This would produce a directory structure like, where `projectX` is a project in a subgroup nested inside `subgroup1`. Note that projects from nested subgroups will appear both flattened and in their original structure. ```sh /GHORG_ABSOLUTE_PATH_TO_CLONE_TO └── group3 └── subgroup1 - ├── projectX ├── project3 - └── project4 + ├── project4 + ├── projectX + └── subgroup2 + └── projectX ``` 1. Clone **a specific subgroup**, **preserving the directory structure** of subgroups @@ -188,10 +190,12 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri > Note: You must set `--base-url` which is the url to your instance. If your instance requires an insecure connection you can use the `--insecure-gitlab-client` flag +> Note: When using "all-users", you must include the `--clone-type=user` flag + 1. Clone **all users**, **preserving the directory structure** of users ```sh - ghorg clone all-users --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir + ghorg clone all-users --clone-type=user --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir ``` This would produce a directory structure like @@ -210,7 +214,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri 1. Clone **all users**, **WITHOUT preserving the directory structure** of users ```sh - ghorg clone all-users --base-url=https:// --scm=gitlab --token=XXXXXX + ghorg clone all-users --clone-type=user --base-url=https:// --scm=gitlab --token=XXXXXX ``` This would produce a directory structure like @@ -225,7 +229,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri 1. Clone **all users**, **preserving the directory structure** of users, preserving scm hostname ```sh - ghorg clone all-users --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir --preserve-scm-hostname + ghorg clone all-users --clone-type=user --base-url=https:// --scm=gitlab --token=XXXXXX --preserve-dir --preserve-scm-hostname ``` This would produce a directory structure like diff --git a/scm/gitlab.go b/scm/gitlab.go index a9b71dcf..5ab95e6f 100644 --- a/scm/gitlab.go +++ b/scm/gitlab.go @@ -299,6 +299,12 @@ func (c Gitlab) GetSnippets(cloneData []Repo, target string) ([]Repo, error) { allSnippetsToClone = append(allSnippetsToClone, snippet) } } + } else if os.Getenv("GHORG_CLONE_TYPE") != "user" { + // Handle single group clones on hosted instances + for _, repo := range cloneData { + repoSnippets := c.getRepoSnippets(repo) + allSnippetsToClone = append(allSnippetsToClone, repoSnippets...) + } } if os.Getenv("GHORG_CLONE_TYPE") == "user" && os.Getenv("GHORG_SCM_BASE_URL") == "" { @@ -571,7 +577,7 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo { wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1) wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1) wiki.CloneBranch = "master" - wiki.Path = fmt.Sprintf("%s%s", p.PathWithNamespace, ".wiki") + wiki.Path = fmt.Sprintf("%s%s", path, ".wiki") repoData = append(repoData, wiki) } } diff --git a/scripts/bitbucket_cloud_integration_tests.sh b/scripts/bitbucket_cloud_integration_tests.sh index 24bda983..a7a4ac77 100755 --- a/scripts/bitbucket_cloud_integration_tests.sh +++ b/scripts/bitbucket_cloud_integration_tests.sh @@ -7,6 +7,7 @@ echo "Running BitBucket Integration Tests" cp ./ghorg /usr/local/bin BITBUCKET_WORKSPACE=ghorg +GHORG_EXIT_CODE_ON_CLONE_ISSUES=0 # clone an org with no config file ghorg clone $BITBUCKET_WORKSPACE --token="${BITBUCKET_TOKEN}" --bitbucket-username="${BITBUCKET_USERNAME}" --scm=bitbucket --output-dir=bb-test-1 diff --git a/scripts/github_cloud_integration_tests.sh b/scripts/github_cloud_integration_tests.sh index 8feb0ee1..5e495e35 100755 --- a/scripts/github_cloud_integration_tests.sh +++ b/scripts/github_cloud_integration_tests.sh @@ -11,6 +11,7 @@ GHORG_TEST_REPO=ghorg-ci-test GHORG_TEST_SELF_PRIVATE_REPO=ghorg_testing_private REPO_WITH_TESTING_TOPIC=ghorg-repo-with-topic-of-testing GITHUB_SELF=gabrie30 +GHORG_EXIT_CODE_ON_CLONE_ISSUES=0 ghorg version diff --git a/scripts/gitlab_cloud_integration_tests.sh b/scripts/gitlab_cloud_integration_tests.sh index 502825f3..24f5c3be 100755 --- a/scripts/gitlab_cloud_integration_tests.sh +++ b/scripts/gitlab_cloud_integration_tests.sh @@ -11,6 +11,7 @@ GITLAB_GROUP=gitlab-examples GITLAB_SUB_GROUP=wayne-enterprises GITLAB_GROUP_2=ghorg-test-group +GHORG_EXIT_CODE_ON_CLONE_ISSUES=0 ghorg clone $GITLAB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --output-dir=examples-flat diff --git a/scripts/local-gitlab/configs/test-scenarios.json b/scripts/local-gitlab/configs/test-scenarios.json index 6a3fe827..b6746d17 100644 --- a/scripts/local-gitlab/configs/test-scenarios.json +++ b/scripts/local-gitlab/configs/test-scenarios.json @@ -495,6 +495,116 @@ "local-gitlab-latest-all-users/testuser1-repo" ] }, + { + "name": "user-clone-wiki-preserve-dir-consistency", + "description": "Test that user cloning with wikis and preserve-dir produces consistent directory structure for repos and wikis", + "command": "ghorg clone testuser1 --scm=gitlab --clone-type=user --base-url={{.BaseURL}} --token={{.Token}} --preserve-dir --clone-wiki --output-dir=testuser1-wiki-preserve-dir-test", + "run_twice": true, + "expected_structure": [ + "testuser1-wiki-preserve-dir-test/testuser1-repo", + "testuser1-wiki-preserve-dir-test/testuser1-repo.wiki" + ] + }, + { + "name": "group-clone-wiki-preserve-dir-consistency", + "description": "Test group cloning with wikis preserves consistent directory structure", + "command": "ghorg clone local-gitlab-group1 --scm=gitlab --base-url={{.BaseURL}} --token={{.Token}} --preserve-dir --clone-wiki --output-dir=group1-wiki-preserve-dir-test", + "run_twice": false, + "expected_structure": [ + "group1-wiki-preserve-dir-test/baz0", + "group1-wiki-preserve-dir-test/baz0.wiki", + "group1-wiki-preserve-dir-test/baz1", + "group1-wiki-preserve-dir-test/baz1.wiki", + "group1-wiki-preserve-dir-test/baz2", + "group1-wiki-preserve-dir-test/baz2.wiki", + "group1-wiki-preserve-dir-test/baz3", + "group1-wiki-preserve-dir-test/baz3.wiki" + ] + }, + { + "name": "all-groups-clone-wiki-preserve-dir-consistency", + "description": "Test all-groups cloning with wikis preserves consistent directory structure", + "command": "ghorg clone all-groups --scm=gitlab --base-url={{.BaseURL}} --token={{.Token}} --preserve-dir --clone-wiki --output-dir=all-groups-wiki-preserve-dir-test", + "run_twice": false, + "expected_structure": [ + "all-groups-wiki-preserve-dir-test/local-gitlab-group1/baz0", + "all-groups-wiki-preserve-dir-test/local-gitlab-group1/baz0.wiki", + "all-groups-wiki-preserve-dir-test/local-gitlab-group1/baz1", + "all-groups-wiki-preserve-dir-test/local-gitlab-group1/baz1.wiki", + "all-groups-wiki-preserve-dir-test/local-gitlab-group2/baz0", + "all-groups-wiki-preserve-dir-test/local-gitlab-group2/baz0.wiki", + "all-groups-wiki-preserve-dir-test/local-gitlab-group3/subgroup-a/subgroup_a_repo_0", + "all-groups-wiki-preserve-dir-test/local-gitlab-group3/subgroup-a/subgroup_a_repo_0.wiki" + ] + }, + { + "name": "all-users-clone-wiki-preserve-dir-consistency", + "description": "Test all-users cloning with wikis preserves consistent directory structure", + "command": "ghorg clone all-users --clone-type=user --scm=gitlab --base-url={{.BaseURL}} --token={{.Token}} --preserve-dir --clone-wiki --output-dir=all-users-wiki-preserve-dir-test", + "run_twice": false, + "expected_structure": [ + "all-users-wiki-preserve-dir-test/root/rootrepos0", + "all-users-wiki-preserve-dir-test/root/rootrepos0.wiki", + "all-users-wiki-preserve-dir-test/testuser1/testuser1-repo", + "all-users-wiki-preserve-dir-test/testuser1/testuser1-repo.wiki" + ] + }, + { + "name": "single-group-snippets-hosted-instance-fix", + "description": "Test that single group snippet cloning works on hosted instances (regression test for bug fix)", + "command": "ghorg clone local-gitlab-group2 --scm=gitlab --base-url={{.BaseURL}} --token={{.Token}} --clone-snippets --output-dir=single-group-snippets-test", + "run_twice": false, + "expected_structure": [ + "single-group-snippets-test/baz0", + "single-group-snippets-test/baz0.snippets", + "single-group-snippets-test/baz1", + "single-group-snippets-test/baz1.snippets", + "single-group-snippets-test/baz2", + "single-group-snippets-test/baz2.snippets", + "single-group-snippets-test/baz3", + "single-group-snippets-test/baz3.snippets" + ] + }, + { + "name": "group-clone-snippets-preserve-dir-consistency", + "description": "Test group cloning with snippets preserves consistent directory structure", + "command": "ghorg clone local-gitlab-group2 --scm=gitlab --base-url={{.BaseURL}} --token={{.Token}} --preserve-dir --clone-snippets --output-dir=group2-snippets-preserve-dir-test", + "run_twice": false, + "expected_structure": [ + "group2-snippets-preserve-dir-test/baz0", + "group2-snippets-preserve-dir-test/baz0.snippets", + "group2-snippets-preserve-dir-test/baz1", + "group2-snippets-preserve-dir-test/baz1.snippets", + "group2-snippets-preserve-dir-test/baz2", + "group2-snippets-preserve-dir-test/baz2.snippets", + "group2-snippets-preserve-dir-test/baz3", + "group2-snippets-preserve-dir-test/baz3.snippets" + ] + }, + { + "name": "all-groups-clone-snippets-preserve-dir-consistency", + "description": "Test all-groups cloning with snippets preserves consistent directory structure", + "command": "ghorg clone all-groups --scm=gitlab --base-url={{.BaseURL}} --token={{.Token}} --preserve-dir --clone-snippets --output-dir=all-groups-snippets-preserve-dir-test", + "run_twice": false, + "expected_structure": [ + "all-groups-snippets-preserve-dir-test/local-gitlab-group2/baz0", + "all-groups-snippets-preserve-dir-test/local-gitlab-group2/baz0.snippets", + "all-groups-snippets-preserve-dir-test/local-gitlab-group3/subgroup-a/subgroup_a_repo_0", + "all-groups-snippets-preserve-dir-test/local-gitlab-group3/subgroup-a/subgroup_a_repo_0.snippets" + ] + }, + { + "name": "all-users-clone-snippets-preserve-dir-consistency", + "description": "Test all-users cloning with snippets preserves consistent directory structure", + "command": "ghorg clone all-users --clone-type=user --scm=gitlab --base-url={{.BaseURL}} --token={{.Token}} --preserve-dir --clone-snippets --output-dir=all-users-snippets-preserve-dir-test", + "run_twice": false, + "expected_structure": [ + "all-users-snippets-preserve-dir-test/root/rootrepos1", + "all-users-snippets-preserve-dir-test/root/rootrepos1.snippets", + "all-users-snippets-preserve-dir-test/testuser1/testuser1-repo", + "all-users-snippets-preserve-dir-test/testuser1/testuser1-repo.snippets" + ] + }, { "_comment": "COMMENTED OUT TEST FROM ORIGINAL SCRIPT - TODO FIXME", "name": "root-level-snippets-test-disabled", diff --git a/scripts/local-gitlab/integration-tests.sh b/scripts/local-gitlab/integration-tests.sh index a24e030b..b0257e5b 100755 --- a/scripts/local-gitlab/integration-tests.sh +++ b/scripts/local-gitlab/integration-tests.sh @@ -48,6 +48,17 @@ if [[ ! -f "${TEST_RUNNER_BINARY}" ]] || [[ "${TEST_RUNNER_DIR}/main.go" -nt "${ cd - fi +# Install ghorg binary for testing if not in CI +if [[ "${CI:-}" != "true" ]] && [[ "${GITHUB_ACTIONS:-}" != "true" ]]; then + echo "Installing ghorg binary for testing..." + GHORG_PROJECT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" + cd "${GHORG_PROJECT_DIR}" + go install . + cd - + echo "Using ghorg binary: $(which ghorg)" + echo "Ghorg version: $(ghorg version)" +fi + # Run the integration tests echo "Running GitLab integration tests..." "${TEST_RUNNER_BINARY}" \ diff --git a/scripts/local-gitlab/test-runner/main.go b/scripts/local-gitlab/test-runner/main.go index 9745b209..20a69547 100644 --- a/scripts/local-gitlab/test-runner/main.go +++ b/scripts/local-gitlab/test-runner/main.go @@ -4,7 +4,6 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" "log" "os" "os/exec" @@ -40,7 +39,7 @@ type TestRunner struct { } func NewTestRunner(configPath string, context *TestContext) (*TestRunner, error) { - data, err := ioutil.ReadFile(configPath) + data, err := os.ReadFile(configPath) if err != nil { return nil, fmt.Errorf("failed to read test config: %w", err) } diff --git a/scripts/windows_gitlab_integration_tests.bat b/scripts/windows_gitlab_integration_tests.bat index c2982f63..3658c271 100644 --- a/scripts/windows_gitlab_integration_tests.bat +++ b/scripts/windows_gitlab_integration_tests.bat @@ -5,6 +5,7 @@ echo Starting Windows GitLab Integration Tests REM Set GitLab test group (same as Linux version) set GITLAB_GROUP=gitlab-examples set GITLAB_SUB_GROUP=wayne-enterprises +set GHORG_EXIT_CODE_ON_CLONE_ISSUES=0 echo. echo ========================================