ghorg/scm/gitlab.go

package scm

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/gabrie30/ghorg/colorlog"
	gitlab "github.com/xanzy/go-gitlab"
)

var (
	_Client         = Gitlab{}
	perPage         = 100
	gitLabAllGroups = false
	gitLabAllUsers  = false
)

func init() {
	registerClient(Gitlab{})
}

type Gitlab struct {
	// extend the gitlab client
	*gitlab.Client
}

func (_ Gitlab) GetType() string {
	return "gitlab"
}

func (_ Gitlab) rootLevelSnippet(url string) bool {
	baseURL := os.Getenv("GHORG_SCM_BASE_URL")
	if baseURL != "" {
		customSnippetPattern := regexp.MustCompile(`^` + baseURL + `/-/snippets/\d+$`)
		if customSnippetPattern.MatchString(url) {
			return true
		}
		return false
	} else {
		// cloud instances
		// Check if the URL follows the pattern of a root level snippet
		rootLevelSnippetPattern := regexp.MustCompile(`^https://gitlab\.com/-/snippets/\d+$`)
		if rootLevelSnippetPattern.MatchString(url) {
			return true
		}
		return false
	}
}

// GetOrgRepos fetches repo data from a specific group
func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
	allGroups := []string{}
	repoData := []Repo{}
	longFetch := false

	if targetOrg == "all-users" {
		colorlog.PrintErrorAndExit("When using the 'all-users' keyword the '--clone-type=user' flag should be set")
	}

	spinningSpinner.Start()
	defer spinningSpinner.Stop()

	if targetOrg == "all-groups" {
		gitLabAllGroups = true
		longFetch = true

		grps, err := c.GetTopLevelGroups()
		if err != nil {
			return nil, fmt.Errorf("error getting groups error: %v", err)
		}

		allGroups = append(allGroups, grps...)

	} else {
		allGroups = append(allGroups, targetOrg)
	}

	if os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX") != "" {
		allGroups = filterGitlabGroupByExcludeMatchRegex(allGroups)
	}

	for i, group := range allGroups {
		if longFetch {
			spinningSpinner.Stop()
			if i == 0 {
				fmt.Println("")
			}
			msg := fmt.Sprintf("fetching repos for group: %v", group)
			colorlog.PrintInfo(msg)
		}
		repos, err := c.GetGroupRepos(group)
		if err != nil {
			return nil, fmt.Errorf("error fetching repos for group '%s', error: %v", group, err)
		}

		repoData = append(repoData, repos...)

	}

	snippets, err := c.GetSnippets(repoData, targetOrg)
	if err != nil {
		spinningSpinner.Stop()
		colorlog.PrintError(fmt.Sprintf("Error getting snippets, error: %v", err))
	}
	repoData = append(repoData, snippets...)

	return repoData, nil
}

// GetTopLevelGroups all top level org groups
func (c Gitlab) GetTopLevelGroups() ([]string, error) {
	allGroups := []string{}

	opt := &gitlab.ListGroupsOptions{
		ListOptions: gitlab.ListOptions{
			PerPage: perPage,
			Page:    1,
		},
		TopLevelOnly: gitlab.Bool(true),
		AllAvailable: gitlab.Bool(true),
	}

	for {

		groups, resp, err := c.Client.Groups.ListGroups(opt)

		if err != nil {
			return allGroups, err
		}

		for _, g := range groups {
			allGroups = append(allGroups, strconv.FormatInt(int64(g.ID), 10))
		}

		// Exit the loop when we've seen all pages.
		if resp.NextPage == 0 {
			break
		}

		// Update the page number to get the next page.
		opt.Page = resp.NextPage
	}

	return allGroups, nil
}

// In this case take the cloneURL from the cloneTartet repo and just inject /snippets/:id before the .git
// cloud example
// http clone target url https://gitlab.com/ghorg-test-group/subgroup-2/foobar.git
// http snippet clone url https://gitlab.com/ghorg-test-group/subgroup-2/foobar/snippets/3711587.git
// ssh clone target url git@gitlab.com:ghorg-test-group/subgroup-2/foobar.git
// ssh snippet clone url git@gitlab.com:ghorg-test-group/subgroup-2/foobar/snippets/3711587.git
func (c Gitlab) createRepoSnippetCloneURL(cloneTargetURL string, snippetID string) string {

	// Split the cloneTargetURL into two parts at the ".git"
	parts := strings.Split(cloneTargetURL, ".git")
	// Insert the "/snippets/:id" before the ".git"
	cloneURL := parts[0] + "/snippets/" + snippetID + ".git"

	if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
		return cloneURL
	}

	// git@gitlab.example.com:local-gitlab-group3/subgroup-a/subgroup-b/subgroup_b_repo_1/snippets/12.git

	// http://gitlab.example.com/snippets/1.git
	if os.Getenv("GHORG_INSECURE_GITLAB_CLIENT") == "true" {
		cloneURL = strings.Replace(cloneURL, "http://", "git@", 1)
	} else {
		cloneURL = strings.Replace(cloneURL, "https://", "git@", 1)
	}
	// git@gitlab.example.com/snippets/1.git
	cloneURL = strings.Replace(cloneURL, "/", ":", 1)
	// git@gitlab.example.com:snippets/1.git
	return cloneURL
}

// hosted example
// root snippet ssh clone url git@gitlab.example.com:snippets/1.git
// root snippet http clone url http://gitlab.example.com/snippets/1.git
func (c Gitlab) createRootLevelSnippetCloneURL(snippetWebURL string) string {
	// Web URL example, http://gitlab.example.com/-/snippets/1
	// Both http and ssh clone urls do not have the /-/ in them so just remove it first and add the .git extention
	cloneURL := strings.Replace(snippetWebURL, "/-/", "/", -1) + ".git"
	if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
		return c.addTokenToCloneURL(cloneURL, os.Getenv("GHORG_GITLAB_TOKEN"))
	}

	if os.Getenv("GHORG_INSECURE_GITLAB_CLIENT") == "true" {
		cloneURL = strings.Replace(cloneURL, "http://", "git@", 1)
	} else {
		cloneURL = strings.Replace(cloneURL, "https://", "git@", 1)
	}
	// git@gitlab.example.com/snippets/1.git
	cloneURL = strings.Replace(cloneURL, "/", ":", 1)
	// git@gitlab.example.com:snippets/1.git
	return cloneURL
}

func (c Gitlab) getRepoSnippets(r Repo) []*gitlab.Snippet {
	var allSnippets []*gitlab.Snippet
	opt := &gitlab.ListProjectSnippetsOptions{
		PerPage: perPage,
		Page:    1,
	}

	for {
		snippets, resp, err := c.ProjectSnippets.ListSnippets(r.ID, opt)

		if resp.StatusCode == 403 {
			break
		}

		if err != nil {
			colorlog.PrintError(fmt.Sprintf("Error fetching snippets for project %s: %v, ignoring error and proceeding to next project", r.Name, err))
			break
		}

		allSnippets = append(allSnippets, snippets...)

		// Exit the loop when we've seen all pages.
		if resp.NextPage == 0 {
			break
		}

		// Update the page number to get the next page.
		opt.Page = resp.NextPage
	}

	return allSnippets
}

func (c Gitlab) getAllSnippets() []*gitlab.Snippet {
	var allSnippets []*gitlab.Snippet
	opt := &gitlab.ListAllSnippetsOptions{
		ListOptions: gitlab.ListOptions{
			PerPage: perPage,
			Page:    1,
		},
	}

	for {
		snippets, resp, err := c.Snippets.ListAllSnippets(opt)
		if err != nil {
			colorlog.PrintError(fmt.Sprintf("Issue fetching all snippets, not all snippets will be cloned error: %v", err))
			return allSnippets
		}

		allSnippets = append(allSnippets, snippets...)

		// Exit the loop when we've seen all pages.
		if resp.NextPage == 0 {
			break
		}

		// Update the page number to get the next page.
		opt.Page = resp.NextPage
	}

	return allSnippets
}

func (c Gitlab) GetSnippets(cloneData []Repo, target string) ([]Repo, error) {

	if os.Getenv("GHORG_CLONE_SNIPPETS") != "true" {
		return []Repo{}, nil
	}

	var allSnippetsToClone []*gitlab.Snippet

	// Snippets are converted into Repos so we can clone them
	snippetsToClone := []Repo{}

	// If it is a cloud group clone iterate over each project and try to get its snippets. We have to do this because if you use the /snippets/all endpoint it will return every public snippet from the cloud.
	if os.Getenv("GHORG_CLONE_TYPE") != "user" && os.Getenv("GHORG_SCM_BASE_URL") == "" {
		// Iterate over all projects in the group. If it has snippets add them
		colorlog.PrintInfo("Note: only snippets you have access to will be cloned. This process may take a while depending on the size of group you are trying to clone, please be patient.")
		for _, repo := range cloneData {
			snippets := c.getRepoSnippets(repo)
			allSnippetsToClone = append(allSnippetsToClone, snippets...)
		}
	} else {
		allSnippets := c.getAllSnippets()

		// if its an all-user or all-group clone, for each repo get its snippets then also include all root level snippets
		if target == "all-users" || target == "all-groups" {
			for _, repo := range cloneData {
				repoSnippets := c.getRepoSnippets(repo)
				allSnippetsToClone = append(allSnippetsToClone, repoSnippets...)
			}

			for _, snippet := range allSnippets {
				if c.rootLevelSnippet(snippet.WebURL) {
					allSnippetsToClone = append(allSnippetsToClone, snippet)
				}
			}
		}

		if os.Getenv("GHORG_CLONE_TYPE") == "user" && os.Getenv("GHORG_SCM_BASE_URL") == "" {

		}

	}

	for _, snippet := range allSnippetsToClone {
		snippetID := strconv.Itoa(snippet.ID)
		snippetTitle := ToSlug(snippet.Title)
		s := Repo{}
		s.IsGitLabSnippet = true
		s.CloneBranch = "main"
		s.GitLabSnippetInfo.Title = snippetTitle
		s.Name = snippetTitle
		s.GitLabSnippetInfo.ID = snippetID
		s.URL = snippet.WebURL
		// If the snippet is not made on any repo its a root level snippet, this works for cloud
		if c.rootLevelSnippet(snippet.WebURL) {
			s.IsGitLabRootLevelSnippet = true
			s.CloneURL = c.createRootLevelSnippetCloneURL(snippet.WebURL)
			cloneData = append(cloneData, s)
		} else {
			// Since this isn't a root level repo we want to find which repo the snippet is coming from
			for _, cloneTarget := range cloneData {
				if cloneTarget.ID == strconv.Itoa(snippet.ProjectID) {
					s.CloneURL = c.createRepoSnippetCloneURL(cloneTarget.CloneURL, snippetID)
					s.Path = cloneTarget.Path
					s.GitLabSnippetInfo.URLOfRepo = cloneTarget.URL
					s.GitLabSnippetInfo.NameOfRepo = cloneTarget.Name
					cloneData = append(cloneData, s)
				}
			}
		}

		snippetsToClone = append(snippetsToClone, s)
	}

	return snippetsToClone, nil
}

// GetGroupRepos fetches repo data from a specific group
func (c Gitlab) GetGroupRepos(targetGroup string) ([]Repo, error) {

	repoData := []Repo{}

	opt := &gitlab.ListGroupProjectsOptions{
		ListOptions: gitlab.ListOptions{
			PerPage: perPage,
			Page:    1,
		},
		IncludeSubGroups: gitlab.Bool(true),
	}

	for {
		// Get the first page with projects.
		ps, resp, err := c.Groups.ListGroupProjects(targetGroup, opt)

		if err != nil {
			if resp != nil && resp.StatusCode == 404 {
				return nil, fmt.Errorf("group '%s' does not exist", targetGroup)
			}
			return []Repo{}, err
		}

		// filter from all the projects we've found so far.
		repoData = append(repoData, c.filter(targetGroup, ps)...)

		// Exit the loop when we've seen all pages.
		if resp.NextPage == 0 {
			break
		}

		// Update the page number to get the next page.
		opt.Page = resp.NextPage
	}

	return repoData, nil
}

// GetUserRepos gets all of a users gitlab repos
func (c Gitlab) GetUserRepos(targetUsername string) ([]Repo, error) {
	cloneData := []Repo{}
	targetUsers := []string{}

	projectOpts := &gitlab.ListProjectsOptions{
		ListOptions: gitlab.ListOptions{
			PerPage: perPage,
			Page:    1,
		},
	}

	userOpts := &gitlab.ListUsersOptions{
		ListOptions: gitlab.ListOptions{
			PerPage: perPage,
			Page:    1,
		},
	}

	spinningSpinner.Start()
	defer spinningSpinner.Stop()

	if targetUsername == "all-users" {
		gitLabAllUsers = true
		for {
			allUsers, resp, err := c.Users.ListUsers(userOpts)
			if err != nil {
				return nil, fmt.Errorf("error getting all users, err: %v", err)
			}

			for _, u := range allUsers {
				targetUsers = append(targetUsers, u.Username)
			}

			if resp.NextPage == 0 {
				break
			}

			// Update the page number to get the next page.
			userOpts.Page = resp.NextPage
		}
	} else {
		targetUsers = append(targetUsers, targetUsername)
	}

	for _, targetUser := range targetUsers {
		for {
			// Get the first page with projects.
			ps, resp, err := c.Projects.ListUserProjects(targetUser, projectOpts)
			if err != nil {
				spinningSpinner.Stop()
				colorlog.PrintError(fmt.Sprintf("Error getting repo for user: %v", targetUser))
				continue
			}

			// filter from all the projects we've found so far.
			cloneData = append(cloneData, c.filter(targetUser, ps)...)

			// Exit the loop when we've seen all pages.
			if resp.NextPage == 0 {
				break
			}

			// Update the page number to get the next page.
			userOpts.Page = resp.NextPage
		}
	}

	snippets, err := c.GetSnippets(cloneData, targetUsername)
	if err != nil {
		spinningSpinner.Stop()
		colorlog.PrintError(fmt.Sprintf("Error getting snippets, error: %v", err))
	}
	cloneData = append(cloneData, snippets...)
	return cloneData, nil
}

// NewClient create new gitlab scm client
func (_ Gitlab) NewClient() (Client, error) {
	baseURL := os.Getenv("GHORG_SCM_BASE_URL")
	token := os.Getenv("GHORG_GITLAB_TOKEN")

	var err error
	var c *gitlab.Client
	if baseURL != "" {
		if os.Getenv("GHORG_INSECURE_GITLAB_CLIENT") == "true" {
			defaultTransport := http.DefaultTransport.(*http.Transport)
			// Create new Transport that ignores self-signed SSL
			customTransport := &http.Transport{
				Proxy:                 defaultTransport.Proxy,
				DialContext:           defaultTransport.DialContext,
				MaxIdleConns:          defaultTransport.MaxIdleConns,
				IdleConnTimeout:       defaultTransport.IdleConnTimeout,
				ExpectContinueTimeout: defaultTransport.ExpectContinueTimeout,
				TLSHandshakeTimeout:   defaultTransport.TLSHandshakeTimeout,
				TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
			}
			client := &http.Client{Transport: customTransport}
			opt := gitlab.WithHTTPClient(client)
			c, err = gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), opt)
			colorlog.PrintError("WARNING: USING AN INSECURE GITLAB CLIENT")
		} else {
			c, err = gitlab.NewClient(token, gitlab.WithBaseURL(baseURL))
		}

	} else {
		c, err = gitlab.NewClient(token)
	}
	return Gitlab{c}, err
}

func (_ Gitlab) addTokenToCloneURL(url string, token string) string {
	// allows for http and https for local testing
	splitURL := strings.Split(url, "://")
	return splitURL[0] + "://oauth2:" + token + "@" + splitURL[1]
}

func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo {
	var repoData []Repo

	isSubgroup := strings.Contains(group, "/")

	for _, p := range ps {

		if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" {
			if p.Archived {
				continue
			}
		}

		if os.Getenv("GHORG_SKIP_FORKS") == "true" {
			if p.ForkedFromProject != nil {
				continue
			}
		}

		if !hasMatchingTopic(p.Topics) {
			continue
		}

		r := Repo{}

		r.Name = p.Name
		r.ID = strconv.Itoa(p.ID)

		if os.Getenv("GHORG_BRANCH") == "" {
			defaultBranch := p.DefaultBranch
			if defaultBranch == "" {
				defaultBranch = "master"
			}
			r.CloneBranch = defaultBranch
		} else {
			r.CloneBranch = os.Getenv("GHORG_BRANCH")
		}

		path := p.PathWithNamespace

		// The PathWithNamespace includes the org/group name
		// https://github.com/gabrie30/ghorg/issues/228
		// https://github.com/gabrie30/ghorg/issues/267
		// https://github.com/gabrie30/ghorg/issues/271
		if !gitLabAllGroups && !gitLabAllUsers {
			if isSubgroup {
				if os.Getenv("GHORG_OUTPUT_DIR") == "" {
					path = strings.TrimPrefix(path, group)
				}
			} else {
				path = strings.TrimPrefix(path, group)
			}
		}

		r.Path = path
		r.ID = fmt.Sprint(p.ID)
		if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" {
			r.CloneURL = c.addTokenToCloneURL(p.HTTPURLToRepo, os.Getenv("GHORG_GITLAB_TOKEN"))
			r.URL = p.HTTPURLToRepo
			repoData = append(repoData, r)
		} else {
			r.CloneURL = p.SSHURLToRepo
			r.URL = p.SSHURLToRepo
			repoData = append(repoData, r)
		}

		if p.WikiEnabled && os.Getenv("GHORG_CLONE_WIKI") == "true" {
			wiki := Repo{}
			// wiki needs name for gitlab name collisions
			wiki.Name = p.Name
			wiki.IsWiki = true
			wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1)
			wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1)
			wiki.CloneBranch = "master"
			wiki.Path = fmt.Sprintf("%s%s", p.PathWithNamespace, ".wiki")
			repoData = append(repoData, wiki)
		}
	}
	return repoData
}

func filterGitlabGroupByExcludeMatchRegex(groups []string) []string {
	filteredGroups := []string{}
	regex := fmt.Sprint(os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX"))

	for i, grp := range groups {
		exclude := false
		re := regexp.MustCompile(regex)
		match := re.FindString(grp)
		if match != "" {
			exclude = true
		}

		if !exclude {
			filteredGroups = append(filteredGroups, groups[i])
		}
	}

	return filteredGroups
}

// ToSlug converts a title into a URL-friendly slug.
func ToSlug(title string) string {
	// Convert to lowercase
	slug := strings.ToLower(title)

	// Replace spaces and special characters with hyphens
	slug = regexp.MustCompile(`[\s\p{P}]+`).ReplaceAllString(slug, "-")

	// Remove any non-alphanumeric characters except for hyphens
	slug = regexp.MustCompile(`[^a-z0-9-]+`).ReplaceAllString(slug, "")

	// Trim any leading or trailing hyphens
	slug = strings.Trim(slug, "-")

	return slug
}