ghorg/scm/github.go

package scm

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"github.com/bradleyfalzon/ghinstallation/v2"
	"github.com/google/go-github/v72/github"
	"golang.org/x/oauth2"
)

var (
	_             Client = Github{}
	reposPerPage         = 100
	tokenUsername        = ""
)

func init() {
	registerClient(Github{})
}

type Github struct {
	// extend the github client
	*github.Client
	// perPage contain the pagination item limit
	perPage int
}

func (_ Github) GetType() string {
	return "github"
}

// GetOrgRepos gets org repos with parallel pagination for performance
func (c Github) GetOrgRepos(targetOrg string) ([]Repo, error) {

	opt := &github.RepositoryListByOrgOptions{
		Type:        "all",
		ListOptions: github.ListOptions{PerPage: c.perPage},
	}

	c.SetTokensUsername()

	spinningSpinner.Start()
	defer spinningSpinner.Stop()

	// Fetch first page to discover total number of pages
	repos, resp, err := c.Repositories.ListByOrg(context.Background(), targetOrg, opt)
	if err != nil {
		return nil, err
	}

	// If only one page, return immediately
	if resp.LastPage == 0 || resp.LastPage == 1 {
		return c.filter(repos), nil
	}

	// Multiple pages - fetch remaining pages in parallel
	return c.fetchOrgReposParallel(targetOrg, repos, resp.LastPage)
}

// GetUserRepos gets user repos with parallel pagination for performance
func (c Github) GetUserRepos(targetUser string) ([]Repo, error) {
	if os.Getenv("GHORG_SCM_BASE_URL") != "" {
		c.BaseURL, _ = url.Parse(os.Getenv("GHORG_SCM_BASE_URL"))
	}

	c.SetTokensUsername()

	spinningSpinner.Start()
	defer spinningSpinner.Stop()

	opt := &github.ListOptions{PerPage: c.perPage, Page: 1}

	// Fetch first page to discover total number of pages
	var repos []*github.Repository
	var resp *github.Response
	var err error

	if targetUser == tokenUsername {
		authOpt := &github.RepositoryListByAuthenticatedUserOptions{
			Type:        os.Getenv("GHORG_GITHUB_USER_OPTION"),
			ListOptions: *opt,
		}
		repos, resp, err = c.Repositories.ListByAuthenticatedUser(context.Background(), authOpt)
	} else {
		userOpt := &github.RepositoryListByUserOptions{
			Type:        os.Getenv("GHORG_GITHUB_USER_OPTION"),
			ListOptions: *opt,
		}
		repos, resp, err = c.Repositories.ListByUser(context.Background(), targetUser, userOpt)
	}

	if err != nil {
		return nil, err
	}

	// Filter user repos if needed
	if targetUser != tokenUsername {
		userRepos := []*github.Repository{}
		for _, repo := range repos {
			if repo.Owner != nil && repo.Owner.Type != nil && *repo.Owner.Type == "User" {
				userRepos = append(userRepos, repo)
			}
		}
		repos = userRepos
	}

	// If only one page, return immediately
	if resp.LastPage == 0 || resp.LastPage == 1 {
		return c.filter(repos), nil
	}

	// Multiple pages - fetch remaining pages in parallel
	return c.fetchUserReposParallel(targetUser, repos, resp.LastPage)
}

// NewClient create new github scm client
func (_ Github) NewClient() (Client, error) {
	ctx := context.Background()
	var tc *http.Client

	if os.Getenv("GHORG_GITHUB_TOKEN") != "" {
		ts := oauth2.StaticTokenSource(
			&oauth2.Token{AccessToken: os.Getenv("GHORG_GITHUB_TOKEN")},
		)
		tc = oauth2.NewClient(ctx, ts)
	}

	// Authenticate as a GitHub App
	// If the user has set GHORG_GITHUB_APP_PEM_PATH, we assume they want to use a GitHub App
	if os.Getenv("GHORG_GITHUB_APP_PEM_PATH") != "" {
		// If the user has set GHORG_GITHUB_APP_INSTALLATION_ID, we assume they want to use a GitHub App
		installID, err := strconv.ParseInt(os.Getenv("GHORG_GITHUB_APP_INSTALLATION_ID"), 10, 64)

		if err != nil {
			return nil, fmt.Errorf("GHORG_GITHUB_APP_INSTALLATION_ID must be set if GHORG_GITHUB_APP_PEM_PATH is set")
		}

		appID, err := strconv.ParseInt(os.Getenv("GHORG_GITHUB_APP_ID"), 10, 64)

		if err != nil {
			return nil, fmt.Errorf("GHORG_GITHUB_APP_ID must be set if GHORG_GITHUB_APP_PEM_PATH is set")
		}

		itr, err := ghinstallation.NewKeyFromFile(
			http.DefaultTransport,
			appID,
			installID,
			os.Getenv("GHORG_GITHUB_APP_PEM_PATH"),
		)
		if err != nil {
			return nil, err
		}
		tc = &http.Client{Transport: itr}
		// get the token from the itr and update the GHORT_GITHUB_TOKEN env var
		token, err := itr.Token(ctx)

		if err != nil {
			return nil, err
		}
		os.Setenv("GHORG_GITHUB_TOKEN", token)
	}

	baseURL := os.Getenv("GHORG_SCM_BASE_URL")
	var ghClient *github.Client

	if baseURL != "" {
		ghClient = github.NewClient(tc)
		ghClient, _ = ghClient.WithEnterpriseURLs(baseURL, baseURL)
	} else {
		ghClient = github.NewClient(tc)
	}

	client := Github{Client: ghClient, perPage: reposPerPage}

	return client, nil
}

func (_ Github) addTokenToHTTPSCloneURL(url string, token string) string {
	splitURL := strings.Split(url, "https://")
	return "https://" + tokenUsername + ":" + token + "@" + splitURL[1]
}

func (c Github) filter(allRepos []*github.Repository) []Repo {
	var repoData []Repo

	for _, ghRepo := range allRepos {

		if os.Getenv("GHORG_SKIP_ARCHIVED") == "true" {
			if *ghRepo.Archived {
				continue
			}
		}

		if os.Getenv("GHORG_SKIP_FORKS") == "true" {
			if *ghRepo.Fork {
				continue
			}
		}

		if !hasMatchingTopic(ghRepo.Topics) {
			continue
		}

		// NOTE: for some reason forks do not always have a language field set so sometimes they get filtered out
		if os.Getenv("GHORG_GITHUB_FILTER_LANGUAGE") != "" {
			if ghRepo.Language != nil {
				ghLang := strings.ToLower(*ghRepo.Language)
				userLangs := strings.Split(strings.ToLower(os.Getenv("GHORG_GITHUB_FILTER_LANGUAGE")), ",")
				matched := false
				for _, userLang := range userLangs {
					if ghLang == userLang {
						matched = true
						break
					}
				}
				if !matched {
					continue
				}
			} else {
				continue
			}
		}

		r := Repo{}

		r.Name = *ghRepo.Name
		r.Path = r.Name

		if os.Getenv("GHORG_BRANCH") == "" {
			defaultBranch := ghRepo.GetDefaultBranch()
			if defaultBranch == "" {
				defaultBranch = "master"
			}
			r.CloneBranch = defaultBranch
		} else {
			r.CloneBranch = os.Getenv("GHORG_BRANCH")
		}

		if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" || os.Getenv("GHORG_GITHUB_APP_PEM_PATH") != "" {
			r.CloneURL = c.addTokenToHTTPSCloneURL(*ghRepo.CloneURL, os.Getenv("GHORG_GITHUB_TOKEN"))
			r.URL = *ghRepo.CloneURL
			repoData = append(repoData, r)
		} else {
			r.CloneURL = ReplaceSSHHostname(*ghRepo.SSHURL, os.Getenv("GHORG_SSH_HOSTNAME"))
			r.URL = *ghRepo.SSHURL
			repoData = append(repoData, r)
		}

		if ghRepo.GetHasWiki() && os.Getenv("GHORG_CLONE_WIKI") == "true" {
			wiki := Repo{}
			wiki.IsWiki = true
			wiki.CloneURL = strings.Replace(r.CloneURL, ".git", ".wiki.git", 1)
			wiki.URL = strings.Replace(r.URL, ".git", ".wiki.git", 1)
			wiki.CloneBranch = "master"
			wiki.Path = fmt.Sprintf("%s%s", r.Name, ".wiki")
			repoData = append(repoData, wiki)
		}
	}

	return repoData
}

// GetUserGists gets all gists for a GitHub user
func (c Github) GetUserGists(targetUser string) ([]Repo, error) {
	c.SetTokensUsername()

	spinningSpinner.Start()
	defer spinningSpinner.Stop()

	opt := &github.GistListOptions{
		ListOptions: github.ListOptions{PerPage: reposPerPage, Page: 1},
	}

	gists, resp, err := c.Gists.List(context.Background(), targetUser, opt)
	if err != nil {
		return nil, err
	}

	allGists := gists

	for page := 2; page <= resp.LastPage; page++ {
		opt.Page = page
		g, _, err := c.Gists.List(context.Background(), targetUser, opt)
		if err != nil {
			return nil, err
		}
		allGists = append(allGists, g...)
	}

	return c.filterGists(allGists), nil
}

// gistFolderName returns the folder name to use for a gist, derived from the
// gist's primary filename (first filename alphabetically). The extension is
// stripped and the result is lowercased. Falls back to the gist ID when the
// gist has no files or the derived base name is empty.
func gistFolderName(gist *github.Gist) string {
	if len(gist.Files) == 0 {
		return strings.ToLower(*gist.ID)
	}

	// Collect and sort filenames for a stable, deterministic result.
	filenames := make([]string, 0, len(gist.Files))
	for fn := range gist.Files {
		filenames = append(filenames, string(fn))
	}
	sort.Strings(filenames)

	firstName := filenames[0]
	ext := filepath.Ext(firstName)
	base := strings.ToLower(firstName[:len(firstName)-len(ext)])
	if base == "" {
		return strings.ToLower(*gist.ID)
	}
	return base
}

func (c Github) filterGists(allGists []*github.Gist) []Repo {
	// First pass: compute a derived folder name for every valid gist and
	// count how many gists share the same name so we can detect collisions.
	type gistEntry struct {
		gist       *github.Gist
		folderName string
	}

	entries := make([]gistEntry, 0, len(allGists))
	nameCount := make(map[string]int)

	for _, gist := range allGists {
		if gist.ID == nil || gist.GitPullURL == nil {
			continue
		}
		name := gistFolderName(gist)
		entries = append(entries, gistEntry{gist: gist, folderName: name})
		nameCount[name]++
	}

	// Second pass: build Repo records, appending the gist ID to the folder
	// name whenever two or more gists share the same derived name.
	var repoData []Repo

	for _, entry := range entries {
		gist := entry.gist
		folderName := entry.folderName

		if nameCount[folderName] > 1 {
			folderName = folderName + "-" + *gist.ID
		}

		r := Repo{}
		r.ID = *gist.ID
		r.Name = folderName
		r.Path = folderName
		r.IsGitHubGist = true
		if os.Getenv("GHORG_BRANCH") != "" {
			r.CloneBranch = os.Getenv("GHORG_BRANCH")
		} else {
			r.CloneBranch = "master"
		}
		r.URL = *gist.GitPullURL

		if os.Getenv("GHORG_CLONE_PROTOCOL") == "https" || os.Getenv("GHORG_GITHUB_APP_PEM_PATH") != "" {
			r.CloneURL = c.addTokenToHTTPSCloneURL(*gist.GitPullURL, os.Getenv("GHORG_GITHUB_TOKEN"))
		} else {
			// Convert HTTPS gist pull URL to SSH URL
			// https://gist.github.com/<id>.git → git@gist.github.com:<id>.git
			sshURL := strings.Replace(*gist.GitPullURL, "https://gist.github.com/", "git@gist.github.com:", 1)
			r.CloneURL = ReplaceSSHHostname(sshURL, os.Getenv("GHORG_SSH_HOSTNAME"))
		}

		repoData = append(repoData, r)
	}

	return repoData
}

// Sets the GitHub username tied to the github token to the package variable tokenUsername
// Then if https clone method is used the clone url will be https://username:token@github.com/org/repo.git
// The username is now needed when using the new fine-grained tokens for github
func (c Github) SetTokensUsername() {
	if os.Getenv("GHORG_GITHUB_TOKEN_FROM_GITHUB_APP") == "true" || os.Getenv("GHORG_GITHUB_APP_PEM_PATH") != "" {
		tokenUsername = "x-access-token"
		return
	}
	userToken, _, err := c.Users.Get(context.Background(), "")
	if err != nil || userToken == nil {
		// Fallback to x-access-token if we can't get the username
		// This prevents panics and allows cloning to continue
		tokenUsername = "x-access-token"
		return
	}
	login := userToken.GetLogin()
	if login == "" {
		tokenUsername = "x-access-token"
		return
	}
	tokenUsername = login
}