From e0006ec2722dcc50ee9601cb1550fda7b1e63be2 Mon Sep 17 00:00:00 2001 From: vqrkxo <108154228+vqrkxo@users.noreply.github.com> Date: Tue, 28 Jun 2022 12:05:29 +0200 Subject: [PATCH] feat(aur): add option to limit concurrent downloads (#1768) * feat(aur): add option to limit concurrent downloads Adds to config file option 'maxconcurrentdownloads' which if set to value != 0, limits number of concurrent AUR downloads to specified. Fixes #1763. * fix lint issue --- aur_source.go | 6 +- aur_source_test.go | 30 ++++--- install.go | 3 +- pkg/settings/config.go | 174 +++++++++++++++++++++-------------------- 4 files changed, 113 insertions(+), 100 deletions(-) diff --git a/aur_source.go b/aur_source.go index 13bc704..d693ed0 100644 --- a/aur_source.go +++ b/aur_source.go @@ -65,7 +65,7 @@ func downloadPKGBUILDSourceWorker(ctx context.Context, wg *sync.WaitGroup, dest } func downloadPKGBUILDSourceFanout(ctx context.Context, cmdBuilder exe.ICmdBuilder, dest string, - bases []dep.Base, incompatible stringset.StringSet) error { + bases []dep.Base, incompatible stringset.StringSet, maxConcurrentDownloads int) error { if len(bases) == 1 { return downloadPKGBUILDSource(ctx, cmdBuilder, dest, bases[0].Pkgbase(), incompatible) } @@ -78,6 +78,10 @@ func downloadPKGBUILDSourceFanout(ctx context.Context, cmdBuilder exe.ICmdBuilde fanInChanErrors = make(chan error) ) + if maxConcurrentDownloads != 0 { + numOfWorkers = maxConcurrentDownloads + } + go func() { for _, base := range bases { c <- base.Pkgbase() diff --git a/aur_source_test.go b/aur_source_test.go index 2aeb275..46728ad 100644 --- a/aur_source_test.go +++ b/aur_source_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "os/exec" "sync/atomic" "testing" @@ -82,13 +83,6 @@ func Test_downloadPKGBUILDSourceError(t *testing.T) { // THEN 5 calls should be made to makepkg func Test_downloadPKGBUILDSourceFanout(t *testing.T) { t.Parallel() - cmdBuilder := &TestMakepkgBuilder{ - parentBuilder: &exe.CmdBuilder{ - MakepkgConfPath: "/etc/not.conf", - MakepkgFlags: []string{"--nocheck"}, MakepkgBin: "makepkg", - }, - test: t, - } bases := []dep.Base{ {&aur.Pkg{PackageBase: "yay"}}, @@ -98,9 +92,21 @@ func Test_downloadPKGBUILDSourceFanout(t *testing.T) { {&aur.Pkg{PackageBase: "yay-v12"}}, } - err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make()) - assert.NoError(t, err) - assert.Equal(t, 5, int(cmdBuilder.passes)) + for _, maxConcurrentDownloads := range []int{0, 3} { + t.Run(fmt.Sprintf("maxconcurrentdownloads set to %d", maxConcurrentDownloads), func(t *testing.T) { + cmdBuilder := &TestMakepkgBuilder{ + parentBuilder: &exe.CmdBuilder{ + MakepkgConfPath: "/etc/not.conf", + MakepkgFlags: []string{"--nocheck"}, MakepkgBin: "makepkg", + }, + test: t, + } + + err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make(), maxConcurrentDownloads) + assert.NoError(t, err) + assert.Equal(t, 5, int(cmdBuilder.passes)) + }) + } } // GIVEN 1 package @@ -120,7 +126,7 @@ func Test_downloadPKGBUILDSourceFanoutNoCC(t *testing.T) { {&aur.Pkg{PackageBase: "yay"}}, } - err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make()) + err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make(), 0) assert.NoError(t, err) assert.Equal(t, 1, int(cmdBuilder.passes)) } @@ -147,7 +153,7 @@ func Test_downloadPKGBUILDSourceFanoutError(t *testing.T) { {&aur.Pkg{PackageBase: "yay-v12"}}, } - err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make()) + err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make(), 0) assert.Error(t, err) assert.Equal(t, 5, int(cmdBuilder.passes)) assert.Len(t, err.(*multierror.MultiError).Errors, 5) diff --git a/install.go b/install.go index 14050b1..2c643b0 100644 --- a/install.go +++ b/install.go @@ -330,7 +330,8 @@ func install(ctx context.Context, cmdArgs *parser.Arguments, dbExecutor db.Execu config.AURURL, config.Runtime.CompletionPath, config.CompletionInterval, false) }() - if errP := downloadPKGBUILDSourceFanout(ctx, config.Runtime.CmdBuilder, config.BuildDir, do.Aur, incompatible); errP != nil { + if errP := downloadPKGBUILDSourceFanout(ctx, config.Runtime.CmdBuilder, config.BuildDir, + do.Aur, incompatible, config.MaxConcurrentDownloads); errP != nil { text.Errorln(errP) } diff --git a/pkg/settings/config.go b/pkg/settings/config.go index be9c073..600b85c 100644 --- a/pkg/settings/config.go +++ b/pkg/settings/config.go @@ -30,50 +30,51 @@ var NoConfirm = false // Configuration stores yay's config. type Configuration struct { - AURURL string `json:"aururl"` - BuildDir string `json:"buildDir"` - Editor string `json:"editor"` - EditorFlags string `json:"editorflags"` - MakepkgBin string `json:"makepkgbin"` - MakepkgConf string `json:"makepkgconf"` - PacmanBin string `json:"pacmanbin"` - PacmanConf string `json:"pacmanconf"` - ReDownload string `json:"redownload"` - ReBuild string `json:"rebuild"` - AnswerClean string `json:"answerclean"` - AnswerDiff string `json:"answerdiff"` - AnswerEdit string `json:"answeredit"` - AnswerUpgrade string `json:"answerupgrade"` - GitBin string `json:"gitbin"` - GpgBin string `json:"gpgbin"` - GpgFlags string `json:"gpgflags"` - MFlags string `json:"mflags"` - SortBy string `json:"sortby"` - SearchBy string `json:"searchby"` - GitFlags string `json:"gitflags"` - RemoveMake string `json:"removemake"` - SudoBin string `json:"sudobin"` - SudoFlags string `json:"sudoflags"` - RequestSplitN int `json:"requestsplitn"` - CompletionInterval int `json:"completionrefreshtime"` - BottomUp bool `json:"bottomup"` - SudoLoop bool `json:"sudoloop"` - TimeUpdate bool `json:"timeupdate"` - Devel bool `json:"devel"` - CleanAfter bool `json:"cleanAfter"` - Provides bool `json:"provides"` - PGPFetch bool `json:"pgpfetch"` - UpgradeMenu bool `json:"upgrademenu"` - CleanMenu bool `json:"cleanmenu"` - DiffMenu bool `json:"diffmenu"` - EditMenu bool `json:"editmenu"` - CombinedUpgrade bool `json:"combinedupgrade"` - UseAsk bool `json:"useask"` - BatchInstall bool `json:"batchinstall"` - SingleLineResults bool `json:"singlelineresults"` - SeparateSources bool `json:"separatesources"` - Runtime *Runtime `json:"-"` - Version string `json:"version"` + AURURL string `json:"aururl"` + BuildDir string `json:"buildDir"` + Editor string `json:"editor"` + EditorFlags string `json:"editorflags"` + MakepkgBin string `json:"makepkgbin"` + MakepkgConf string `json:"makepkgconf"` + PacmanBin string `json:"pacmanbin"` + PacmanConf string `json:"pacmanconf"` + ReDownload string `json:"redownload"` + ReBuild string `json:"rebuild"` + AnswerClean string `json:"answerclean"` + AnswerDiff string `json:"answerdiff"` + AnswerEdit string `json:"answeredit"` + AnswerUpgrade string `json:"answerupgrade"` + GitBin string `json:"gitbin"` + GpgBin string `json:"gpgbin"` + GpgFlags string `json:"gpgflags"` + MFlags string `json:"mflags"` + SortBy string `json:"sortby"` + SearchBy string `json:"searchby"` + GitFlags string `json:"gitflags"` + RemoveMake string `json:"removemake"` + SudoBin string `json:"sudobin"` + SudoFlags string `json:"sudoflags"` + RequestSplitN int `json:"requestsplitn"` + CompletionInterval int `json:"completionrefreshtime"` + MaxConcurrentDownloads int `json:"maxconcurrentdownloads"` + BottomUp bool `json:"bottomup"` + SudoLoop bool `json:"sudoloop"` + TimeUpdate bool `json:"timeupdate"` + Devel bool `json:"devel"` + CleanAfter bool `json:"cleanAfter"` + Provides bool `json:"provides"` + PGPFetch bool `json:"pgpfetch"` + UpgradeMenu bool `json:"upgrademenu"` + CleanMenu bool `json:"cleanmenu"` + DiffMenu bool `json:"diffmenu"` + EditMenu bool `json:"editmenu"` + CombinedUpgrade bool `json:"combinedupgrade"` + UseAsk bool `json:"useask"` + BatchInstall bool `json:"batchinstall"` + SingleLineResults bool `json:"singlelineresults"` + SeparateSources bool `json:"separatesources"` + Runtime *Runtime `json:"-"` + Version string `json:"version"` } // SaveConfig writes yay config to file. @@ -178,48 +179,49 @@ func (c *Configuration) setPrivilegeElevator() error { func DefaultConfig(version string) *Configuration { return &Configuration{ - AURURL: "https://aur.archlinux.org", - BuildDir: os.ExpandEnv("$HOME/.cache/yay"), - CleanAfter: false, - Editor: "", - EditorFlags: "", - Devel: false, - MakepkgBin: "makepkg", - MakepkgConf: "", - PacmanBin: "pacman", - PGPFetch: true, - PacmanConf: "/etc/pacman.conf", - GpgFlags: "", - MFlags: "", - GitFlags: "", - BottomUp: true, - CompletionInterval: 7, - SortBy: "votes", - SearchBy: "name-desc", - SudoLoop: false, - GitBin: "git", - GpgBin: "gpg", - SudoBin: "sudo", - SudoFlags: "", - TimeUpdate: false, - RequestSplitN: 150, - ReDownload: "no", - ReBuild: "no", - BatchInstall: false, - AnswerClean: "", - AnswerDiff: "", - AnswerEdit: "", - AnswerUpgrade: "", - RemoveMake: "ask", - Provides: true, - UpgradeMenu: true, - CleanMenu: true, - DiffMenu: true, - EditMenu: false, - UseAsk: false, - CombinedUpgrade: false, - SeparateSources: true, - Version: version, + AURURL: "https://aur.archlinux.org", + BuildDir: os.ExpandEnv("$HOME/.cache/yay"), + CleanAfter: false, + Editor: "", + EditorFlags: "", + Devel: false, + MakepkgBin: "makepkg", + MakepkgConf: "", + PacmanBin: "pacman", + PGPFetch: true, + PacmanConf: "/etc/pacman.conf", + GpgFlags: "", + MFlags: "", + GitFlags: "", + BottomUp: true, + CompletionInterval: 7, + MaxConcurrentDownloads: 0, + SortBy: "votes", + SearchBy: "name-desc", + SudoLoop: false, + GitBin: "git", + GpgBin: "gpg", + SudoBin: "sudo", + SudoFlags: "", + TimeUpdate: false, + RequestSplitN: 150, + ReDownload: "no", + ReBuild: "no", + BatchInstall: false, + AnswerClean: "", + AnswerDiff: "", + AnswerEdit: "", + AnswerUpgrade: "", + RemoveMake: "ask", + Provides: true, + UpgradeMenu: true, + CleanMenu: true, + DiffMenu: true, + EditMenu: false, + UseAsk: false, + CombinedUpgrade: false, + SeparateSources: true, + Version: version, } }