Added -wait command line option (#11)

Co-authored-by: Michel Casabianca <michel.casabianca@intercloud.com>

Author: c4s4

README.md

@@ -91,7 +91,10 @@ Configuration file is in YAML format as follows:
 
 ```yaml
 api-key: "28c6112c-a7bc-4a4e-9b14-75be6da02211"
+wait: false
 strict: false
+ignore:
+- "CVE-2020-14040"
 memcachier:
   address:    "mcx.cy.eu-central-1.ec2.memcachier.com:11211"
   username:   "username"
@@ -105,26 +108,27 @@ memcached:
 file:
   name:       "~/.gobinsec-cache.yml"
   expiration: "24h"
-ignore:
-- "CVE-2020-14040"
 ```
 
 Configuration fields are the following:
 
 - **api-key**: this is your NVD API key
+- **wait**: tells if we should wait between NVD API calls to ensure that we are below rate limits
 - **strict**: tells if we should consider vulnerability matches without version as matching dependency
+- **ignore**: a list of CVE vulnerabilities to ignore
 - **memcachier** is the configuration for *memcachier*, see below
 - **memcached** is the configuration for *memcached*, see below
 - **file** is the configuration for *file* cache, see below
-- **ignore**: a list of CVE vulnerabilities to ignore
 
 You can also set NVD API Key in your environment with variable *NVD_API_KEY*. This key may be overwritten with value in configuration file. Your API key must be set in environment to be able to run integration tests (with target *integ*).
 
 Note that without API key, you will be limited to *10* requests in a rolling *60* second window while this limit is *100* with an API key.
 
 ## Cache
 
-A cache is useful because if you perform more call to NVD database than allowed, your calls will significantly slow down. Gobinsec tries to build caches in this order:
+A cache is useful to limit NVD API calls. If you perform more call to NVD database than allowed, your calls will significantly slow down or you will get status code *403* calling the API.
+
+Gobinsec tries to build caches in this order:
 
 ### Memcachier
 

go.mod

@@ -12,5 +12,5 @@ require (
 require (
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
+	golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect
 )

go.sum

@@ -14,8 +14,8 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc=
+golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=

gobinsec/binary.go

@@ -61,7 +61,11 @@ func (b *Binary) GetDependencies() error {
 		dependencies <- dependency
 		wg.Add(1)
 	}
-	for i := 0; i < NumGoroutines; i++ {
+	numGoroutines := NumGoroutines
+	if config.Wait {
+		numGoroutines = 1
+	}
+	for i := 0; i < numGoroutines; i++ {
 		go LoadVulnerabilities(dependencies, &wg)
 	}
 	wg.Wait()

gobinsec/config.go

@@ -7,6 +7,7 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+// Config is the configuration from YAML config file and command line options
 type Config struct {
 	APIKey     string            `yaml:"api-key"`
 	Memcached  *MemcachedConfig  `yaml:"memcached"`
@@ -16,30 +17,31 @@ type Config struct {
 	Strict     bool              `yaml:"strict"`
 	Verbose    bool              `yaml:"verbose"`
 	Cache      bool              `yaml:"cache"`
+	Wait       bool              `yaml:"wait"`
 }
 
+// config is shared
 var config Config
 
-// LoadConfig loads configuration from given file
-func LoadConfig(path string, strict, verbose, cache bool) error {
-	if path == "" {
-		config.Strict = strict
-		config.Verbose = verbose
-		config.Cache = cache
-		return nil
-	}
-	bytes, err := os.ReadFile(path)
-	if err != nil {
-		return fmt.Errorf("loading configuration file: %v", err)
-	}
-	if err := yaml.Unmarshal(bytes, &config); err != nil {
-		return fmt.Errorf("parsing configuration: %v", err)
+// LoadConfig loads configuration from given file and overwrite with command line options
+func LoadConfig(path string, strict, wait, verbose, cache bool) error {
+	if path != "" {
+		bytes, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("loading configuration file: %v", err)
+		}
+		if err := yaml.Unmarshal(bytes, &config); err != nil {
+			return fmt.Errorf("parsing configuration: %v", err)
+		}
 	}
 	if config.APIKey == "" {
 		config.APIKey = os.Getenv("NVD_API_KEY")
 	}
 	if strict {
-		config.Strict = strict
+		config.Strict = true
+	}
+	if wait {
+		config.Wait = true
 	}
 	config.Verbose = verbose
 	config.Cache = cache

gobinsec/dependency.go

@@ -5,13 +5,19 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"time"
 )
 
 const (
-	URL             = "https://services.nvd.nist.gov/rest/json/cves/1.0/?keyword="
-	StatusCodeLimit = 300
+	URL                  = "https://services.nvd.nist.gov/rest/json/cves/1.0/?keyword="
+	StatusCodeLimit      = 300
+	RateMinuteWithoutKey = 10
+	RateMinuteWithKey    = 100
 )
 
+// timeLastCall is time for last NVD API call
+var timeLastCall time.Time
+
 // Dependency is a dependency with vulnerabilities
 type Dependency struct {
 	Name            string
@@ -37,6 +43,7 @@ func (d *Dependency) LoadVulnerabilities() error {
 		return err
 	}
 	if vulnerabilities == nil {
+		WaitBeforeCall()
 		url := URL + d.Name
 		if config.APIKey != "" {
 			url += "&apiKey=" + config.APIKey
@@ -75,6 +82,22 @@ func (d *Dependency) LoadVulnerabilities() error {
 	return nil
 }
 
+// WaitBeforeCall waits in order not to exceed NVD call rate limit
+func WaitBeforeCall() {
+	if config.Wait {
+		elapsedSinceLastCall := time.Since(timeLastCall)
+		timeToSleep := (60000 / RateMinuteWithoutKey) * time.Millisecond
+		if config.APIKey != "" {
+			timeToSleep = (60000 / RateMinuteWithKey) * time.Millisecond
+		}
+		timeToWait := timeToSleep - elapsedSinceLastCall
+		if timeToWait > 0 {
+			time.Sleep(timeToWait)
+		}
+		timeLastCall = time.Now()
+	}
+}
+
 // Key returns a key as a string for caching
 func (d *Dependency) Key() string {
 	return d.Name

main.go

@@ -19,6 +19,7 @@ func main() {
 	version := flag.Bool("version", false, "Print gobinsec version")
 	verbose := flag.Bool("verbose", false, "Print additional information in terminal")
 	cache := flag.Bool("cache", false, "Print cache information in terminal")
+	wait := flag.Bool("wait", false, "Wait between NVD API calls")
 	strict := flag.Bool("strict", false, "Vulnerabilities without version are exposed")
 	config := flag.String("config", "", "Configuration file")
 	flag.Parse()
@@ -30,7 +31,7 @@ func main() {
 		println("ERROR you must pass binary/ies to analyze on command line")
 		os.Exit(CodeError)
 	}
-	if err := gobinsec.LoadConfig(*config, *strict, *verbose, *cache); err != nil {
+	if err := gobinsec.LoadConfig(*config, *strict, *wait, *verbose, *cache); err != nil {
 		println(fmt.Sprintf("ERROR %v", err))
 		os.Exit(CodeError)
 	}