breaking-change(rdb): update gorm v2 (#47)

* feat(rdb): update gorm v2 * feat(rdb): change models * fix(fetch): delete old records with only exploit type to fetch * feat(fetch): use batch insert * feat(fetch): add FetchMeta * feat(integration): add diff test template * chore(integration): change git stash in build-integration * fix(integration): fix makefile * fix(integration): git stash * fix(integration): fix fetched db path * feat(integration): change json output for diff test * feat(db): reproduced output of GORMv1 * chore(search): eval struct * chore(rdb): fix typo * feat(fetch): eval len old record slices * chore: fix typo * fix: args by environment variable * chore: fix path * chore: fix typo * chore: fix error * chore: fix typo * chore: remove comment * style: format query indent * chore: remove GitHubJSON * chore: viper, cobra update * refactor(fetcher): simplify FetchExploitDB * fix(fetcher): fix insert data * Revert "fix(fetcher): fix insert data" This reverts commit 60cf5bb. * fix(fetcher): fix insert data * fix(githubrepos): fill in GitHubRepository.ExploitUniqueID * feat(redis): set to nil when ExploitUniqueID is not filled * fix(fetcher/exploitdb): fix insert data * fix: fix panic error * chore: fix message * feat(redis): init exploits slice * chore: run test command with default * feat(models): do not show ExploitUniqueID multiple times * feat(models): add gorm struct tag * chore: fix typo * chore(integration): add sample_rate * chore: change log * feat(rdb): add batch size option * feat(cmd): add version command
vulsio · Aug 19, 2021 · ad3f6bf · ad3f6bf
1 parent 3893f58
commit ad3f6bf
Show file tree

Hide file tree

Showing 38 changed files with 66,035 additions and 468 deletions.
diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.md b/.github/ISSUE_TEMPLATE/BUG_REPORT.md
@@ -25,19 +25,17 @@ Please re-run the command using ```-debug``` and provide the output below.
 
 * Go environment (`go env`):
 
-* Vuls environment:
+* go-exploitdb environment:
 
 Hash : ____
 
 To check the commit hash of HEAD
-$ vuls -v
+$ go-exploitdb version
 
 or
 
-$ cd $GOPATH/src/github.com/future-architect/vuls 
+$ cd $GOPATH/src/github.com/vulsio/go-exploitdb 
 $ git rev-parse --short HEAD 
 
-* config.toml:
-
 * command:
 
diff --git a/Dockerfile b/Dockerfile
@@ -6,12 +6,12 @@ ENV REPOSITORY github.com/prince-chrismc/go-exploitdb
 COPY . $GOPATH/src/$REPOSITORY
 RUN cd $GOPATH/src/$REPOSITORY && make install
 
-FROM alpine:3.11
+FROM alpine:3.14
 
-MAINTAINER princechrismc
+LABEL maintainer princechrismc
 
-ENV LOGDIR /var/log/vuls
-ENV WORKDIR /vuls
+ENV LOGDIR /var/log/go-exploitdb
+ENV WORKDIR /go-exploitdb
 
 RUN apk add --no-cache ca-certificates \
     && mkdir -p $WORKDIR $LOGDIR

diff --git a/GNUmakefile b/GNUmakefile
@@ -11,12 +11,23 @@
 	test \
 	integration \
 	cov \
-	clean
+	clean \
+	build-integration \
+	clean-integration \
+	fetch-rdb \
+	fetch-redis \
+	diff-cveid \
+	diff-package \
+	diff-server-rdb \
+	diff-server-redis \
+	diff-server-rdb-redis
 
 SRCS = $(shell git ls-files '*.go')
 PKGS =  ./commands ./config ./db ./fetcher ./models ./util ./server
 VERSION := $(shell git describe --tags --abbrev=0)
-LDFLAGS := -X 'main.version=$(VERSION)' \
+REVISION := $(shell git rev-parse --short HEAD)
+LDFLAGS := -X 'github.com/vulsio/go-exploitdb/config.Version=$(VERSION)' \
+	-X 'github.com/vulsio/go-exploitdb/config.Revision=$(REVISION)'
 
 all: build test
 
@@ -57,3 +68,74 @@ cov:
 clean:
 	$(foreach pkg,$(PKGS),go clean $(pkg) || exit;)
 
+PWD := $(shell pwd)
+BRANCH := $(shell git symbolic-ref --short HEAD)
+build-integration:
+	@ git stash save "build-integration"
+	go build -ldflags "$(LDFLAGS)" -o integration/exploitdb.new
+	git checkout $(shell git describe --tags --abbrev=0)
+	@git reset --hard
+	go build -ldflags "$(LDFLAGS)" -o integration/exploitdb.old
+	git checkout $(BRANCH)
+	@ git stash apply stash@{0} && git stash drop stash@{0}
+
+clean-integration:
+	-pkill exploitdb.old
+	-pkill exploitdb.new
+	-rm integration/exploitdb.old integration/exploitdb.new integration/go-exploitdb.old.sqlite3 integration/go-exploitdb.new.sqlite3
+	-docker kill redis-old redis-new
+	-docker rm redis-old redis-new
+
+fetch-rdb:
+	integration/exploitdb.old fetch awesomepoc --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
+	integration/exploitdb.old fetch exploitdb --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
+	integration/exploitdb.old fetch githubrepos --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3
+
+	integration/exploitdb.new fetch awesomepoc --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
+	integration/exploitdb.new fetch exploitdb --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
+	integration/exploitdb.new fetch githubrepos --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3
+
+fetch-redis:
+	docker run --name redis-old -d -p 127.0.0.1:6379:6379 redis
+	docker run --name redis-new -d -p 127.0.0.1:6380:6379 redis
+
+	integration/exploitdb.old fetch awesomepoc --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
+	integration/exploitdb.old fetch exploitdb --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
+	integration/exploitdb.old fetch githubrepos --dbtype redis --dbpath "redis://127.0.0.1:6379/0"
+
+	integration/exploitdb.new fetch awesomepoc --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
+	integration/exploitdb.new fetch exploitdb --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
+	integration/exploitdb.new fetch githubrepos --dbtype redis --dbpath "redis://127.0.0.1:6380/0"
+
+diff-cveid:
+	@ python integration/diff_server_mode.py cveid --sample_rate 0.01 awesomepoc
+	@ python integration/diff_server_mode.py cveid --sample_rate 0.01 exploitdb
+	@ python integration/diff_server_mode.py cveid --sample_rate 0.01 githubrepos
+
+diff-uniqueid:
+	@ python integration/diff_server_mode.py uniqueid --sample_rate 0.01 awesomepoc
+	@ python integration/diff_server_mode.py uniqueid --sample_rate 0.01 exploitdb
+	@ python integration/diff_server_mode.py uniqueid --sample_rate 0.01 githubrepos
+
+diff-server-rdb:
+	integration/exploitdb.old server --dbpath=$(PWD)/integration/go-exploitdb.old.sqlite3 --port 1325 > /dev/null 2>&1 & 
+	integration/exploitdb.new server --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3 --port 1326 > /dev/null 2>&1 &
+	make diff-cveid
+	make diff-uniqueid
+	pkill exploitdb.old
+	pkill exploitdb.new
+
+diff-server-redis:
+	integration/exploitdb.old server --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --port 1325 > /dev/null 2>&1 & 
+	integration/exploitdb.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null 2>&1 &
+	make diff-cveid
+	make diff-uniqueid
+	pkill exploitdb.old
+	pkill exploitdb.new
+
+diff-server-rdb-redis:
+	integration/exploitdb.new server --dbpath=$(PWD)/integration/go-exploitdb.new.sqlite3 --port 1325 > /dev/null 2>&1 &
+	integration/exploitdb.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null 2>&1 &
+	make diff-cveid
+	make diff-uniqueid
+	pkill exploitdb.new
diff --git a/README.md b/README.md
@@ -56,8 +56,9 @@ Available Commands:
   githubrepos Fetch the data of github repos
 
 Flags:
-      --expire uint   timeout to set for Redis keys in seconds. If set to 0, the key is persistent.
-  -h, --help          help for fetch
+      --batch-size int   The number of batch size to insert. NOTE: This Option does not work for dbtype: redis. (default 500)
+      --expire uint      timeout to set for Redis keys in seconds. If set to 0, the key is persistent.
+  -h, --help             help for fetch
 
 Global Flags:
       --config string       config file (default is $HOME/.go-exploitdb.yaml)
@@ -68,7 +69,7 @@ Global Flags:
       --http-proxy string   http://proxy-url:port (default: empty)
       --log-dir string      /path/to/log
       --log-json            output log as JSON
-      --quiet               quiet mode (no output)
+      --log-to-file         output log to file
 
 Use "go-exploitdb fetch [command] --help" for more information about a command.
 ```
@@ -101,7 +102,7 @@ Global Flags:
       --http-proxy string   http://proxy-url:port (default: empty)
       --log-dir string      /path/to/log
       --log-json            output log as JSON
-      --quiet               quiet mode (no output)
+      --log-to-file         output log to file
 ```
 
 ###### Search Exploits by CVE(ex. CVE-2009-4091)
@@ -199,7 +200,7 @@ Global Flags:
       --http-proxy string   http://proxy-url:port (default: empty)
       --log-dir string      /path/to/log
       --log-json            output log as JSON
-      --quiet               quiet mode (no output)
+      --log-to-file         output log to file
 ```
 
 ###### Starting Server

diff --git a/commands/fetch-awesomepoc.go b/commands/fetch-awesomepoc.go
@@ -7,6 +7,8 @@ import (
 	"github.com/vulsio/go-exploitdb/db"
 	"github.com/vulsio/go-exploitdb/fetcher"
 	"github.com/vulsio/go-exploitdb/models"
+	"github.com/vulsio/go-exploitdb/util"
+	"golang.org/x/xerrors"
 )
 
 var fetchAwesomePocCmd = &cobra.Command{
@@ -21,6 +23,10 @@ func init() {
 }
 
 func fetchAwesomePoc(cmd *cobra.Command, args []string) (err error) {
+	if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
 	driver, locked, err := db.NewDB(
 		viper.GetString("dbtype"),
 		viper.GetString("dbpath"),
@@ -33,8 +39,18 @@ func fetchAwesomePoc(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
+	fetchMeta, err := driver.GetFetchMeta()
+	if err != nil {
+		log15.Error("Failed to get FetchMeta from DB.", "err", err)
+		return err
+	}
+	if fetchMeta.OutDated() {
+		log15.Error("Failed to Insert CVEs into DB. SchemaVersion is old", "SchemaVersion", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
+		return xerrors.New("Failed to Insert CVEs into DB. SchemaVersion is old")
+	}
+
 	log15.Info("Fetching Awesome Poc Exploit")
-	var exploits []*models.Exploit
+	var exploits []models.Exploit
 	if exploits, err = fetcher.FetchAwesomePoc(); err != nil {
 		log15.Error("Failed to fetch AwesomePoc Exploit", "err", err)
 		return err
@@ -46,5 +62,11 @@ func fetchAwesomePoc(cmd *cobra.Command, args []string) (err error) {
 		log15.Error("Failed to insert.", "dbpath", viper.GetString("dbpath"), "err", err)
 		return err
 	}
+
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		log15.Error("Failed to upsert FetchMeta to DB.", "dbpath", viper.GetString("dbpath"), "err", err)
+		return err
+	}
+
 	return nil
 }
diff --git a/commands/fetch-exploitdb.go b/commands/fetch-exploitdb.go
@@ -7,6 +7,8 @@ import (
 	"github.com/vulsio/go-exploitdb/db"
 	"github.com/vulsio/go-exploitdb/fetcher"
 	"github.com/vulsio/go-exploitdb/models"
+	"github.com/vulsio/go-exploitdb/util"
+	"golang.org/x/xerrors"
 )
 
 var fetchExploitDBCmd = &cobra.Command{
@@ -21,6 +23,10 @@ func init() {
 }
 
 func fetchExploitDB(cmd *cobra.Command, args []string) (err error) {
+	if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
 	driver, locked, err := db.NewDB(
 		viper.GetString("dbtype"),
 		viper.GetString("dbpath"),
@@ -33,8 +39,18 @@ func fetchExploitDB(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
+	fetchMeta, err := driver.GetFetchMeta()
+	if err != nil {
+		log15.Error("Failed to get FetchMeta from DB.", "err", err)
+		return err
+	}
+	if fetchMeta.OutDated() {
+		log15.Error("Failed to Insert CVEs into DB. SchemaVersion is old", "SchemaVersion", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
+		return xerrors.New("Failed to Insert CVEs into DB. SchemaVersion is old")
+	}
+
 	log15.Info("Fetching Offensive Security Exploit")
-	var exploits []*models.Exploit
+	var exploits []models.Exploit
 	if exploits, err = fetcher.FetchExploitDB(); err != nil {
 		log15.Error("Failed to fetch Exploit", "err", err)
 		return err
@@ -46,5 +62,11 @@ func fetchExploitDB(cmd *cobra.Command, args []string) (err error) {
 		log15.Error("Failed to insert.", "dbpath", viper.GetString("dbpath"), "err", err)
 		return err
 	}
+
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		log15.Error("Failed to upsert FetchMeta to DB.", "dbpath", viper.GetString("dbpath"), "err", err)
+		return err
+	}
+
 	return nil
 }
diff --git a/commands/fetch-githubrepos.go b/commands/fetch-githubrepos.go
@@ -7,6 +7,8 @@ import (
 	"github.com/vulsio/go-exploitdb/db"
 	"github.com/vulsio/go-exploitdb/fetcher"
 	"github.com/vulsio/go-exploitdb/models"
+	"github.com/vulsio/go-exploitdb/util"
+	"golang.org/x/xerrors"
 )
 
 var fetchGitHubReposCmd = &cobra.Command{
@@ -27,6 +29,10 @@ func init() {
 }
 
 func fetchGitHubRepos(cmd *cobra.Command, args []string) (err error) {
+	if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil {
+		return xerrors.Errorf("Failed to SetLogger. err: %w", err)
+	}
+
 	driver, locked, err := db.NewDB(
 		viper.GetString("dbtype"),
 		viper.GetString("dbpath"),
@@ -39,8 +45,18 @@ func fetchGitHubRepos(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
+	fetchMeta, err := driver.GetFetchMeta()
+	if err != nil {
+		log15.Error("Failed to get FetchMeta from DB.", "err", err)
+		return err
+	}
+	if fetchMeta.OutDated() {
+		log15.Error("Failed to Insert CVEs into DB. SchemaVersion is old", "SchemaVersion", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion})
+		return xerrors.New("Failed to Insert CVEs into DB. SchemaVersion is old")
+	}
+
 	log15.Info("Fetching GitHub Repos Exploit")
-	var exploits []*models.Exploit
+	var exploits []models.Exploit
 	if exploits, err = fetcher.FetchGitHubRepos(viper.GetInt("threshold-stars"), viper.GetInt("threshold-forks")); err != nil {
 		log15.Error("Failed to fetch GitHubRepo Exploit", "err", err)
 		return err
@@ -52,5 +68,11 @@ func fetchGitHubRepos(cmd *cobra.Command, args []string) (err error) {
 		log15.Error("Failed to insert.", "dbpath", viper.GetString("dbpath"), "err", err)
 		return err
 	}
+
+	if err := driver.UpsertFetchMeta(fetchMeta); err != nil {
+		log15.Error("Failed to upsert FetchMeta to DB.", "dbpath", viper.GetString("dbpath"), "err", err)
+		return err
+	}
+
 	return nil
 }
diff --git a/commands/fetch.go b/commands/fetch.go
@@ -17,4 +17,7 @@ func init() {
 
 	fetchCmd.PersistentFlags().Uint("expire", 0, "timeout to set for Redis keys in seconds. If set to 0, the key is persistent.")
 	_ = viper.BindPFlag("expire", fetchCmd.PersistentFlags().Lookup("expire"))
+
+	fetchCmd.PersistentFlags().Int("batch-size", 500, "The number of batch size to insert. NOTE: This Option does not work for dbtype: redis.")
+	_ = viper.BindPFlag("batch-size", fetchCmd.PersistentFlags().Lookup("batch-size"))
 }
diff --git a/commands/root.go b/commands/root.go
@@ -28,6 +28,9 @@ func init() {
 
 	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.go-exploitdb.yaml)")
 
+	RootCmd.PersistentFlags().Bool("log-to-file", false, "output log to file")
+	_ = viper.BindPFlag("log-to-file", RootCmd.PersistentFlags().Lookup("log-to-file"))
+
 	RootCmd.PersistentFlags().String("log-dir", "", "/path/to/log")
 	_ = viper.BindPFlag("log-dir", RootCmd.PersistentFlags().Lookup("log-dir"))
 	viper.SetDefault("log-dir", util.GetDefaultLogDir())
@@ -36,10 +39,6 @@ func init() {
 	_ = viper.BindPFlag("log-json", RootCmd.PersistentFlags().Lookup("log-json"))
 	viper.SetDefault("log-json", false)
 
-	RootCmd.PersistentFlags().Bool("quiet", false, "quiet mode (no output)")
-	_ = viper.BindPFlag("quiet", RootCmd.PersistentFlags().Lookup("quiet"))
-	viper.SetDefault("quiet", false)
-
 	RootCmd.PersistentFlags().Bool("debug", false, "debug mode (default: false)")
 	_ = viper.BindPFlag("debug", RootCmd.PersistentFlags().Lookup("debug"))
 	viper.SetDefault("debug", false)
@@ -85,9 +84,4 @@ func initConfig() {
 	if err := viper.ReadInConfig(); err == nil {
 		fmt.Println("Using config file:", viper.ConfigFileUsed())
 	}
-	logDir := viper.GetString("log-dir")
-	quiet := viper.GetBool("quiet")
-	debug := viper.GetBool("debug")
-	logJSON := viper.GetBool("log-json")
-	util.SetLogger(logDir, quiet, debug, logJSON)
 }