test population

kopolindo · Mar 27, 2018 · aa5ba39 · aa5ba39
1 parent eb1f032
commit aa5ba39
Show file tree

Hide file tree

Showing 10 changed files with 386,497 additions and 103 deletions.
diff --git a/README.md b/README.md
@@ -24,3 +24,6 @@ Creo un file cveDB.go, per la gestione del db (sqlite) in modo da creare, riempi
 
 su rapid7 c'è il modulo msf che viene citato ma non esplicitato su nessus
 rapid7 può essere un'altra fonte di CVE
+
+╰─$ <sw.list § grep -Ev "`echo $line | cut -d';' -f1`" <(cat mitre_cve_to_insert.txt | cut -f5- -d' ' | sort -u) ° | wc -l
+
diff --git a/db/populate.go b/db/populate.go
@@ -2,13 +2,47 @@ package db
 
 import (
 	"database/sql"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
 
+	"github.com/kopolindo/cve-scraper/cwe"
+	"github.com/kopolindo/cve-scraper/mitre"
+	"github.com/kopolindo/cve-scraper/nvd"
 	_ "github.com/mattn/go-sqlite3"
 )
 
+var completeListNVD = "./.sources/nvd/nvdcve-1.0-YEAR.json"
+var completeListMITRE = "./.sources/mitre/allitems.xml"
+var database = ".local/vuln_db"
+
+func UnmarshalMITRE() {
+	_panic := func(err error) {
+		if err != nil {
+			panic(err)
+		}
+	}
+	if _, err := os.Stat(completeListMITRE); err != nil {
+		fmt.Println("Database file doesn't exist")
+	}
+	f, err := os.Open(completeListMITRE)
+	_panic(err)
+	d := xml.NewDecoder(f)
+	var cveMitre mitre.Cve
+	err = d.Decode(&cveMitre)
+	_panic(err)
+	for _, cve := range cveMitre.Item {
+		fmt.Println(cve.Name, " DESCRIPTION: ", cve.Desc)
+		//fmt.Println("REFERENCES :", cve.Refs)
+	}
+}
+
 func Insert(cveDB *sql.DB, params []string) {
 	//Insert query
-	//fmt.Println("LEN_SLICE", len(params))
 	switch l := len(params); l {
 	case 3:
 		querySw, _ := cveDB.Prepare("INSERT OR IGNORE INTO sw (id,software,version) VALUES (?, ?, ?)")
@@ -18,3 +52,91 @@ func Insert(cveDB *sql.DB, params []string) {
 		queryCve.Exec(params[0], params[1], params[2], params[3], params[4], params[5], params[6])
 	}
 }
+
+func Populate() {
+	//Check Database File existence
+	if _, err := os.Stat(database); err != nil {
+		fmt.Println("Database file doesn't exist")
+	}
+	//Open db file
+	cveDB, err_opening := sql.Open("sqlite3", database)
+	if err_opening != nil {
+		fmt.Println("ERRORE IN FASE DI APERTURA:")
+		panic(err_opening)
+	}
+	//Assign to cweList the complete list of weaknesses
+	cweList := cwe.Load()
+	//Declaration
+	for i := 2002; i < 2019; i++ {
+		var complete nvd.RootComplete
+		json_ := strings.Replace(completeListNVD, "YEAR", strconv.Itoa(i), -1)
+		fmt.Println("PROCESSING ", json_)
+		completeContent, _ := ioutil.ReadFile(json_)
+		unm_err_c := json.Unmarshal(completeContent, &complete)
+		if unm_err_c != nil {
+			panic(unm_err_c)
+		}
+		cveArray := complete.CVEItems
+		//Testo tramite print :S (complete list)
+		for _, cve := range cveArray {
+			//ID
+			ID := cve.CVE.CVEDataMeta.ID
+			//CVSSv2 Vector
+			CvssV2Vector := cve.Impact.BaseV2.CVSSv2.VectorString
+			//CVSSv3 Vector
+			CvssV3Vector := cve.Impact.BaseV3.CVSSv3.VectorString
+			//CVE Description
+			CveDescription := cve.CVE.Description.DescriptionData
+
+			//DESCRIPTION
+			tmpDescArray := []string{}
+			for _, cvedesc := range CveDescription {
+				tmpDescArray = append(tmpDescArray, cvedesc.Value)
+			}
+			cveDescArray := strings.Join(tmpDescArray, "|")
+
+			//REFERENCES
+			tmpRefArray := []string{}
+			References := cve.CVE.References.ReferenceData
+			for _, refs := range References {
+				tmpRefArray = append(tmpRefArray, refs.Url)
+			}
+			cveRefArray := strings.Join(tmpRefArray, "|")
+			//PROBLEMTYPE
+			tmpCWEArray := []string{}
+			tmpCWEDescArray := []string{}
+			problemType := cve.CVE.Problemtype.ProblemtypeData
+			for _, problem := range problemType {
+				for _, desc := range problem.Description {
+					tmpCWEArray = append(tmpCWEArray, desc.Value)
+					tmpCWEDescArray = append(tmpCWEDescArray, cweList[desc.Value])
+				}
+			}
+			cveCWEArray := strings.Join(tmpCWEArray, "|")
+			cveCWEDescArray := strings.Join(tmpCWEDescArray, "|")
+
+			/******************
+			*		SOFTWARE		*
+			******************/
+			var vendorData = cve.CVE.Affects.Vendor.VendorData
+			for _, software := range vendorData {
+				var product = software.Product.ProductData
+				//PER OGNI VENDOR PRENDO LA LISTA DEI SW
+				for _, sw := range product {
+					//PER OGNI SW PRENDO LA LISTA DELLE VERSIONI
+					tmpVersionArray := []string{}
+					for _, ver := range sw.Version.VersionData {
+						tmpVersionArray = append(tmpVersionArray, ver.VersionValue)
+					}
+					verArray := strings.Join(tmpVersionArray, "|")
+					swInsertParams := [3]string{ID, sw.ProductName, verArray}
+					Insert(cveDB, swInsertParams[:])
+				}
+			}
+			cveInsertParams := [7]string{ID, cveDescArray, cveCWEArray, cveCWEDescArray, cveRefArray, CvssV2Vector, CvssV3Vector}
+			Insert(cveDB, cveInsertParams[:])
+		}
+		fmt.Println("END PROCESSING ", json_)
+	}
+	_ = cveDB.Close()
+}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -24,3 +24,6 @@ Creo un file cveDB.go, per la gestione del db (sqlite) in modo da creare, riempi

		su rapid7 c'è il modulo msf che viene citato ma non esplicitato su nessus
		rapid7 può essere un'altra fonte di CVE

		╰─$ <sw.list § grep -Ev "`echo $line \| cut -d';' -f1`" <(cat mitre_cve_to_insert.txt \| cut -f5- -d' ' \| sort -u) ° \| wc -l