Database populated with NVD data

kopolindo · Mar 25, 2018 · d14d000 · d14d000
1 parent a2e9b40
commit d14d000
Show file tree

Hide file tree

Showing 9 changed files with 3,259 additions and 85 deletions.
diff --git a/cwe/1000.csv b/cwe/1000.csv
diff --git a/cwe/1008.csv b/cwe/1008.csv
diff --git a/cwe/699.csv b/cwe/699.csv
diff --git a/cwe/CWE.csv b/cwe/CWE.csv
diff --git a/cwe/CWE.csv.bak b/cwe/CWE.csv.bak
diff --git a/cwe/cwe.go b/cwe/cwe.go
@@ -0,0 +1,36 @@
+package cwe
+
+import (
+	"encoding/csv"
+	"io"
+	"os"
+)
+
+var cweList = "cwe/CWE.csv"
+
+func Load() map[string]string {
+	//Initialize map m
+	m := make(map[string]string)
+	//Open file
+	file, err := os.Open(cweList)
+	if err != nil {
+		panic(err)
+	}
+	//Create new csv reader
+	reader := csv.NewReader(file)
+	reader.Comma = ','
+	//Read it and store in map m
+	for {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			panic(err)
+		}
+		cweID := record[0]
+		cweDescription := record[1]
+		m[cweID] = cweDescription
+		//fmt.Println(record[0], ",", record[1])
+	}
+	return m
+}
diff --git a/db/populate.go b/db/populate.go
@@ -1,37 +1,21 @@
 package db
 
 import (
-	"fmt"
 	"database/sql"
-	"os"
-	_"github.com/mattn/go-sqlite3"
-)
+	"fmt"
 
-var database = ".local/vuln_db"
+	_ "github.com/mattn/go-sqlite3"
+)
 
-func main(){
-	//Check file existence
-	if _, err := os.Stat(database); err != nil {
-		fmt.Println("non esiste")
-	} else {
-		fmt.Println("esiste")
-	}
-	//Open db file
-	cveDB,err_opening := sql.Open("sqlite3",database)
-	if err_opening != nil{
-		fmt.Println("ERRORE IN FASE DI APERTURA:")
-		panic(err_opening)
-	}
-	//Create table query
-	createTest,err_preparing := cveDB.Prepare("CREATE TABLE IF NOT EXISTS people(id INTEGER PRIMARY KEY, firstname TEXT, lastname TEXT, CONSTRAINT identity UNIQUE (firstname, lastname))")
-	if err_preparing != nil{
-		fmt.Println("ERRORE IN FASE DI PREPARAZIONE QUERY")
-		panic(err_preparing)
-	}
-   createTest.Exec()
+func Insert(cveDB *sql.DB, params []string) {
 	//Insert query
-	query,_ := cveDB.Prepare("INSERT OR IGNORE INTO people (firstname, lastname) VALUES (?, ?)")
-	query.Exec("alex","conti")
-	//Close db file
-	cveDB.Close()
+	fmt.Println("LEN_SLICE", len(params))
+	switch l := len(params); l {
+	case 3:
+		querySw, _ := cveDB.Prepare("INSERT OR IGNORE INTO sw (id,software,version) VALUES (?, ?, ?)")
+		querySw.Exec(params[0], params[1], params[2])
+	case 7:
+		queryCve, _ := cveDB.Prepare("INSERT OR IGNORE INTO cve (id,cve_description,cwe,cwe_description,refs,cvssv2,cvssv3) VALUES (?, ?, ?, ?, ?, ?, ?)")
+		queryCve.Exec(params[0], params[1], params[2], params[3], params[4], params[5], params[6])
+	}
 }
diff --git a/main.go b/main.go
@@ -1,28 +1,106 @@
 package main
 
 import (
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
+	"strings"
+
+	"./cwe"
+	"./db"
 	"./nvd"
-	//"./db"
+	_ "github.com/mattn/go-sqlite3"
 )
-var completeList = "./.sources/nvd/nvdcve-1.0-2002.json"
+
+var completeList = "./.sources/nvd/nvdcve-1.0-2017.json"
+var database = ".local/vuln_db"
+
 func main() {
+	//Check Database File existence
+	if _, err := os.Stat(database); err != nil {
+		fmt.Println("Database file doesn't exist")
+	}
+	//Open db file
+	cveDB, err_opening := sql.Open("sqlite3", database)
+	if err_opening != nil {
+		fmt.Println("ERRORE IN FASE DI APERTURA:")
+		panic(err_opening)
+	}
+	//Assign to cweList the complete list of weaknesses
+	cweList := cwe.Load()
 	//Declaration
 	var complete nvd.RootComplete
 
 	completeContent, _ := ioutil.ReadFile(completeList)
-	unm_err_c := json.Unmarshal(completeContent,&complete)
-	if unm_err_c != nil{
+	unm_err_c := json.Unmarshal(completeContent, &complete)
+	if unm_err_c != nil {
 		panic(unm_err_c)
 	}
 	cveArray := complete.CVEItems
 	//Testo tramite print :S (complete list)
-	for _,cve := range cveArray{
-		var ID = cve.CVE.CVEDataMeta.ID
-		var CvssV2Vector = cve.Impact.BaseV2.CVSSv2.VectorString
-		var CvssV3Vector = cve.Impact.BaseV3.CVSSv3.VectorString
-		fmt.Println(ID,"\n",CvssV2Vector,"\n",CvssV3Vector)
+	for _, cve := range cveArray {
+		//ID
+		ID := cve.CVE.CVEDataMeta.ID
+		//CVSSv2 Vector
+		CvssV2Vector := cve.Impact.BaseV2.CVSSv2.VectorString
+		//CVSSv3 Vector
+		CvssV3Vector := cve.Impact.BaseV3.CVSSv3.VectorString
+		//CVE Description
+		CveDescription := cve.CVE.Description.DescriptionData
+
+		//DESCRIPTION
+		tmpDescArray := []string{}
+		for _, cvedesc := range CveDescription {
+			tmpDescArray = append(tmpDescArray, cvedesc.Value)
+		}
+		cveDescArray := strings.Join(tmpDescArray, "|")
+
+		//REFERENCES
+		tmpRefArray := []string{}
+		References := cve.CVE.References.ReferenceData
+		for _, refs := range References {
+			tmpRefArray = append(tmpRefArray, refs.Url)
+		}
+		cveRefArray := strings.Join(tmpRefArray, "|")
+		//PROBLEMTYPE
+		tmpCWEArray := []string{}
+		tmpCWEDescArray := []string{}
+		problemType := cve.CVE.Problemtype.ProblemtypeData
+		for _, problem := range problemType {
+			for _, desc := range problem.Description {
+				tmpCWEArray = append(tmpCWEArray, desc.Value)
+				tmpCWEDescArray = append(tmpCWEDescArray, cweList[desc.Value])
+			}
+		}
+		cveCWEArray := strings.Join(tmpCWEArray, "|")
+		cveCWEDescArray := strings.Join(tmpCWEDescArray, "|")
+
+		/******************
+		*		SOFTWARE		*
+		******************/
+		var vendorData = cve.CVE.Affects.Vendor.VendorData
+		for _, software := range vendorData {
+			var product = software.Product.ProductData
+			//PER OGNI VENDOR PRENDO LA LISTA DEI SW
+			for _, sw := range product {
+				//fmt.Println("\tSoftware(", i, ": ", sw.ProductName)
+				//PER OGNI SW PRENDO LA LISTA DELLE VERSIONI
+				tmpVersionArray := []string{}
+				for _, ver := range sw.Version.VersionData {
+					//fmt.Println("\tVersion:", ver.VersionValue)
+					tmpVersionArray = append(tmpVersionArray, ver.VersionValue)
+				}
+				verArray := strings.Join(tmpVersionArray, "|")
+				//fmt.Println("SOFTWARE: ", sw.ProductName, "VERSIONS: ", verArray)
+				swInsertParams := [3]string{ID, sw.ProductName, verArray}
+				db.Insert(cveDB, swInsertParams[:])
+			}
+		}
+		//fmt.Println(ID, ",", cveDescArray, ",", cveCWEArray, ",", cveCWEDescArray, ",", cveRefArray, ",", CvssV2Vector, ",", CvssV3Vector)
+		cveInsertParams := [7]string{ID, cveDescArray, cveCWEArray, cveCWEDescArray, cveRefArray, CvssV2Vector, CvssV3Vector}
+		db.Insert(cveDB, cveInsertParams[:])
 	}
+	cveDB.Close()
 }
diff --git a/nvd/cve_4.0.min.go b/nvd/cve_4.0.min.go