Skip to content

Commit

Permalink
merge with FD-418
Browse files Browse the repository at this point in the history
  • Loading branch information
@PaulSnow
PaulSnow committed Mar 21, 2018
2 parents 493d18b + 5535908 commit a7e6f33
Show file tree
Hide file tree
Showing 21 changed files with 505 additions and 380 deletions.
46 changes: 39 additions & 7 deletions Utilities/DatabaseIntegrityCheck/DatabaseIntegrityCheck.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"os"

"github.com/FactomProject/factomd/common/adminBlock"
"github.com/FactomProject/factomd/common/constants"
"github.com/FactomProject/factomd/common/directoryBlock"
"github.com/FactomProject/factomd/common/entryCreditBlock"
"github.com/FactomProject/factomd/common/factoid"
Expand Down Expand Up @@ -53,6 +54,7 @@ func main() {
dbo := databaseOverlay.NewOverlay(dbase)
CheckDatabase(dbo)
//CheckMinuteNumbers(dbo)
fmt.Println("\n")
}

func CheckDatabase(dbo interfaces.DBOverlay) {
Expand Down Expand Up @@ -219,6 +221,9 @@ func CheckDatabase(dbo interfaces.DBOverlay) {
}
}

ecChains := 0
ecEntries := 0

ecBlocks, err := dbo.FetchAllECBlockKeys()
if err != nil {
panic(err)
Expand All @@ -230,27 +235,53 @@ func CheckDatabase(dbo interfaces.DBOverlay) {
if hashMap[block.String()] == "" {
fmt.Printf("Free-floating ECBlock - %v\n", block.String())
}
ecblk, err := dbo.FetchECBlock(block)
if err == nil {
for _, ebe := range ecblk.GetEntries() {
switch ebe.ECID() {
case constants.ECIDEntryCommit:
ecEntries++
eec := ebe.(*entryCreditBlock.CommitEntry)
if e, err := dbo.FetchEntry(eec.EntryHash); err != nil || e == nil {
fmt.Printf("\t **** Failed to find entry %x for the commit. dbht %d\n",
eec.EntryHash.Bytes(),
ecblk.GetHeader().GetDBHeight())
}
case constants.ECIDChainCommit:
ecChains++
default:

}
}
}
}

fmt.Printf("\tEntry Credit Block found chains: %v entries: %v total: %v \n",
ecChains,
ecEntries,
ecChains+ecEntries)

fmt.Printf("\tFinished looking for free-floating blocks\n")

fmt.Printf("\tLooking for missing EBlocks\n")

foundBlocks := 0
missingBlocks := 0
missingDBlocks := 0
for _, dHash := range dBlocks {
dBlock, err := dbo.FetchDBlock(dHash)
if err != nil {
panic(err)
missingDBlocks++
}
if dBlock == nil {
fmt.Printf("Could not find DBlock %v!", dHash.String())
panic("")
missingDBlocks++
}
eBlockEntries := dBlock.GetEBlockDBEntries()
for _, v := range eBlockEntries {
eBlock, err := dbo.FetchEBlock(v.GetKeyMR())
if err != nil {
panic(err)
missingBlocks++
}
if eBlock == nil {
fmt.Errorf("Could not find eBlock %v!\n", v.GetKeyMR())
Expand All @@ -260,7 +291,7 @@ func CheckDatabase(dbo interfaces.DBOverlay) {
}
}

fmt.Printf("\tFinished looking for missing EBlocks - found %v\n", foundBlocks)
fmt.Printf("\tFinished looking for missing EBlocks. Missing %d Found %v\n", missingBlocks, foundBlocks)

fmt.Printf("\tLooking for missing EBlock Entries\n")

Expand All @@ -270,6 +301,7 @@ func CheckDatabase(dbo interfaces.DBOverlay) {
}
checkCount := 0
missingCount := 0

for _, chain := range chains {
blocks, err := dbo.FetchAllEBlocksByChain(chain)
if err != nil {
Expand Down Expand Up @@ -309,9 +341,9 @@ func CheckDatabase(dbo interfaces.DBOverlay) {
}
}
}
fmt.Printf("Found %v entries, missing %v\n", checkCount, missingCount)
fmt.Printf("\tFound %v entries, missing %v\n", checkCount, missingCount)
fmt.Printf("\tFinished looking for missing EBlock Entries\n")

fmt.Printf("\tDifference between entries and commits: **** %d ****", ecEntries+ecChains-checkCount)
//CheckMinuteNumbers(dbo)
}

Expand All @@ -327,7 +359,7 @@ func CheckMinuteNumbers(dbo interfaces.DBOverlay) {
found := 0
lastNumber := 0
for _, e := range entries {
if e.ECID() == entryCreditBlock.ECIDMinuteNumber {
if e.ECID() == constants.ECIDMinuteNumber {
number := int(e.(*entryCreditBlock.MinuteNumber).Number)
if number != lastNumber+1 {
fmt.Printf("Block #%v %v, Minute Number %v is not last minute plus 1\n", v.GetDatabaseHeight(), v.GetHash().String(), number)
Expand Down
75 changes: 42 additions & 33 deletions common/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,39 +11,39 @@ import (

// Messages
const (
EOM_MSG = iota
ACK_MSG
FED_SERVER_FAULT_MSG
AUDIT_SERVER_FAULT_MSG
FULL_SERVER_FAULT_MSG
COMMIT_CHAIN_MSG
COMMIT_ENTRY_MSG
DIRECTORY_BLOCK_SIGNATURE_MSG
EOM_TIMEOUT_MSG
FACTOID_TRANSACTION_MSG
HEARTBEAT_MSG
INVALID_ACK_MSG
INVALID_DIRECTORY_BLOCK_MSG

REVEAL_ENTRY_MSG
REQUEST_BLOCK_MSG
SIGNATURE_TIMEOUT_MSG
MISSING_MSG
MISSING_DATA
DATA_RESPONSE
MISSING_MSG_RESPONSE

DBSTATE_MSG
DBSTATE_MISSING_MSG
ADDSERVER_MSG
CHANGESERVER_KEY_MSG
REMOVESERVER_MSG

BOUNCE_MSG // test message
BOUNCEREPLY_MSG // test message

MISSING_ENTRY_BLOCKS
ENTRY_BLOCK_RESPONSE
EOM_MSG byte = iota // 0
ACK_MSG // 1
FED_SERVER_FAULT_MSG // 2
AUDIT_SERVER_FAULT_MSG // 3
FULL_SERVER_FAULT_MSG // 4
COMMIT_CHAIN_MSG // 5
COMMIT_ENTRY_MSG // 6
DIRECTORY_BLOCK_SIGNATURE_MSG // 7
EOM_TIMEOUT_MSG // 8
FACTOID_TRANSACTION_MSG // 9
HEARTBEAT_MSG // 10
INVALID_ACK_MSG // 11
INVALID_DIRECTORY_BLOCK_MSG // 12

REVEAL_ENTRY_MSG // 13
REQUEST_BLOCK_MSG // 14
SIGNATURE_TIMEOUT_MSG // 15
MISSING_MSG // 16
MISSING_DATA // 17
DATA_RESPONSE // 18
MISSING_MSG_RESPONSE //19

DBSTATE_MSG // 20
DBSTATE_MISSING_MSG // 21
ADDSERVER_MSG // 22
CHANGESERVER_KEY_MSG // 23
REMOVESERVER_MSG // 24

BOUNCE_MSG // 25 test message
BOUNCEREPLY_MSG // 26 test message

MISSING_ENTRY_BLOCKS //27
ENTRY_BLOCK_RESPONSE //28

INTERNALADDLEADER
INTERNALREMOVELEADER
Expand All @@ -61,6 +61,15 @@ const (
NUM_MESSAGES // Not used, just a counter for the number of messages.
)

// Entry Credit Block entries
const (
ECIDServerIndexNumber byte = iota // 0 Must be these values, per the specification
ECIDMinuteNumber // 1
ECIDChainCommit // 2
ECIDEntryCommit // 3
ECIDBalanceIncrease // 4
)

func MessageName(Type byte) string {
switch Type {
case EOM_MSG:
Expand Down
5 changes: 3 additions & 2 deletions common/entryBlock/entry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@ func (e *Entry) GetHash() interfaces.IHash {
h := primitives.NewZeroHash()
entry, err := e.MarshalBinary()
if err != nil {
return h
fmt.Println("Failed to Marshal Entry", e.String())
return nil
}

h1 := sha512.Sum512(entry)
Expand Down Expand Up @@ -210,7 +211,7 @@ func (e *Entry) MarshalBinary() ([]byte, error) {
// Content
buf.Write(e.Content.Bytes)

return buf.DeepCopyBytes(), nil
return buf.Bytes(), nil
}

// MarshalExtIDsBinary marshals the ExtIDs into a []byte containing a series of
Expand Down
3 changes: 2 additions & 1 deletion common/entryCreditBlock/commitchain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"encoding/binary"
"fmt"

"github.com/FactomProject/factomd/common/constants"
"github.com/FactomProject/factomd/common/interfaces"
"github.com/FactomProject/factomd/common/primitives"
)
Expand Down Expand Up @@ -320,7 +321,7 @@ func (c *CommitChain) ValidateSignatures() error {
}

func (c *CommitChain) ECID() byte {
return ECIDChainCommit
return constants.ECIDChainCommit
}

func (c *CommitChain) UnmarshalBinaryData(data []byte) ([]byte, error) {
Expand Down
3 changes: 2 additions & 1 deletion common/entryCreditBlock/commitentry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"encoding/binary"
"fmt"

"github.com/FactomProject/factomd/common/constants"
"github.com/FactomProject/factomd/common/interfaces"
"github.com/FactomProject/factomd/common/primitives"
)
Expand Down Expand Up @@ -280,7 +281,7 @@ func (c *CommitEntry) ValidateSignatures() error {
}

func (c *CommitEntry) ECID() byte {
return ECIDEntryCommit
return constants.ECIDEntryCommit
}

func (c *CommitEntry) UnmarshalBinaryData(data []byte) ([]byte, error) {
Expand Down
27 changes: 12 additions & 15 deletions common/entryCreditBlock/ecblock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,11 @@ package entryCreditBlock
import (
"fmt"

"github.com/FactomProject/factomd/common/constants"
"github.com/FactomProject/factomd/common/interfaces"
"github.com/FactomProject/factomd/common/primitives"
)

const (
ECIDServerIndexNumber byte = iota
ECIDMinuteNumber
ECIDChainCommit
ECIDEntryCommit
ECIDBalanceIncrease
)

// The Entry Credit Block consists of a header and a body. The body is composed
// of primarily Commits and Balance Increases with Minute Markers and Server
// Markers distributed throughout.
Expand Down Expand Up @@ -102,7 +95,9 @@ func (c *ECBlock) GetEntryHashes() []interfaces.IHash {
entries := c.GetBody().GetEntries()
answer := make([]interfaces.IHash, 0, len(entries))
for _, entry := range entries {
if entry.ECID() == ECIDBalanceIncrease || entry.ECID() == ECIDChainCommit || entry.ECID() == ECIDEntryCommit {
if entry.ECID() == constants.ECIDBalanceIncrease ||
entry.ECID() == constants.ECIDChainCommit ||
entry.ECID() == constants.ECIDEntryCommit {
answer = append(answer, entry.Hash())
}
}
Expand All @@ -113,7 +108,9 @@ func (c *ECBlock) GetEntrySigHashes() []interfaces.IHash {
entries := c.GetBody().GetEntries()
answer := make([]interfaces.IHash, 0, len(entries))
for _, entry := range entries {
if entry.ECID() == ECIDBalanceIncrease || entry.ECID() == ECIDChainCommit || entry.ECID() == ECIDEntryCommit {
if entry.ECID() == constants.ECIDBalanceIncrease ||
entry.ECID() == constants.ECIDChainCommit ||
entry.ECID() == constants.ECIDEntryCommit {
sHash := entry.GetSigHash()
if sHash != nil {
answer = append(answer, sHash)
Expand Down Expand Up @@ -302,38 +299,38 @@ func (e *ECBlock) unmarshalBodyBinaryData(data []byte) ([]byte, error) {
newData = newData[1:]

switch id {
case ECIDServerIndexNumber:
case constants.ECIDServerIndexNumber:
s := NewServerIndexNumber()
newData, err = s.UnmarshalBinaryData(newData)
if err != nil {
return nil, err
}
allentries[i] = s
case ECIDMinuteNumber:
case constants.ECIDMinuteNumber:
m := NewMinuteNumber(0)
_, err = m.UnmarshalBinaryData(newData[:1])
if err != nil {
return nil, err
}
allentries[i] = m
newData = newData[1:]
case ECIDChainCommit:
case constants.ECIDChainCommit:
c := NewCommitChain()
_, err = c.UnmarshalBinaryData(newData[:200])
if err != nil {
return nil, err
}
allentries[i] = c
newData = newData[200:]
case ECIDEntryCommit:
case constants.ECIDEntryCommit:
c := NewCommitEntry()
_, err = c.UnmarshalBinaryData(newData[:136])
if err != nil {
return nil, err
}
allentries[i] = c
newData = newData[136:]
case ECIDBalanceIncrease:
case constants.ECIDBalanceIncrease:
c := NewIncreaseBalance()
newData, err = c.UnmarshalBinaryData(newData)
if err != nil {
Expand Down
3 changes: 2 additions & 1 deletion common/entryCreditBlock/increasebalance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package entryCreditBlock
import (
"fmt"

"github.com/FactomProject/factomd/common/constants"
"github.com/FactomProject/factomd/common/interfaces"
"github.com/FactomProject/factomd/common/primitives"
)
Expand Down Expand Up @@ -104,7 +105,7 @@ func (e *IncreaseBalance) GetSigHash() interfaces.IHash {
}

func (b *IncreaseBalance) ECID() byte {
return ECIDBalanceIncrease
return constants.ECIDBalanceIncrease
}

func (b *IncreaseBalance) IsInterpretable() bool {
Expand Down
3 changes: 2 additions & 1 deletion common/entryCreditBlock/minutenumber.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package entryCreditBlock
import (
"fmt"

"github.com/FactomProject/factomd/common/constants"
"github.com/FactomProject/factomd/common/interfaces"
"github.com/FactomProject/factomd/common/primitives"
)
Expand Down Expand Up @@ -81,7 +82,7 @@ func NewMinuteNumber(number uint8) *MinuteNumber {
}

func (m *MinuteNumber) ECID() byte {
return ECIDMinuteNumber
return constants.ECIDMinuteNumber
}

func (m *MinuteNumber) MarshalBinary() ([]byte, error) {
Expand Down
3 changes: 2 additions & 1 deletion common/entryCreditBlock/serverindexnumber.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package entryCreditBlock
import (
"fmt"

"github.com/FactomProject/factomd/common/constants"
"github.com/FactomProject/factomd/common/interfaces"
"github.com/FactomProject/factomd/common/primitives"
)
Expand Down Expand Up @@ -92,7 +93,7 @@ func NewServerIndexNumber2(number uint8) *ServerIndexNumber {
}

func (s *ServerIndexNumber) ECID() byte {
return ECIDServerIndexNumber
return constants.ECIDServerIndexNumber
}

func (s *ServerIndexNumber) MarshalBinary() ([]byte, error) {
Expand Down
Loading

0 comments on commit a7e6f33

Please sign in to comment.