Check file size on FileStreamFactory to files small than PAGE_SIZE

modanbinghan · Dec 23, 2022 · 1d98f39 · 1d98f39
1 parent caf0503
commit 1d98f39
Show file tree

Hide file tree

Showing 9 changed files with 37 additions and 17 deletions.
diff --git a/ConsoleApp1/Program.cs b/ConsoleApp1/Program.cs
@@ -34,7 +34,11 @@
     }
 
     // do checkpoint
-    db.Checkpoint();
+    c.Insert(new BsonDocument { ["Name"] = "test2" });
+
+    var doc2 = c.FindAll().Last();
+    Console.WriteLine(JsonSerializer.Serialize(doc2));
+
 }
 
 Console.WriteLine("Done");

diff --git a/LiteDB/Engine/Disk/DiskService.cs b/LiteDB/Engine/Disk/DiskService.cs
@@ -62,7 +62,7 @@ public DiskService(EngineSettings settings, int[] memorySegmentSizes)
             // get initial data file length
             _dataLength = _dataFactory.GetLength() - PAGE_SIZE;
 
-            // get initial log file length
+            // get initial log file length (should be 1 page before)
             if (_logFactory.Exists())
             {
                 _logLength = _logFactory.GetLength() - PAGE_SIZE;
@@ -192,9 +192,10 @@ public int WriteAsync(IEnumerable<PageBuffer> pages)
         }
 
         /// <summary>
-        /// Get virtual file length: real file can be small but async thread can still writing on disk
+        /// Get virtual file length: real file can be small because async thread can still writing on disk
+        /// and incrementing file size (Log file)
         /// </summary>
-        public long GetLength(FileOrigin origin)
+        public long GetVirtualLength(FileOrigin origin)
         {
             if (origin == FileOrigin.Log)
             {
@@ -251,7 +252,7 @@ public IEnumerable<PageBuffer> ReadFull(FileOrigin origin)
             try
             {
                 // get length before starts (avoid grow during loop)
-                var length = this.GetLength(origin);
+                var length = this.GetVirtualLength(origin);
 
                 stream.Position = 0;
 

diff --git a/LiteDB/Engine/Disk/StreamFactory/FileStreamFactory.cs b/LiteDB/Engine/Disk/StreamFactory/FileStreamFactory.cs
@@ -60,8 +60,19 @@ public Stream GetStream(bool canWrite, bool sequencial)
         /// </summary>
         public long GetLength()
         {
-            // getting size from OS - if encrypted must remove salt first page
-            return new FileInfo(_filename).Length - (_password == null ? 0 : PAGE_SIZE);
+            // if not file do not exists, returns 0
+            if (!this.Exists()) return 0;
+
+            // get physical file length from OS
+            var length = new FileInfo(_filename).Length;
+
+            // if length < PAGE_SIZE, ignore file length (should be 0)
+            if (length < PAGE_SIZE) return 0;
+
+            ENSURE(length % PAGE_SIZE == 0, $"file length must be PAGE_SIZE module. length={length}, file={Path.GetFileName(_filename)}");
+
+            // if encrypted must remove salt first page
+            return length - (_password == null ? 0 : PAGE_SIZE);
         }
 
         /// <summary>

diff --git a/LiteDB/Engine/Disk/Streams/AesStream.cs b/LiteDB/Engine/Disk/Streams/AesStream.cs
@@ -20,7 +20,8 @@ public class AesStream : Stream
         private readonly CryptoStream _reader;
         private readonly CryptoStream _writer;
 
-        private static readonly byte[] _decryptedZeroes = new byte[16];
+        private readonly byte[] _decryptedZeroes = new byte[16];
+
         private static readonly byte[] _emptyContent = new byte[PAGE_SIZE - 1 - 16]; // 1 for aes indicator + 16 for salt 
 
         public byte[] Salt { get; }
@@ -48,6 +49,9 @@ public AesStream(string password, Stream stream)
 
             var isNew = _stream.Length < PAGE_SIZE;
 
+            // start stream from zero position
+            _stream.Position = 0;
+
             try
             {
                 // new file? create new salt

diff --git a/LiteDB/Engine/Engine/Rebuild.cs b/LiteDB/Engine/Engine/Rebuild.cs
@@ -23,7 +23,7 @@ public long Rebuild(RebuildOptions options)
                 // do a checkpoint before starts
                 _walIndex.Checkpoint();
 
-                var originalLength = _disk.GetLength(FileOrigin.Data);
+                var originalLength = _disk.GetVirtualLength(FileOrigin.Data);
 
                 // create a savepoint in header page - restore if any error occurs
                 savepoint = _header.Savepoint();
@@ -32,7 +32,7 @@ public long Rebuild(RebuildOptions options)
                 _disk.Cache.Clear();
 
                 // must check if there is no data log
-                if (_disk.GetLength(FileOrigin.Log) > 0) throw new LiteException(0, "Rebuild operation requires no log file - run Checkpoint before continue");
+                if (_disk.GetVirtualLength(FileOrigin.Log) > 0) throw new LiteException(0, "Rebuild operation requires no log file - run Checkpoint before continue");
 
                 // initialize V8 file reader
                 var reader = new FileReaderV8(_header, _disk);
@@ -67,7 +67,7 @@ public long Rebuild(RebuildOptions options)
                 _disk.SetLength((_header.LastPageID + 1) * PAGE_SIZE, FileOrigin.Data);
 
                 // get new filelength to compare
-                var newLength = _disk.GetLength(FileOrigin.Data);
+                var newLength = _disk.GetVirtualLength(FileOrigin.Data);
 
                 return originalLength - newLength;
             }

diff --git a/LiteDB/Engine/Engine/Transaction.cs b/LiteDB/Engine/Engine/Transaction.cs
@@ -100,7 +100,7 @@ private void CommitAndReleaseTransaction(TransactionService transaction)
         {
             transaction.Commit();
             _monitor.ReleaseTransaction(transaction);
-            if (_header.Pragmas.Checkpoint > 0 && _disk.GetLength(FileOrigin.Log) > (_header.Pragmas.Checkpoint * PAGE_SIZE))
+            if (_header.Pragmas.Checkpoint > 0 && _disk.GetVirtualLength(FileOrigin.Log) > (_header.Pragmas.Checkpoint * PAGE_SIZE))
                 _walIndex.TryCheckpoint();
         }
     }

diff --git a/LiteDB/Engine/LiteEngine.cs b/LiteDB/Engine/LiteEngine.cs
@@ -90,7 +90,7 @@ public LiteEngine(EngineSettings settings)
                 _walIndex = new WalIndexService(_disk, _locker);
 
                 // if exists log file, restore wal index references (can update full _header instance)
-                if (_disk.GetLength(FileOrigin.Log) > 0)
+                if (_disk.GetVirtualLength(FileOrigin.Log) > 0)
                 {
                     _walIndex.RestoreIndex(ref _header);
                 }

diff --git a/LiteDB/Engine/Services/WalIndexService.cs b/LiteDB/Engine/Services/WalIndexService.cs
@@ -246,7 +246,7 @@ public void RestoreIndex(ref HeaderPage header)
         public int Checkpoint()
         {
             // no log file or no confirmed transaction, just exit
-            if (_disk.GetLength(FileOrigin.Log) == 0 || _confirmTransactions.Count == 0) return 0;
+            if (_disk.GetVirtualLength(FileOrigin.Log) == 0 || _confirmTransactions.Count == 0) return 0;
 
             var mustExit = _locker.EnterExclusive();
 
@@ -269,7 +269,7 @@ public int Checkpoint()
         public int TryCheckpoint()
         {
             // no log file or no confirmed transaction, just exit
-            if (_disk.GetLength(FileOrigin.Log) == 0 || _confirmTransactions.Count == 0) return 0;
+            if (_disk.GetVirtualLength(FileOrigin.Log) == 0 || _confirmTransactions.Count == 0) return 0;
 
             if (_locker.TryEnterExclusive(out var mustExit) == false) return 0;
 

diff --git a/LiteDB/Engine/SystemCollections/SysDatabase.cs b/LiteDB/Engine/SystemCollections/SysDatabase.cs
@@ -24,8 +24,8 @@ private IEnumerable<BsonDocument> SysDatabase()
 
                 ["creationTime"] = _header.CreationTime,
 
-                ["dataFileSize"] = (int)_disk.GetLength(FileOrigin.Data),
-                ["logFileSize"] = (int)_disk.GetLength(FileOrigin.Log),
+                ["dataFileSize"] = (int)_disk.GetVirtualLength(FileOrigin.Data),
+                ["logFileSize"] = (int)_disk.GetVirtualLength(FileOrigin.Log),
                 ["asyncQueueLength"] = _disk.Queue.Length,
 
                 ["currentReadVersion"] = _walIndex.CurrentReadVersion,