WIP

duplexsystem · Dec 31, 2022 · 5e3c10a · 5e3c10a
1 parent a76835d
commit 5e3c10a
Show file tree

Hide file tree

Showing 12 changed files with 383 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+/target
+/Cargo.lock
diff --git a/.idea/.gitignore b/.idea/.gitignore
diff --git a/.idea/discord.xml b/.idea/discord.xml
diff --git a/.idea/modules.xml b/.idea/modules.xml
diff --git a/.idea/p2vec.iml b/.idea/p2vec.iml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/Cargo.toml b/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "p2vec"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+
+# Utilites
+once_cell = { version = "1.13.1", features = ["parking_lot"] } # For dynamic dyspatch tables
+parking_lot = { version = "0.12.1", features = ["hardware-lock-elision"] } # Enable faster backed for oncecell
+dashmap = "5.4.0"
+bitvec = "1.0.1"
+ahash = "0.8.2"
+
+# Compression
+libdeflater = "0.11.0" # For defalte based compression
+flate2 = { version = "1.0.25", features = ["zlib"], default-features = false } # System zlib for streaming data. Slower but used as a fallback in case we can't use libdeflate. Doesn't take up much space because it uses the system zlib
+
+# Encryption
+openssl = "0.10.45" # System openssl for encryption. Well respected and it a common system libary.
+
+# File system
+memmap2 = "0.5.8" # For memory mapping files
+file-guard = "0.1.0" # For locking files
diff --git a/src/compression.rs b/src/compression.rs
@@ -0,0 +1,101 @@
+use libdeflater::{CompressionLvl, Compressor, Decompressor};
+use std::io::{Error, Read};
+
+// CompressionType is an enum that represents different compression types that this code can handle
+pub enum CompressionType {
+    Gzip,
+    Zlib,
+    Uncompressed,
+}
+
+// Implementations of various methods for the CompressionType enum
+impl CompressionType {
+    // Converts a u8 value to the corresponding CompressionType variant
+    pub fn from_u8(int: u8) -> Option<CompressionType> {
+        match int {
+            1 => Some(CompressionType::Gzip),
+            2 => Some(CompressionType::Zlib),
+            3 => Some(CompressionType::Uncompressed),
+            _ => None,
+        }
+    }
+
+    // Converts a CompressionType variant to the corresponding u8 value
+    pub fn to_u8(&self) -> u8 {
+        match self {
+            CompressionType::Gzip => 1,
+            CompressionType::Zlib => 2,
+            CompressionType::Uncompressed => 3,
+        }
+    }
+
+    // Decompresses a given slice of bytes using the decompression method corresponding to the CompressionType variant
+    pub fn decompress(&self, data: &[u8]) -> Result<Vec<u8>, Error> {
+        match self {
+            // For gzip compression, use libdeflate to decompress the data
+            CompressionType::Gzip => {
+                // gzip RFC1952: a valid gzip file has an ISIZE field in the
+                // footer, which is a little-endian u32 number representing the
+                // decompressed size. This is ideal for libdeflate, which needs
+                // preallocating the decompressed buffer.
+                let isize = {
+                    let isize_start = data.len() - 4;
+                    let isize_bytes = &data[isize_start..];
+                    let mut ret: u32 = isize_bytes[0] as u32;
+                    ret |= (isize_bytes[1] as u32) << 8;
+                    ret |= (isize_bytes[2] as u32) << 16;
+                    ret |= (isize_bytes[3] as u32) << 26;
+                    ret as usize
+                };
+
+                let mut decompressor = Decompressor::new();
+                let mut outbuf = Vec::new();
+                outbuf.resize(isize, 0);
+                decompressor.gzip_decompress(data, &mut outbuf).unwrap();
+                Ok(outbuf)
+            }
+            // For zlib compression, use the system zlib implementation provided by the `flate2` crate to decompress the data
+            CompressionType::Zlib => {
+                //we don't know the decompressed size, so we have to use system zlib here
+                let mut decoder = flate2::read::ZlibDecoder::new(data);
+                let mut buffer = Vec::new();
+                decoder.read_to_end(&mut buffer)?;
+                Ok(buffer)
+            }
+            // For uncompressed data, return a copy of the input data
+            CompressionType::Uncompressed => Ok(data.to_vec()),
+        }
+    }
+
+    // Compresses a given slice of bytes using the compression method corresponding to the CompressionType variant
+    pub fn compress(&self, data: &[u8], compression: CompressionLvl) -> Result<Vec<u8>, Error> {
+        match self {
+            // For gzip compression, use libdeflate to compress the data
+            CompressionType::Gzip => {
+                let mut compressor = Compressor::new(compression);
+                let max_sz = compressor.gzip_compress_bound(data.len());
+                let mut compressed_data = Vec::new();
+                compressed_data.resize(max_sz, 0);
+                let actual_sz = compressor
+                    .gzip_compress(data, &mut compressed_data)
+                    .unwrap();
+                compressed_data.resize(actual_sz, 0);
+                Ok(compressed_data)
+            }
+            // For zlib compression, use libdeflate to compress the data
+            CompressionType::Zlib => {
+                let mut compressor = Compressor::new(compression);
+                let max_sz = compressor.zlib_compress_bound(data.len());
+                let mut compressed_data = Vec::new();
+                compressed_data.resize(max_sz, 0);
+                let actual_sz = compressor
+                    .zlib_compress(data, &mut compressed_data)
+                    .unwrap();
+                compressed_data.resize(actual_sz, 0);
+                Ok(compressed_data)
+            }
+            // For uncompressed data, return a copy of the input data
+            CompressionType::Uncompressed => Ok(data.to_vec()),
+        }
+    }
+}
diff --git a/src/file.rs b/src/file.rs
@@ -0,0 +1,18 @@
+use std::fs;
+use std::fs::{File, OpenOptions};
+use std::io::Error;
+use std::path::Path;
+
+pub fn open_file(path: &Path) -> Result<File, Error> {
+    if !path.is_file() {
+        fs::create_dir_all(path.parent().unwrap())?;
+    }
+
+    let file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .open(path)?;
+
+    Ok(file)
+}
diff --git a/src/lib.rs b/src/lib.rs
@@ -0,0 +1,19 @@
+mod compression;
+mod file;
+mod region;
+mod util;
+
+pub fn add(left: usize, right: usize) -> usize {
+    left + right
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let result = add(2, 2);
+        assert_eq!(result, 4);
+    }
+}
diff --git a/src/region.rs b/src/region.rs
@@ -0,0 +1,174 @@
+use std::fs::File;
+use std::io::Error;
+use std::mem::{transmute, MaybeUninit};
+use std::path::Path;
+
+use crate::compression::CompressionType;
+use ahash::RandomState;
+use bitvec::vec::BitVec;
+use dashmap::mapref::one::RefMut;
+use dashmap::DashMap;
+use libdeflater::CompressionLvl;
+use memmap2::MmapMut;
+use once_cell::sync::Lazy;
+use parking_lot::RwLock;
+
+use crate::file::open_file;
+use crate::util::get_alignment_vector;
+
+pub struct Region<'a> {
+    directory: &'static str,
+    file: File,
+    data: RwLock<RegionData>,
+    chunks: Box<[[RwLock<Chunk<'a>>; 32]; 32]>,
+}
+
+struct RegionData {
+    file: MmapMut,
+    map: BitVec,
+}
+
+struct Chunk<'a> {
+    header_data: RwLock<HeaderData<'a>>,
+    data: RwLock<ChunkData<'a>>,
+}
+
+struct HeaderData<'a> {
+    location: &'a [u8],
+    timestamp: &'a [u8],
+}
+
+struct ChunkData<'a> {
+    data: &'a [u8],
+    oversized_data: Option<File>,
+}
+
+static REGIONS: Lazy<DashMap<(&'static str, i32, i32), Region, RandomState>> =
+    Lazy::new(|| DashMap::with_capacity_and_hasher(1, RandomState::default()));
+
+pub fn open_region(
+    directory: &'static str,
+    x: i32,
+    z: i32,
+) -> Result<RefMut<(&str, i32, i32), Region, RandomState>, Error> {
+    Ok(REGIONS
+        .entry((directory, x, z))
+        .or_insert_with(|| create_region(directory, x, z)))
+}
+
+fn create_region(directory: &'static str, region_x: i32, region_z: i32) -> Region {
+    let file = open_file(Path::new(&format!(
+        "{directory}/r.{region_x}.{region_z}.mca"
+    )))
+    .unwrap();
+    let memmap = unsafe { MmapMut::map_mut(&file).unwrap() };
+    let mut map = BitVec::new();
+    let chunks = {
+        // Create an array of uninitialized values.
+        let mut x_array: [MaybeUninit<[RwLock<Chunk>; 32]>; 32] =
+            unsafe { MaybeUninit::uninit().assume_init() };
+
+        let x_int = 0;
+        for x in x_array.iter_mut() {
+            // Create an array of uninitialized values.
+            let mut z_array: [MaybeUninit<RwLock<Chunk>>; 32] =
+                unsafe { MaybeUninit::uninit().assume_init() };
+
+            let z_int = 0;
+            for z in z_array.iter_mut() {
+                let offset = ((x_int % 32) + (z_int % 32) * 32) * 4;
+                let location = ((memmap[offset] as usize) << 16)
+                    | ((memmap[offset + 1] as usize) << 8)
+                    | memmap[offset + 3] as usize;
+                let sectors = memmap[offset + 4] as usize * 4096;
+                for i in location..sectors {
+                    map.set(i, true);
+                }
+                let mut file: Option<File> = None;
+                if memmap[location] == 0
+                    && memmap[location + 1] == 0
+                    && memmap[location + 2] == 0
+                    && memmap[location + 3] == 1
+                    && memmap[location + 4] == 82
+                {
+                    let chunk_x = x_int << 5 | region_x as usize;
+                    let chunk_z = z_int << 5 | region_z as usize;
+
+                    let mut chunk_file = open_file(Path::new(&format!(
+                        "{}/c.{}.{}.mcc",
+                        directory, chunk_x, chunk_z
+                    )))
+                    .unwrap();
+                    file = Some(chunk_file);
+                }
+                *z = MaybeUninit::new(RwLock::new(Chunk {
+                    header_data: RwLock::new(HeaderData {
+                        location: &memmap[offset..offset + 4],
+                        timestamp: &memmap[offset + 4096..offset + 4100],
+                    }),
+                    data: RwLock::new(ChunkData {
+                        data: &memmap[location..location + sectors],
+                        oversized_data: file,
+                    }),
+                }));
+            }
+
+            *x = MaybeUninit::new(unsafe { transmute::<_, [RwLock<Chunk>; 32]>(z_array) });
+        }
+
+        Box::new(unsafe { transmute::<_, [[RwLock<Chunk>; 32]; 32]>(x_array) })
+    };
+
+    Region {
+        directory,
+        file,
+        data: RwLock::new(RegionData { file: memmap, map }),
+        chunks,
+    }
+}
+
+pub fn close_region(directory: &'static str, region_x: i32, region_z: i32) -> Result<(), Error> {
+    let region_option = REGIONS.get_mut(&(directory, region_x, region_z));
+
+    if region_option.is_none() {
+        return Ok(());
+    }
+
+    let region = region_option.unwrap();
+
+    let region_data_lock = region.data.write();
+
+    let mut chunk_locks = Vec::new();
+
+    for x in 0..32 {
+        for z in 0..32 {
+            let chunk_lock = region.chunks[x][z]
+            let chunk_header_data_lock = chunk_lock.header_data\
+            let chunk_data_lock = chunk_lock.header_data.write();
+            chunk_locks.push((chunk_lock, chunk_header_data_lock, chunk_data_lock));
+        }
+    }
+
+    Ok(())
+}
+
+fn write_chunk_data(
+    directory: &'static str,
+    chunk_x: i32,
+    chunk_z: i32,
+    timestamp: u64,
+    data: &[u8],
+    compression_type: CompressionType,
+) -> Result<(), Error> {
+    // TODO Compression heuristic
+    let compressed_data = compression_type.compress(data, CompressionLvl::default())?;
+
+    let alignment_data = get_alignment_vector(compressed_data.len(), 4096);
+
+    let region_x = chunk_x >> 5;
+    let region_z = chunk_z >> 5;
+
+    let region = open_region(directory, region_x, region_z)?;
+
+    Ok(())
+}
diff --git a/src/util.rs b/src/util.rs
@@ -0,0 +1,3 @@
+pub fn get_alignment_vector(number: usize, alignment: usize) -> Vec<u8> {
+    vec![0_u8; (alignment - number % alignment) % alignment]
+}