Unverified Commit 5376617c authored by Nugine's avatar Nugine Committed by GitHub
Browse files

s3s: checksum: ChecksumHasher (#149)

parent e9bf1a2e

crates/s3s-fs/Cargo.toml

+0 −5
Original line number Diff line number Diff line
@@ -26,11 +26,8 @@ chrono = { version = "0.4.26", default-features = false, features = [ 
] }

clap = { version = "4.3.21", optional = true, features = ["derive"] }

crc32c = "0.6.4"

crc32fast = "1.3.2"

digest = "0.10.7"

futures = "0.3.28"

hex-simd = "0.8.0"



hyper-util = { version = "0.1.3", optional = true, features = [

    "server",

    "http1",
@@ -44,8 +41,6 @@ numeric_cast = "0.2.1" 
path-absolutize = "3.1.0"

s3s = { version = "0.10.1-dev", path = "../s3s" }

serde_json = "1.0.104"

sha1 = "0.10.5"

sha2 = "0.10.7"

thiserror = "1.0.45"

time = "0.3.25"

tokio = { version = "1.31.0", features = ["fs", "io-util"] }

crates/s3s-fs/src/checksum.rs

+1 −55
Original line number Diff line number Diff line 
use std::hash::Hasher;



use digest::Digest;

use numeric_cast::TruncatingCast;

use rust_utils::default::default;



use crate::fs::InternalInfo;



#[derive(Default)]

pub struct ChecksumCalculator {

    pub crc32: Option<crc32fast::Hasher>,

    pub crc32c: Option<crc32c::Crc32cHasher>,

    pub sha1: Option<sha1::Sha1>,

    pub sha256: Option<sha2::Sha256>,

}



impl ChecksumCalculator {

    pub fn update(&mut self, data: &[u8]) {

        if let Some(crc32) = &mut self.crc32 {

            crc32.update(data);

        }

        if let Some(crc32c) = &mut self.crc32c {

            crc32c.write(data);

        }

        if let Some(sha1) = &mut self.sha1 {

            sha1.update(data);

        }

        if let Some(sha256) = &mut self.sha256 {

            sha256.update(data);

        }

    }



    pub fn finalize(self) -> s3s::dto::Checksum {

        let mut ans: s3s::dto::Checksum = default();

        if let Some(crc32) = self.crc32 {

            let sum = crc32.finalize().to_be_bytes();

            ans.checksum_crc32 = Some(base64(&sum));

        }

        if let Some(crc32c) = self.crc32c {

            let sum = crc32c.finish().truncating_cast::<u32>().to_be_bytes();

            ans.checksum_crc32c = Some(base64(&sum));

        }

        if let Some(sha1) = self.sha1 {

            let sum = sha1.finalize();

            ans.checksum_sha1 = Some(base64(sum.as_ref()));

        }

        if let Some(sha256) = self.sha256 {

            let sum = sha256.finalize();

            ans.checksum_sha256 = Some(base64(sum.as_ref()));

        }

        ans

    }

}



fn base64(input: &[u8]) -> String {

    base64_simd::STANDARD.encode_to_string(input)

}

use rust_utils::default::default;



pub fn modify_internal_info(info: &mut serde_json::Map<String, serde_json::Value>, checksum: &s3s::dto::Checksum) {

    if let Some(checksum_crc32) = &checksum.checksum_crc32 {

crates/s3s-fs/src/s3.rs

+1 −1
Original line number Diff line number Diff line
@@ -446,7 +446,7 @@ impl S3 for FileSystem { 


        let Some(body) = body else { return Err(s3_error!(IncompleteBody)) };



        let mut checksum: crate::checksum::ChecksumCalculator = default();

        let mut checksum: s3s::checksum::ChecksumHasher = default();

        if input.checksum_crc32.is_some() {

            checksum.crc32 = Some(default());

        }

crates/s3s/Cargo.toml

+3 −0
Original line number Diff line number Diff line
@@ -27,7 +27,9 @@ base64-simd = "0.8.0" 
bytes = "1.4.0"

bytestring = "1.3.0"

chrono = { version = "0.4.26", default-features = false }

crc32c = "0.6.4"

crc32fast = "1.3.2"

digest = "0.10.7"

futures = { version = "0.3.28", default-features = false, features = ["std"] }

hex-simd = "0.8.0"

hmac = "0.12.1"
@@ -40,6 +42,7 @@ memchr = "2.6.2" 
mime = "0.3.17"

nom = "7.1.3"

nugine-rust-utils = "0.3.1"

numeric_cast = "0.2.1"

pin-project-lite = "0.2.12"

quick-xml = { version = "0.31.0", features = ["serialize"] }

serde = { version = "1.0.183", features = ["derive"] }

crates/s3s/src/checksum.rs

0 → 100644
+58 −0
Original line number Diff line number Diff line 
use crate::dto::Checksum;



use std::hash::Hasher;



use digest::Digest;

use numeric_cast::TruncatingCast;

use rust_utils::default::default;



#[derive(Default)]

pub struct ChecksumHasher {

    pub crc32: Option<crc32fast::Hasher>,

    pub crc32c: Option<crc32c::Crc32cHasher>,

    pub sha1: Option<sha1::Sha1>,

    pub sha256: Option<sha2::Sha256>,

}



impl ChecksumHasher {

    pub fn update(&mut self, data: &[u8]) {

        if let Some(crc32) = &mut self.crc32 {

            crc32.update(data);

        }

        if let Some(crc32c) = &mut self.crc32c {

            crc32c.write(data);

        }

        if let Some(sha1) = &mut self.sha1 {

            sha1.update(data);

        }

        if let Some(sha256) = &mut self.sha256 {

            sha256.update(data);

        }

    }



    #[must_use]

    pub fn finalize(self) -> Checksum {

        let mut ans: Checksum = default();

        if let Some(crc32) = self.crc32 {

            let sum = crc32.finalize().to_be_bytes();

            ans.checksum_crc32 = Some(Self::base64(&sum));

        }

        if let Some(crc32c) = self.crc32c {

            let sum = crc32c.finish().truncating_cast::<u32>().to_be_bytes();

            ans.checksum_crc32c = Some(Self::base64(&sum));

        }

        if let Some(sha1) = self.sha1 {

            let sum = sha1.finalize();

            ans.checksum_sha1 = Some(Self::base64(sum.as_ref()));

        }

        if let Some(sha256) = self.sha256 {

            let sum = sha256.finalize();

            ans.checksum_sha256 = Some(Self::base64(sum.as_ref()));

        }

        ans

    }



    fn base64(input: &[u8]) -> String {

        base64_simd::STANDARD.encode_to_string(input)

    }

}