Unverified Commit 368a6060 authored by Don MacAskill's avatar Don MacAskill Committed by GitHub
Browse files

Use crc-fast instead of crc32fast & crc64fast-nvme (#380) 



* Use crc-fast instead of crc32fast & crc64fast-nvme

The crc-fast crate is faster, more full featured, and better maintained.

Plus it calculates both CRC variants in a single library.

* fix lints

* fix wasm

* fix AI review

---------

Co-authored-by: default avatarNugine <nugine@foxmail.com>
parent 1ab064b6

Cargo.lock

+16 −26
Original line number Diff line number Diff line
@@ -973,15 +973,6 @@ dependencies = [ 
 "cfg-if",

]



[[package]]

name = "crc64fast-nvme"

version = "1.2.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3"

dependencies = [

 "crc",

]



[[package]]

name = "crunchy"

version = "0.2.4"
@@ -1330,21 +1321,21 @@ dependencies = [ 
 "cfg-if",

 "js-sys",

 "libc",

 "wasi 0.11.1+wasi-snapshot-preview1",

 "wasi",

 "wasm-bindgen",

]



[[package]]

name = "getrandom"

version = "0.3.3"

version = "0.3.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"

checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"

dependencies = [

 "cfg-if",

 "js-sys",

 "libc",

 "r-efi",

 "wasi 0.14.4+wasi-0.2.4",

 "wasip2",

 "wasm-bindgen",

]
@@ -1861,7 +1852,7 @@ version = "0.1.34" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"

dependencies = [

 "getrandom 0.3.3",

 "getrandom 0.3.4",

 "libc",

]
@@ -2025,7 +2016,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" 
checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"

dependencies = [

 "libc",

 "wasi 0.11.1+wasi-snapshot-preview1",

 "wasi",

 "windows-sys 0.59.0",

]
@@ -2363,7 +2354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" 
checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"

dependencies = [

 "bytes",

 "getrandom 0.3.3",

 "getrandom 0.3.4",

 "lru-slab",

 "rand 0.9.2",

 "ring",
@@ -2462,7 +2453,7 @@ version = "0.9.3" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"

dependencies = [

 "getrandom 0.3.3",

 "getrandom 0.3.4",

]



[[package]]
@@ -2772,9 +2763,7 @@ dependencies = [ 
 "cfg-if",

 "chrono",

 "const-str",

 "crc32c",

 "crc32fast",

 "crc64fast-nvme",

 "crc-fast",

 "futures",

 "hex-simd",

 "hmac 0.13.0-rc.3",
@@ -2960,6 +2949,7 @@ name = "s3s-wasm" 
version = "0.0.0"

dependencies = [

 "futures",

 "getrandom 0.3.4",

 "http 1.3.1",

 "s3s",

 "wasm-bindgen-test",
@@ -3655,7 +3645,7 @@ version = "1.18.1" 
source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"

dependencies = [

 "getrandom 0.3.3",

 "getrandom 0.3.4",

 "js-sys",

 "serde",

 "wasm-bindgen",
@@ -3711,10 +3701,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" 
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"



[[package]]

name = "wasi"

version = "0.14.4+wasi-0.2.4"

name = "wasip2"

version = "1.0.1+wasi-0.2.4"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "88a5f4a424faf49c3c2c344f166f0662341d470ea185e939657aaff130f0ec4a"

checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"

dependencies = [

 "wit-bindgen",

]
@@ -4096,9 +4086,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" 


[[package]]

name = "wit-bindgen"

version = "0.45.1"

version = "0.46.0"

source = "registry+https://github.com/rust-lang/crates.io-index"

checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36"

checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"



[[package]]

name = "writeable"

crates/s3s-wasm/Cargo.toml

+1 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ publish = false 


[dependencies]

futures = "0.3.31"

getrandom = { version = "0.3.4", features = ["wasm_js"] }

http = "1.3.1"

s3s = { version = "0.12.0-rc.3", path = "../s3s", default-features = false }

crates/s3s/Cargo.toml

+1 −3
Original line number Diff line number Diff line
@@ -32,8 +32,7 @@ base64-simd = "0.8.0" 
bytes = "1.10.1"

bytestring = "1.5.0"

chrono = { version = "0.4.42", default-features = false }

crc32c = "0.6.8"

crc32fast = "1.5.0"

crc-fast = "1"

futures = { version = "0.3.31", default-features = false, features = ["std"] }

hex-simd = "0.8.0"

hmac.workspace = true
@@ -64,7 +63,6 @@ zeroize = "1.8.2" 
std-next = "0.1.9"

sync_wrapper = { version = "1.0.2", default-features = false }

tokio = { version = "1.48.0", features = ["time"] }

crc64fast-nvme = "1.2.0"

const-str = "0.7.0"

http = "1.3.1"

subtle = "2.6.1"

crates/s3s/src/crypto.rs

+37 −11
Original line number Diff line number Diff line 
use numeric_cast::TruncatingCast;



pub trait Checksum {

    type Output: AsRef<[u8]>;
@@ -20,8 +22,22 @@ pub trait Checksum { 
    }

}



#[derive(Default)]

pub struct Crc32(crc32fast::Hasher);

pub struct Crc32(crc_fast::Digest);



impl Default for Crc32 {

    fn default() -> Self {

        Self(crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc))

    }

}



impl Crc32 {

    #[must_use]

    pub fn checksum_u32(data: &[u8]) -> u32 {

        let mut hasher = Self::new();

        hasher.update(data);

        hasher.0.finalize().truncating_cast::<u32>()

    }

}



impl Checksum for Crc32 {

    type Output = [u8; 4];
@@ -35,12 +51,17 @@ impl Checksum for Crc32 { 
    }



    fn finalize(self) -> Self::Output {

        self.0.finalize().to_be_bytes()

        self.0.finalize().truncating_cast::<u32>().to_be_bytes()

    }

}



#[derive(Default)]

pub struct Crc32c(u32);

pub struct Crc32c(crc_fast::Digest);



impl Default for Crc32c {

    fn default() -> Self {

        Self(crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32Iscsi))

    }

}



impl Checksum for Crc32c {

    type Output = [u8; 4];
@@ -50,16 +71,21 @@ impl Checksum for Crc32c { 
    }



    fn update(&mut self, data: &[u8]) {

        self.0 = crc32c::crc32c_append(self.0, data);

        self.0.update(data);

    }



    fn finalize(self) -> Self::Output {

        self.0.to_be_bytes()

        self.0.finalize().truncating_cast::<u32>().to_be_bytes()

    }

}



#[derive(Default)]

pub struct Crc64Nvme(crc64fast_nvme::Digest);

pub struct Crc64Nvme(crc_fast::Digest);



impl Default for Crc64Nvme {

    fn default() -> Self {

        Self(crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc64Nvme))

    }

}



impl Checksum for Crc64Nvme {

    type Output = [u8; 8];
@@ -69,11 +95,11 @@ impl Checksum for Crc64Nvme { 
    }



    fn update(&mut self, data: &[u8]) {

        self.0.write(data);

        self.0.update(data);

    }



    fn finalize(self) -> Self::Output {

        self.0.sum64().to_be_bytes()

        self.0.finalize().to_be_bytes()

    }

}

crates/s3s/src/dto/event_stream.rs

+3 −2
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ use super::{ContinuationEvent, EndEvent, ProgressEvent, RecordsEvent, StatsEvent 
use crate::S3Error;

use crate::S3Result;

use crate::StdError;

use crate::crypto::Crc32;

use crate::stream::ByteStream;

use crate::stream::DynByteStream;

use crate::{S3ErrorCode, xml};
@@ -137,7 +138,7 @@ impl Message { 
        buf.put_u32(total_byte_length);

        buf.put_u32(headers_byte_length);



        let prelude_crc = crc32fast::hash(&buf);

        let prelude_crc = Crc32::checksum_u32(&buf);

        buf.put_u32(prelude_crc);



        for h in &self.headers {
@@ -156,7 +157,7 @@ impl Message { 
            buf.put(payload);

        }



        let message_crc = crc32fast::hash(&buf);

        let message_crc = Crc32::checksum_u32(&buf);

        buf.put_u32(message_crc);



        Ok(buf.into())