Unverified Commit 54124ba3 authored by Adam Cimarosti's avatar Adam Cimarosti Committed by GitHub
Browse files

s3s-fs: fix list_objects_v2 on Windows (#101)

parent 8ffaeff6

crates/s3s-fs/src/s3.rs

+29 −4
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ use std::collections::VecDeque; 
use std::io;

use std::ops::Neg;

use std::ops::Not;

use std::path::PathBuf;

use std::path::{Path, PathBuf};



use tokio::fs;

use tokio::io::AsyncSeekExt;
@@ -23,9 +23,32 @@ use futures::TryStreamExt; 
use md5::{Digest, Md5};

use numeric_cast::NumericCast;

use rust_utils::default::default;

use std::path::Component;

use std::string::ToString;

use tracing::debug;

use uuid::Uuid;



fn normalize_path(path: &Path, delimiter: &str) -> Option<String> {

    let mut normalized = String::new();

    let mut first = true;

    for component in path.components() {

        match component {

            Component::RootDir | Component::CurDir | Component::ParentDir | Component::Prefix(_) => {

                return None;

            }

            Component::Normal(name) => {

                let name = name.to_str()?;

                if !first {

                    normalized.push_str(delimiter);

                }

                normalized.push_str(name);

                first = false;

            }

        }

    }

    Some(normalized)

}



#[async_trait::async_trait]

impl S3 for FileSystem {

    #[tracing::instrument]
@@ -335,10 +358,12 @@ impl S3 for FileSystem { 
                } else {

                    let file_path = entry.path();

                    let key = try_!(file_path.strip_prefix(&path));

                    let Some(key) = key.to_str() else { continue };

                    let delimiter = input.delimiter.as_ref().map_or("/", |d| d.as_str());

                    let Some(key_str) = normalize_path(key, delimiter) else { continue };



                    if let Some(ref prefix) = input.prefix {

                        if !key.starts_with(prefix) {

                        let prefix_path: PathBuf = prefix.split(delimiter).collect();

                        if !key.starts_with(prefix_path) {

                            continue;

                        }

                    }
@@ -348,7 +373,7 @@ impl S3 for FileSystem { 
                    let size = metadata.len();



                    let object = Object {

                        key: Some(key.to_owned()),

                        key: Some(key_str),

                        last_modified: Some(last_modified),

                        size: try_!(i64::try_from(size)),

                        ..Default::default()

crates/s3s-fs/tests/it_aws.rs

+77 −5
Original line number Diff line number Diff line
@@ -111,18 +111,90 @@ async fn delete_bucket(c: &Client, bucket: &str) -> Result<()> { 
    Ok(())

}



macro_rules! log_and_unwrap {

    ($result:expr) => {

        match $result {

            Ok(ans) => {

                debug!(?ans);

                ans

            }

            Err(err) => {

                error!(?err);

                return Err(err.into());

            }

        }

    };

}



#[tokio::test]

#[tracing::instrument]

async fn test_list_buckets() -> Result<()> {

    let c = Client::new(config());

    let result = c.list_buckets().send().await;

    let response1 = log_and_unwrap!(c.list_buckets().send().await);

    assert!(response1.buckets().is_some());



    let bucket1 = format!("test-list-buckets-1-{}", Uuid::new_v4());

    let bucket1_str = bucket1.as_str();

    let bucket2 = format!("test-list-buckets-2-{}", Uuid::new_v4());

    let bucket2_str = bucket2.as_str();



    create_bucket(&c, bucket1_str).await?;

    create_bucket(&c, bucket2_str).await?;



    let response2 = log_and_unwrap!(c.list_buckets().send().await);

    assert!(response2.buckets().is_some());

    let bucket_names: Vec<_> = response2

        .buckets()

        .unwrap()

        .iter()

        .filter_map(|bucket| bucket.name())

        .collect();

    assert!(bucket_names.contains(&bucket1_str));

    assert!(bucket_names.contains(&bucket2_str));



    Ok(())

}



#[tokio::test]

#[tracing::instrument]

async fn test_list_objects_v2() -> Result<()> {

    let c = Client::new(config());

    let bucket = format!("test-list-objects-v2-{}", Uuid::new_v4());

    let bucket_str = bucket.as_str();

    create_bucket(&c, bucket_str).await?;



    match result {

        Ok(ref ans) => debug!(?ans),

        Err(ref err) => error!(?err),

    let test_prefix = "/this/is/a/test/";

    let key1 = "this/is/a/test/path/file1.txt";

    let key2 = "this/is/a/test/path/file2.txt";

    {

        let content = "hello world\nनमस्ते दुनिया\n";

        let crc32c = base64_simd::STANDARD.encode_to_string(crc32c::crc32c(content.as_bytes()).to_be_bytes());

        c.put_object()

            .bucket(bucket_str)

            .key(key1)

            .body(ByteStream::from_static(content.as_bytes()))

            .checksum_crc32_c(crc32c.as_str())

            .send()

            .await?;

        c.put_object()

            .bucket(bucket_str)

            .key(key2)

            .body(ByteStream::from_static(content.as_bytes()))

            .checksum_crc32_c(crc32c.as_str())

            .send()

            .await?;

    }



    result?;

    let result = c.list_objects_v2().bucket(bucket_str).prefix(test_prefix).send().await;



    let response = log_and_unwrap!(result);



    assert!(response.contents().is_some());

    let contents: Vec<_> = response.contents().unwrap().iter().filter_map(|obj| obj.key()).collect();

    assert!(!contents.is_empty());

    assert!(contents.contains(&key1));

    assert!(contents.contains(&key2));



    Ok(())

}