From f97e6f3b101229a5442963b002cbad4cc5df62d5 Mon Sep 17 00:00:00 2001 From: Zelda Hessler Date: Wed, 27 Mar 2024 22:10:32 -0500 Subject: [PATCH] fix the `./gradlew :aws:sdk:assemble` command and aws-config tests on Windows (#3522) tracked in issue #3518 ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --- .../aws-config/src/credential_process.rs | 4 +++ .../src/default_provider/credentials.rs | 10 ++++-- .../aws-config/src/imds/client.rs | 2 ++ .../aws-config/src/imds/credentials.rs | 2 ++ .../aws-config/src/profile/credentials.rs | 8 +++-- .../aws-config/src/profile/parser/source.rs | 2 ++ aws/rust-runtime/aws-config/src/sso/cache.rs | 4 +++ aws/rust-runtime/aws-config/src/sso/token.rs | 14 ++++++++ aws/sdk/build.gradle.kts | 9 +++-- .../rust/codegen/core/rustlang/RustWriter.kt | 6 +++- tools/ci-build/crate-hasher/src/file_list.rs | 35 +++++++++++++------ tools/ci-scripts/test-windows.sh | 2 ++ 12 files changed, 78 insertions(+), 20 deletions(-) diff --git a/aws/rust-runtime/aws-config/src/credential_process.rs b/aws/rust-runtime/aws-config/src/credential_process.rs index e60ce4a24..6aec2a988 100644 --- a/aws/rust-runtime/aws-config/src/credential_process.rs +++ b/aws/rust-runtime/aws-config/src/credential_process.rs @@ -231,7 +231,9 @@ mod test { use time::OffsetDateTime; use tokio::time::timeout; + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths #[tokio::test] + #[cfg_attr(windows, ignore)] async fn test_credential_process() { let provider = CredentialProcessProvider::new(String::from( r#"echo '{ "Version": 1, "AccessKeyId": "ASIARTESTID", "SecretAccessKey": "TESTSECRETKEY", "SessionToken": "TESTSESSIONTOKEN", "Expiration": "2022-05-02T18:36:00+00:00" }'"#, @@ -252,7 +254,9 @@ mod test { ); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths #[tokio::test] + #[cfg_attr(windows, ignore)] async fn test_credential_process_no_expiry() { let provider = CredentialProcessProvider::new(String::from( r#"echo '{ "Version": 1, "AccessKeyId": "ASIARTESTID", "SecretAccessKey": "TESTSECRETKEY" }'"#, diff --git a/aws/rust-runtime/aws-config/src/default_provider/credentials.rs b/aws/rust-runtime/aws-config/src/default_provider/credentials.rs index 464b5dcb4..809a23dfe 100644 --- a/aws/rust-runtime/aws-config/src/default_provider/credentials.rs +++ b/aws/rust-runtime/aws-config/src/default_provider/credentials.rs @@ -299,17 +299,21 @@ mod test { make_test!(ecs_credentials_invalid_profile); make_test!(eks_pod_identity_credentials); + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is disabled on Windows because it uses Unix-style paths + #[cfg(not(windows))] make_test!(eks_pod_identity_no_token_file); #[cfg(not(feature = "sso"))] make_test!(sso_assume_role #[should_panic(expected = "This behavior requires following cargo feature(s) enabled: sso")]); - #[cfg(not(feature = "sso"))] - make_test!(sso_no_token_file #[should_panic(expected = "This behavior requires following cargo feature(s) enabled: sso")]); #[cfg(feature = "sso")] make_test!(sso_assume_role); - #[cfg(feature = "sso")] + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is disabled on Windows because it uses Unix-style paths + #[cfg(not(any(feature = "sso", windows)))] + make_test!(sso_no_token_file #[should_panic(expected = "This behavior requires following cargo feature(s) enabled: sso")]); + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is disabled on Windows because it uses Unix-style paths + #[cfg(all(feature = "sso", not(windows)))] make_test!(sso_no_token_file); #[cfg(feature = "credentials-sso")] diff --git a/aws/rust-runtime/aws-config/src/imds/client.rs b/aws/rust-runtime/aws-config/src/imds/client.rs index a660369c5..7bdc14f2a 100644 --- a/aws/rust-runtime/aws-config/src/imds/client.rs +++ b/aws/rust-runtime/aws-config/src/imds/client.rs @@ -982,6 +982,8 @@ pub(crate) mod test { http_client.assert_requests_match(&[]); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] /// Verify that the end-to-end real client has a 1-second connect timeout #[tokio::test] #[cfg(feature = "rustls")] diff --git a/aws/rust-runtime/aws-config/src/imds/credentials.rs b/aws/rust-runtime/aws-config/src/imds/credentials.rs index 896e66d6d..888237d1a 100644 --- a/aws/rust-runtime/aws-config/src/imds/credentials.rs +++ b/aws/rust-runtime/aws-config/src/imds/credentials.rs @@ -454,6 +454,8 @@ mod test { ); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] #[tokio::test] #[cfg(feature = "rustls")] async fn external_timeout_during_credentials_refresh_should_yield_last_retrieved_credentials() { diff --git a/aws/rust-runtime/aws-config/src/profile/credentials.rs b/aws/rust-runtime/aws-config/src/profile/credentials.rs index 47478a4c4..9d8b4255b 100644 --- a/aws/rust-runtime/aws-config/src/profile/credentials.rs +++ b/aws/rust-runtime/aws-config/src/profile/credentials.rs @@ -594,9 +594,11 @@ mod test { make_test!(retry_on_error); make_test!(invalid_config); make_test!(region_override); - #[cfg(feature = "credentials-process")] + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is disabled on Windows because it uses Unix-style paths + #[cfg(all(feature = "credentials-process", not(windows)))] make_test!(credential_process); - #[cfg(feature = "credentials-process")] + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is disabled on Windows because it uses Unix-style paths + #[cfg(all(feature = "credentials-process", not(windows)))] make_test!(credential_process_failure); #[cfg(feature = "credentials-process")] make_test!(credential_process_invalid); @@ -622,6 +624,8 @@ mod sso_tests { use aws_types::os_shim_internal::{Env, Fs}; use std::collections::HashMap; + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] // In order to preserve the SSO token cache, the inner provider must only // be created once, rather than once per credential resolution. #[tokio::test] diff --git a/aws/rust-runtime/aws-config/src/profile/parser/source.rs b/aws/rust-runtime/aws-config/src/profile/parser/source.rs index dc47c9b99..26dc4e129 100644 --- a/aws/rust-runtime/aws-config/src/profile/parser/source.rs +++ b/aws/rust-runtime/aws-config/src/profile/parser/source.rs @@ -413,7 +413,9 @@ mod tests { assert_eq!(contents, source.files[0].contents); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths #[tokio::test] + #[cfg_attr(windows, ignore)] async fn programmatically_include_default_files() { let config_contents = "[default]\nregion = us-east-1"; let credentials_contents = "[default]\n\ diff --git a/aws/rust-runtime/aws-config/src/sso/cache.rs b/aws/rust-runtime/aws-config/src/sso/cache.rs index 5d78798f8..3955d1b3a 100644 --- a/aws/rust-runtime/aws-config/src/sso/cache.rs +++ b/aws/rust-runtime/aws-config/src/sso/cache.rs @@ -514,6 +514,8 @@ mod tests { ); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] #[test] fn determine_correct_cache_filenames() { assert_eq!( @@ -534,6 +536,8 @@ mod tests { ); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] #[tokio::test] async fn save_cached_token() { let expires_at = SystemTime::UNIX_EPOCH + Duration::from_secs(50_000_000); diff --git a/aws/rust-runtime/aws-config/src/sso/token.rs b/aws/rust-runtime/aws-config/src/sso/token.rs index 98b2b28f5..0cf6f0572 100644 --- a/aws/rust-runtime/aws-config/src/sso/token.rs +++ b/aws/rust-runtime/aws-config/src/sso/token.rs @@ -513,6 +513,8 @@ mod tests { } } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] #[tokio::test] async fn use_unexpired_cached_token() { let fs = Fs::from_slice(&[( @@ -536,6 +538,8 @@ mod tests { req_rx.expect_no_request(); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] #[tokio::test] async fn expired_cached_token() { let fs = Fs::from_slice(&[( @@ -557,6 +561,8 @@ mod tests { req_rx.expect_no_request(); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] #[tokio::test] async fn expired_token_and_expired_client_registration() { let fs = Fs::from_slice(&[( @@ -584,6 +590,8 @@ mod tests { req_rx.expect_no_request(); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] #[tokio::test] async fn expired_token_refresh_with_refresh_token() { let fs = Fs::from_slice(&[( @@ -666,6 +674,8 @@ mod tests { ); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] #[tokio::test] async fn expired_token_refresh_fails() { let fs = Fs::from_slice(&[( @@ -705,6 +715,8 @@ mod tests { let _ = req_rx.expect_request(); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] // Expired token refresh without new refresh token #[tokio::test] async fn expired_token_refresh_without_new_refresh_token() { @@ -755,6 +767,8 @@ mod tests { let _ = req_rx.expect_request(); } + // TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) This test is ignored on Windows because it uses Unix-style paths + #[cfg_attr(windows, ignore)] #[tokio::test] async fn refresh_timings() { let _logs = capture_test_logs(); diff --git a/aws/sdk/build.gradle.kts b/aws/sdk/build.gradle.kts index 6d293dd92..bcc25bbe4 100644 --- a/aws/sdk/build.gradle.kts +++ b/aws/sdk/build.gradle.kts @@ -87,6 +87,9 @@ fun generateSmithyBuild(services: AwsServices): String { } val moduleName = "aws-sdk-${service.module}" val eventStreamAllowListMembers = eventStreamAllowList.joinToString(", ") { "\"$it\"" } + val defaultConfigPath = services.defaultConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") } + val partitionsConfigPath = services.partitionsConfigPath.let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") } + val integrationTestPath = project.projectDir.resolve("integration-tests").let { software.amazon.smithy.utils.StringUtils.escapeJavaString(it, "") } """ "${service.module}": { "imports": [${files.joinToString()}], @@ -117,9 +120,9 @@ fun generateSmithyBuild(services: AwsServices): String { "awsSdk": { "awsSdkBuild": true, "awsConfigVersion": "$awsConfigVersion", - "defaultConfigPath": "${services.defaultConfigPath}", - "partitionsConfigPath": "${services.partitionsConfigPath}", - "integrationTestPath": "${project.projectDir.resolve("integration-tests")}" + "defaultConfigPath": $defaultConfigPath, + "partitionsConfigPath": $partitionsConfigPath, + "integrationTestPath": $integrationTestPath } } ${service.extraConfig ?: ""} diff --git a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt index 8481cfadb..7034ed1f1 100644 --- a/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt +++ b/codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/RustWriter.kt @@ -624,7 +624,11 @@ class RustWriter private constructor( init { expressionStart = '#' if (filename.endsWith(".rs")) { - require(namespace.startsWith("crate") || filename.startsWith("tests/") || filename == "build.rs") { + require( + namespace.startsWith("crate") || + filename.startsWith("tests${File.separator}") || + filename == "build.rs", + ) { "We can only write into files in the crate (got $namespace)" } } diff --git a/tools/ci-build/crate-hasher/src/file_list.rs b/tools/ci-build/crate-hasher/src/file_list.rs index 698caca2b..4d9c81819 100644 --- a/tools/ci-build/crate-hasher/src/file_list.rs +++ b/tools/ci-build/crate-hasher/src/file_list.rs @@ -3,13 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ +use anyhow::{Context, Result}; use std::collections::BTreeSet; use std::fmt::Write; use std::fs::Metadata; use std::path::Path; -use anyhow::{Context, Result}; - #[derive(Debug, Default)] pub struct FileList(BTreeSet); @@ -95,16 +94,30 @@ impl FileMetadata { } } -/// Returns the file mode (permissions) for the given path +/// Returns the file mode (permissions) for the given path. +/// +/// On Windows, this returns the file attributes instead of the permissions. fn file_mode(path: &Path, metadata: &Metadata) -> Result { - use std::os::unix::fs::PermissionsExt; - - if metadata.file_type().is_symlink() { - let actual_path = std::fs::read_link(path).context("follow symlink")?; - let actual_metadata = std::fs::metadata(&actual_path).context("file metadata")?; - file_mode(&actual_path, &actual_metadata) - } else { - Ok(metadata.permissions().mode()) + #[cfg(windows)] + { + use std::os::windows::fs::MetadataExt; + let _ = path; // Unused on Windows + + // Not actually equivalent to a unix permission, but good enough to be hashed. + Ok(metadata.file_attributes()) + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + if metadata.file_type().is_symlink() { + let actual_path = std::fs::read_link(path).context("follow symlink")?; + let actual_metadata = std::fs::metadata(&actual_path).context("file metadata")?; + file_mode(&actual_path, &actual_metadata) + } else { + Ok(metadata.permissions().mode()) + } } } diff --git a/tools/ci-scripts/test-windows.sh b/tools/ci-scripts/test-windows.sh index 1c1b8a142..8a260bc95 100755 --- a/tools/ci-scripts/test-windows.sh +++ b/tools/ci-scripts/test-windows.sh @@ -16,4 +16,6 @@ for runtime_path in "rust-runtime" "aws/rust-runtime"; do cargo doc --no-deps --document-private-items --all-features --workspace "${exclusions[@]}" popd &>/dev/null done +# TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) We don't have a way to codegen the deps needed by the aws-config crate +# (cd aws/rust-runtime/aws-config && cargo test --all-features) # aws-config is not part of the workspace so we have to test it separately (cd rust-runtime && cargo test -p aws-smithy-experimental --features crypto-ring) # only ring works on windows -- GitLab