Unverified Commit d1bbd018 authored by Zelda Hessler's avatar Zelda Hessler Committed by GitHub
Browse files

re-add support for sourcing endpoint URLs from service-specific env config (#3568) 



The service env config work I did was flawed in that it didn't respect
the precedence of programmatic config. This PR
adds a way of tracking the precedence up to the point of converting the
SdkConfig into a service config. Now, env config will only be resolved
when config was not set programmatically. I added tests to `aws-config`
for the origin tracking to ensure it works.

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Co-authored-by: default avatarysaito1001 <awsaito@amazon.com>
parent 8040cced

aws/rust-runtime/aws-config/Cargo.toml

+1 −1
Original line number Diff line number Diff line 
[package]

name = "aws-config"

version = "1.1.10"

version = "1.2.0"

authors = [

    "AWS Rust SDK Team <aws-sdk-rust@amazon.com>",

    "Russell Cohen <rcoh@amazon.com>",

aws/rust-runtime/aws-config/src/default_provider/endpoint_url.rs

+24 −0
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ use crate::environment::parse_url; 
use crate::provider_config::ProviderConfig;

use aws_runtime::env_config::EnvConfigValue;

use aws_smithy_types::error::display::DisplayErrorContext;

use aws_types::origin::Origin;



mod env {

    pub(super) const ENDPOINT_URL: &str = "AWS_ENDPOINT_URL";
@@ -37,6 +38,29 @@ pub async fn endpoint_url_provider(provider_config: &ProviderConfig) -> Option<S 
        .unwrap_or(None)

}



/// Load the value for an endpoint URL

///

/// This checks the following sources:

/// 1. The environment variable `AWS_ENDPOINT_URL=http://localhost`

/// 2. The profile key `endpoint_url=http://localhost`

///

/// If invalid values are found, the provider will return None and an error will be logged.

pub async fn endpoint_url_provider_with_origin(

    provider_config: &ProviderConfig,

) -> (Option<String>, Origin) {

    let env = provider_config.env();

    let profiles = provider_config.profile().await;



    EnvConfigValue::new()

        .env(env::ENDPOINT_URL)

        .profile(profile_key::ENDPOINT_URL)

        .validate_and_return_origin(&env, profiles, parse_url)

        .map_err(

            |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for endpoint URL setting"),

        )

        .unwrap_or_default()

}



#[cfg(test)]

mod test {

    use super::endpoint_url_provider;

aws/rust-runtime/aws-config/src/lib.rs

+80 −4
Original line number Diff line number Diff line
@@ -223,6 +223,7 @@ mod loader { 
    use aws_smithy_types::timeout::TimeoutConfig;

    use aws_types::app_name::AppName;

    use aws_types::docs_for;

    use aws_types::origin::Origin;

    use aws_types::os_shim_internal::{Env, Fs};

    use aws_types::sdk_config::SharedHttpClient;

    use aws_types::SdkConfig;
@@ -605,7 +606,6 @@ mod loader { 
        ///

        /// ```no_run

        /// use aws_config::profile::{ProfileFileCredentialsProvider, ProfileFileRegionProvider};

        /// use aws_config::profile::profile_file::{ProfileFiles, ProfileFileKind};

        ///

        /// # async fn example() {

        /// let sdk_config = aws_config::from_env()
@@ -624,8 +624,8 @@ mod loader { 
        /// exists to set a static endpoint for tools like `LocalStack`. When sending requests to

        /// production AWS services, this method should only be used for service-specific behavior.

        ///

        /// When this method is used, the [`Region`](aws_types::region::Region) is only used for

        /// signing; it is not used to route the request.

        /// When this method is used, the [`Region`](aws_types::region::Region) is only used for signing;

        /// It is **not** used to route the request.

        ///

        /// # Examples

        ///
@@ -823,6 +823,7 @@ mod loader { 


            // If an endpoint URL is set programmatically, then our work is done.

            let endpoint_url = if self.endpoint_url.is_some() {

                builder.insert_origin("endpoint_url", Origin::shared_config());

                self.endpoint_url

            } else {

                // Otherwise, check to see if we should ignore EP URLs set in the environment.
@@ -840,7 +841,9 @@ mod loader { 
                    None

                } else {

                    // Otherwise, attempt to resolve one.

                    endpoint_url::endpoint_url_provider(&conf).await

                    let (v, origin) = endpoint_url::endpoint_url_provider_with_origin(&conf).await;

                    builder.insert_origin("endpoint_url", origin);

                    v

                }

            };

            builder.set_endpoint_url(endpoint_url);
@@ -884,6 +887,7 @@ mod loader { 
        use aws_smithy_runtime::client::http::test_util::{infallible_client_fn, NeverClient};

        use aws_smithy_runtime::test_util::capture_test_logs::capture_test_logs;

        use aws_types::app_name::AppName;

        use aws_types::origin::Origin;

        use aws_types::os_shim_internal::{Env, Fs};

        use std::sync::atomic::{AtomicUsize, Ordering};

        use std::sync::Arc;
@@ -950,6 +954,78 @@ mod loader { 
                .http_client(no_traffic_client())

        }



        #[tokio::test]

        async fn test_origin_programmatic() {

            let _ = tracing_subscriber::fmt::try_init();

            let loader = base_conf()

                .test_credentials()

                .profile_name("custom")

                .profile_files(

                    #[allow(deprecated)]

                    ProfileFiles::builder()

                        .with_contents(

                            #[allow(deprecated)]

                            ProfileFileKind::Config,

                            "[profile custom]\nendpoint_url = http://localhost:8989",

                        )

                        .build(),

                )

                .endpoint_url("http://localhost:1111")

                .load()

                .await;

            assert_eq!(Origin::shared_config(), loader.get_origin("endpoint_url"));

        }



        #[tokio::test]

        async fn test_origin_env() {

            let _ = tracing_subscriber::fmt::try_init();

            let env = Env::from_slice(&[("AWS_ENDPOINT_URL", "http://localhost:7878")]);

            let loader = base_conf()

                .test_credentials()

                .env(env)

                .profile_name("custom")

                .profile_files(

                    #[allow(deprecated)]

                    ProfileFiles::builder()

                        .with_contents(

                            #[allow(deprecated)]

                            ProfileFileKind::Config,

                            "[profile custom]\nendpoint_url = http://localhost:8989",

                        )

                        .build(),

                )

                .load()

                .await;

            assert_eq!(

                Origin::shared_environment_variable(),

                loader.get_origin("endpoint_url")

            );

        }



        #[tokio::test]

        async fn test_origin_fs() {

            let _ = tracing_subscriber::fmt::try_init();

            let loader = base_conf()

                .test_credentials()

                .profile_name("custom")

                .profile_files(

                    #[allow(deprecated)]

                    ProfileFiles::builder()

                        .with_contents(

                            #[allow(deprecated)]

                            ProfileFileKind::Config,

                            "[profile custom]\nendpoint_url = http://localhost:8989",

                        )

                        .build(),

                )

                .load()

                .await;

            assert_eq!(

                Origin::shared_profile_file(),

                loader.get_origin("endpoint_url")

            );

        }



        #[tokio::test]

        async fn load_fips() {

            let conf = base_conf().use_fips(true).load().await;

aws/rust-runtime/aws-runtime/Cargo.toml

+1 −1
Original line number Diff line number Diff line 
[package]

name = "aws-runtime"

version = "1.1.9"

version = "1.2.0"

authors = ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>"]

description = "Runtime support code for the AWS SDK. This crate isn't intended to be used directly."

edition = "2021"

aws/rust-runtime/aws-runtime/src/env_config.rs

+60 −7
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 


use crate::env_config::property::PropertiesKey;

use crate::env_config::section::EnvConfigSections;

use aws_types::origin::Origin;

use aws_types::os_shim_internal::Env;

use aws_types::service_config::ServiceConfigKey;

use std::borrow::Cow;
@@ -81,6 +82,20 @@ pub struct EnvConfigSource<'a> { 
    scope: Scope<'a>,

}



#[allow(clippy::from_over_into)]

impl Into<Origin> for &EnvConfigSource<'_> {

    fn into(self) -> Origin {

        match (&self.scope, &self.location) {

            (Scope::Global, Location::Environment) => Origin::shared_environment_variable(),

            (Scope::Global, Location::Profile { .. }) => Origin::shared_profile_file(),

            (Scope::Service { .. }, Location::Environment) => {

                Origin::service_environment_variable()

            }

            (Scope::Service { .. }, Location::Profile { .. }) => Origin::service_profile_file(),

        }

    }

}



impl<'a> EnvConfigSource<'a> {

    pub(crate) fn global_from_env(key: Cow<'a, str>) -> Self {

        Self {
@@ -186,7 +201,7 @@ impl<'a> EnvConfigValue<'a> { 
        self

    }



    /// Load the value from `provider_config`, validating with `validator`

    /// Load the value from the env or profile files, validating with `validator`

    pub fn validate<T, E: Error + Send + Sync + 'static>(

        self,

        env: &Env,
@@ -204,6 +219,30 @@ impl<'a> EnvConfigValue<'a> { 
            .transpose()

    }



    /// Load the value from the env or profile files, validating with `validator`

    ///

    /// This version of the function will also return the origin of the config.

    pub fn validate_and_return_origin<T, E: Error + Send + Sync + 'static>(

        self,

        env: &Env,

        profiles: Option<&EnvConfigSections>,

        validator: impl Fn(&str) -> Result<T, E>,

    ) -> Result<(Option<T>, Origin), EnvConfigError<E>> {

        let value = self.load(env, profiles);

        match value {

            Some((v, ctx)) => {

                let origin: Origin = (&ctx).into();

                validator(v.as_ref())

                    .map_err(|err| EnvConfigError {

                        property_source: format!("{}", ctx),

                        err,

                    })

                    .map(|value| (Some(value), origin))

            }

            None => Ok((None, Origin::unknown())),

        }

    }



    /// Load the value from the environment

    pub fn load(

        &self,
@@ -222,9 +261,16 @@ impl<'a> EnvConfigValue<'a> { 
                )

            });



            let value = service_config.or(global_config);

            tracing::trace!("ENV value = {value:?}");

            value

            if let Some(v) = service_config {

                tracing::trace!("(service env) {env_var} = {v:?}");

                Some(v)

            } else if let Some(v) = global_config {

                tracing::trace!("(global env) {env_var} = {v:?}");

                Some(v)

            } else {

                tracing::trace!("(env) no value set for {env_var}");

                None

            }

        });



        let profile_value = match (profiles, self.profile_key.as_ref()) {
@@ -245,9 +291,16 @@ impl<'a> EnvConfigValue<'a> { 
                    )

                });



                let value = service_config.or(global_config);

                tracing::trace!("PROFILE value = {value:?}");

                value

                if let Some(v) = service_config {

                    tracing::trace!("(service profile) {profile_key} = {v:?}");

                    Some(v)

                } else if let Some(v) = global_config {

                    tracing::trace!("(global profile) {profile_key} = {v:?}");

                    Some(v)

                } else {

                    tracing::trace!("(service profile) no value set for {profile_key}");

                    None

                }

            }

            _ => None,

        };