Add support for parsing profiles (#600) (8aabc0fb) · Commits · Public Repositories / Smithy Rust

aws/rust-runtime/aws-types/Cargo.toml

+1 −0

Original line number	Diff line number	Diff line
		@@ -15,6 +15,7 @@ tracing = "0.1"
		tracing-test = "0.1.0"
		serde = { version = "1", features = ["derive"]}
		serde_json = "1"
		arbitrary = "1"

		[build-dependencies]
		rustc_version = "0.3.3"

aws/rust-runtime/aws-types/fuzz/.gitignore

0 → 100644

+4 −0

Original line number	Diff line number	Diff line

		target
		corpus
		artifacts

aws/rust-runtime/aws-types/fuzz/Cargo.toml

0 → 100644

+26 −0

Original line number	Diff line number	Diff line

		[package]
		name = "aws-types-fuzz"
		version = "0.0.0"
		authors = ["Automatically generated"]
		publish = false
		edition = "2018"

		[package.metadata]
		cargo-fuzz = true

		[dependencies]
		libfuzzer-sys = "0.4"

		[dependencies.aws-types]
		path = ".."

		# Prevent this from interfering with workspaces
		[workspace]
		members = ["."]

		[[bin]]
		name = "profile-parser"
		path = "fuzz_targets/profile-parser.rs"
		test = false
		doc = false

aws/rust-runtime/aws-types/fuzz/fuzz_targets/profile-parser.rs

0 → 100644

+26 −0

Original line number	Diff line number	Diff line
		#![no_main]
		use aws_types::os_shim_internal::{Env, Fs};
		use aws_types::profile;
		use libfuzzer_sys::fuzz_target;
		use std::collections::HashMap;
		use std::ffi::OsString;

		// Fuzz on a tuple of (`config`, `credentials`)
		fuzz_target!(\|data: (Option<&str>, Option<&str>)\| {
		let mut fs: HashMap<OsString, Vec<u8>> = HashMap::new();
		if let Some(config) = data.0 {
		fs.insert(
		"~/.aws/config".to_string().into(),
		config.to_string().into(),
		);
		}
		if let Some(creds) = data.1 {
		fs.insert(
		"~/.aws/credentials".to_string().into(),
		creds.to_string().into(),
		);
		}
		let fs = Fs::from_raw_map(fs);
		let env = Env::real();
		let _ = profile::load(&fs, &env);
		});

aws/rust-runtime/aws-types/src/profile.rs

+295 −4

Original line number	Diff line number	Diff line
		@@ -3,10 +3,301 @@
		* SPDX-License-Identifier: Apache-2.0.
		*/

		mod normalize;
		mod parse;
		mod source;

		// exposed only to remove unused code warnings until the parser side is added
		#[doc(hidden)]
		pub use source::load;
		#[doc(hidden)]
		pub use source::Source;
		use crate::os_shim_internal::{Env, Fs};
		use crate::profile::parse::parse_profile_file;
		pub use crate::profile::parse::ProfileParseError;
		use crate::profile::source::{FileKind, Source};
		use std::borrow::Cow;
		use std::collections::HashMap;

		/// Read & parse AWS config files
		///
		/// Loads and parses profile files according to the spec:
		///
		/// ## Location of Profile Files
		/// * The location of the config file will be loaded from `$AWS_CONFIG_FILE` with a fallback to
		/// `~/.aws/config`
		/// * The location of the credentials file will be loaded from `$AWS_SHARED_CREDENTIALS_FILE` with a
		/// fallback to `~/.aws/credentials`
		///
		/// ## Home directory resolution
		/// Home directory resolution is implemented to match the behavior of the CLI & Python. `~` is only
		/// used for home directory resolution when it:
		/// - Starts the path
		/// - Is followed immediately by `/` or a platform specific separator. (On windows, `~/` and `~\` both
		/// resolve to the home directory.
		///
		/// When determining the home directory, the following environment variables are checked:
		/// - `$HOME` on all platforms
		/// - `$USERPROFILE` on Windows
		/// - `$HOMEDRIVE$HOMEPATH` on Windows
		///
		/// ## Profile file syntax
		///
		/// Profile files have a general form similar to INI but with a number of quirks and edge cases. These
		/// behaviors are largely to match existing parser implementations and these cases are documented in `test-data/profile-parser-tests.json`
		/// in this repo.
		///
		/// ### The config file `~/.aws/config`
		/// ```ini
		/// # ~/.aws/config
		/// [profile default]
		/// key = value
		///
		/// # profiles must begin with `profile`
		/// [profile other]
		/// key = value2
		/// ```
		///
		/// ### The credentials file `~/.aws/credentials`
		/// The main difference is that in ~/.aws/credentials, profiles MUST NOT be prefixed with profile:
		/// ```ini
		/// [default]
		/// aws_access_key_id = 123
		///
		/// [other]
		/// aws_access_key_id = 456
		/// ```
		pub fn load(fs: &Fs, env: &Env) -> Result<ProfileSet, ProfileParseError> {
		let source = source::load(&env, &fs);
		ProfileSet::parse(source)
		}

		/// A top-level configuration source containing multiple named profiles
		#[derive(Debug, Eq, Clone, PartialEq)]
		pub struct ProfileSet {
		profiles: HashMap<String, Profile>,
		selected_profile: Cow<'static, str>,
		}

		impl ProfileSet {
		fn parse(source: Source) -> Result<Self, ProfileParseError> {
		let mut base = ProfileSet::empty();
		base.selected_profile = source.profile;

		normalize::merge_in(
		&mut base,
		parse_profile_file(&source.config_file)?,
		FileKind::Config,
		);
		normalize::merge_in(
		&mut base,
		parse_profile_file(&source.credentials_file)?,
		FileKind::Credentials,
		);
		Ok(base)
		}

		fn empty() -> Self {
		Self {
		profiles: Default::default(),
		selected_profile: "default".into(),
		}
		}

		/// Retrieves a key-value pair from the currently selected profile
		pub fn get(&self, key: &str) -> Option<&Property> {
		self.profiles
		.get(self.selected_profile.as_ref())
		.and_then(\|profile\| profile.get(key))
		}

		/// Retrieve a named profile from the profile set
		pub fn get_profile(&self, profile_name: &str) -> Option<&Profile> {
		self.profiles.get(profile_name)
		}
		}

		/// An individual configuration profile
		///
		/// An AWS config may be composed of a multiple named profiles within a [`ProfileSet`](ProfileSet)
		#[derive(Debug, Clone, Eq, PartialEq)]
		pub struct Profile {
		name: String,
		properties: HashMap<String, Property>,
		}

		impl Profile {
		pub fn new(name: String, properties: HashMap<String, Property>) -> Self {
		Self { name, properties }
		}

		pub fn get(&self, name: &str) -> Option<&Property> {
		self.properties.get(name)
		}
		}

		/// Key-Value property pair
		#[derive(Debug, Clone, Eq, PartialEq)]
		pub struct Property {
		key: String,
		value: String,
		}

		impl Property {
		pub fn value(&self) -> &str {
		&self.value
		}

		pub fn key(&self) -> &str {
		&self.key
		}

		pub fn new(key: String, value: String) -> Self {
		Property { key, value }
		}
		}

		#[cfg(test)]
		mod test {
		use crate::profile::source::{File, Source};
		use crate::profile::ProfileSet;
		use arbitrary::{Arbitrary, Unstructured};
		use serde::Deserialize;
		use std::collections::HashMap;
		use std::error::Error;
		use std::fs;
		use tracing_test::traced_test;

		/// Run all tests from profile-parser-tests.json
		///
		/// These represent the bulk of the test cases and reach effectively 100% coverage
		#[test]
		#[traced_test]
		fn run_tests() -> Result<(), Box<dyn Error>> {
		let tests = fs::read_to_string("test-data/profile-parser-tests.json")?;
		let tests: ParserTests = serde_json::from_str(&tests)?;
		for (i, test) in tests.tests.into_iter().enumerate() {
		eprintln!("test: {}", i);
		check(test);
		}
		Ok(())
		}

		/// Run all tests from the fuzzing corpus to validate coverage
		#[test]
		#[ignore]
		fn run_fuzz_tests() -> Result<(), Box<dyn Error>> {
		let fuzz_corpus = fs::read_dir("fuzz/corpus/profile-parser")?
		.map(\|res\| res.map(\|entry\| entry.path()))
		.collect::<Result<Vec<_>, _>>()?;
		for file in fuzz_corpus {
		let raw = fs::read(file)?;
		let mut unstructured = Unstructured::new(&raw);
		let (conf, creds): (Option<&str>, Option<&str>) =
		Arbitrary::arbitrary(&mut unstructured)?;
		let profile_source = Source {
		config_file: File {
		path: "~/.aws/config".to_string(),
		contents: conf.unwrap_or_default().to_string(),
		},
		credentials_file: File {
		path: "~/.aws/config".to_string(),
		contents: creds.unwrap_or_default().to_string(),
		},
		profile: "default".into(),
		};
		// don't care if parse fails, just don't panic
		let _ = ProfileSet::parse(profile_source);
		}

		Ok(())
		}

		// for test comparison purposes, flatten a profile into a hashmap
		fn flatten(profile: ProfileSet) -> HashMap<String, HashMap<String, String>> {
		profile
		.profiles
		.into_iter()
		.map(\|(_name, profile)\| {
		(
		profile.name,
		profile
		.properties
		.into_iter()
		.map(\|(_, prop)\| (prop.key, prop.value))
		.collect(),
		)
		})
		.collect()
		}

		fn make_source(input: ParserInput) -> Source {
		Source {
		config_file: File {
		path: "~/.aws/config".to_string(),
		contents: input.config_file.unwrap_or_default(),
		},
		credentials_file: File {
		path: "~/.aws/credentials".to_string(),
		contents: input.credentials_file.unwrap_or_default(),
		},
		profile: Default::default(),
		}
		}

		// wrapper to generate nicer errors during test failure
		fn check(test_case: ParserTest) {
		let copy = test_case.clone();
		let parsed = ProfileSet::parse(make_source(test_case.input));
		let res = match (parsed.map(flatten), &test_case.output) {
		(Ok(actual), ParserOutput::Profiles(expected)) if &actual != expected => Err(format!(
		"mismatch:\nExpected: {:#?}\nActual: {:#?}",
		expected, actual
		)),
		(Ok(_), ParserOutput::Profiles(_)) => Ok(()),
		(Err(msg), ParserOutput::ErrorContaining(substr)) => {
		if format!("{}", msg).contains(substr) {
		Ok(())
		} else {
		Err(format!("Expected {} to contain {}", msg, substr))
		}
		}
		(Ok(output), ParserOutput::ErrorContaining(err)) => Err(format!(
		"expected an error: {} but parse succeeded:\n{:#?}",
		err, output
		)),
		(Err(err), ParserOutput::Profiles(_expected)) => {
		Err(format!("Expected to succeed but got: {}", err))
		}
		};
		if let Err(e) = res {
		eprintln!("Test case failed: {:#?}", copy);
		eprintln!("failure: {}", e);
		panic!("test failed")
		}
		}

		#[derive(Deserialize, Debug)]
		#[serde(rename_all = "camelCase")]
		struct ParserTests {
		tests: Vec<ParserTest>,
		}

		#[derive(Deserialize, Debug, Clone)]
		#[serde(rename_all = "camelCase")]
		struct ParserTest {
		name: String,
		input: ParserInput,
		output: ParserOutput,
		}

		#[derive(Deserialize, Debug, Clone)]
		#[serde(rename_all = "camelCase")]
		enum ParserOutput {
		Profiles(HashMap<String, HashMap<String, String>>),
		ErrorContaining(String),
		}

		#[derive(Deserialize, Debug, Clone)]
		#[serde(rename_all = "camelCase")]
		struct ParserInput {
		config_file: Option<String>,
		credentials_file: Option<String>,
		}
		}