Unverified Commit 2f06f30a authored by Landon James's avatar Landon James Committed by Falk Woldmann
Browse files

Updates for `PREVIEW` builds (#4108) 

## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
See internal notes

## Description
<!--- Describe your changes in detail -->

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->
----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent 0cc6f3b4

tools/ci-build/publisher/src/subcommand/fix_manifests.rs

+19 −38
Original line number Diff line number Diff line
@@ -11,13 +11,14 @@ 


use crate::fs::Fs;

use crate::package::discover_manifests;

use crate::SDK_REPO_NAME;

use anyhow::{bail, Context, Result};

use clap::Parser;

use semver::Version;

use smithy_rs_tool_common::{ci::running_in_ci, package::parse_version};

use smithy_rs_tool_common::{

    ci::{is_in_example_dir, is_preview_build, running_in_ci},

    package::parse_version,

};

use std::collections::BTreeMap;

use std::ffi::OsStr;

use std::path::{Path, PathBuf};

use toml::value::Table;

use toml::Value;
@@ -64,6 +65,7 @@ pub async fn subcommand_fix_manifests( 
    Ok(())

}



#[derive(Debug)]

struct Manifest {

    path: PathBuf,

    metadata: toml::Value,
@@ -82,6 +84,7 @@ impl Manifest { 
    }

}



#[derive(Debug)]

struct Versions(BTreeMap<String, VersionWithMetadata>);

#[derive(Copy, Clone)]

enum FilterType {
@@ -110,6 +113,7 @@ impl Versions { 
    }

}



#[derive(Debug)]

struct VersionWithMetadata {

    version: Version,

    publish: bool,
@@ -205,32 +209,19 @@ fn fix_dep_sets(versions: &VersionView, metadata: &mut toml::Value) -> Result<us 
    Ok(changed)

}



fn is_example_manifest(manifest_path: impl AsRef<Path>) -> bool {

    // Examine parent directories until either `examples/` or `aws-sdk-rust/` is found

    let mut path = manifest_path.as_ref();

    while let Some(parent) = path.parent() {

        path = parent;

        if path.file_name() == Some(OsStr::new("examples")) {

            return true;

        } else if path.file_name() == Some(OsStr::new(SDK_REPO_NAME)) {

            break;

        }

    }

    false

}



fn conditionally_disallow_publish(

    manifest_path: &Path,

    metadata: &mut toml::Value,

) -> Result<bool> {

    let is_github_actions = running_in_ci();

    let is_example = is_example_manifest(manifest_path);

    let is_gh_action_or_smithy_rs_docker = running_in_ci();

    let is_example = is_in_example_dir(manifest_path);

    let is_preview_build = is_preview_build();



    // Safe-guard to prevent accidental publish to crates.io. Add some friction

    // to publishing from a local development machine by detecting that the tool

    // is not being run from CI, and disallow publish in that case. Also disallow

    // publishing of examples.

    if !is_github_actions || is_example {

    // publishing of examples and Trebuchet preview builds.

    if !is_gh_action_or_smithy_rs_docker || is_example || is_preview_build {

        if let Some(value) = set_publish_false(manifest_path, metadata, is_example) {

            return Ok(value);

        }
@@ -293,6 +284,13 @@ async fn fix_manifests( 
}



fn fix_manifest(versions: &Versions, manifest: &mut Manifest) -> Result<usize> {

    // In the case of a preview build we do not update the examples manifests

    // since most SDKs will not be generated so the particular crate referred to

    // by an example is unlikely to exist

    if is_in_example_dir(&manifest.path) && is_preview_build() {

        debug!(package = ?&manifest.path, "Skipping example package for preview build");

        return Ok(0);

    }

    let mut view = versions.published();

    if !manifest.publish()? {

        debug!(package = ?&manifest.path, "package has publishing disabled, allowing unpublished crates to be used");
@@ -428,21 +426,4 @@ mod tests { 
            actual_build_deps.to_string()

        );

    }



    #[test]

    fn test_is_example_manifest() {

        assert!(!is_example_manifest("aws-sdk-rust/sdk/s3/Cargo.toml"));

        assert!(!is_example_manifest(

            "aws-sdk-rust/sdk/aws-config/Cargo.toml"

        ));

        assert!(!is_example_manifest(

            "/path/to/aws-sdk-rust/sdk/aws-config/Cargo.toml"

        ));

        assert!(!is_example_manifest("sdk/aws-config/Cargo.toml"));

        assert!(is_example_manifest("examples/foo/Cargo.toml"));

        assert!(is_example_manifest("examples/foo/bar/Cargo.toml"));

        assert!(is_example_manifest(

            "aws-sdk-rust/examples/foo/bar/Cargo.toml"

        ));

    }

}

tools/ci-build/publisher/src/subcommand/generate_version_manifest.rs

+12 −2
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ use anyhow::{bail, Context, Result}; 
use clap::Parser;

use semver::Version;

use serde::Deserialize;

use smithy_rs_tool_common::ci::is_preview_build;

use smithy_rs_tool_common::git::{find_git_repository_root, Git, GitCLI};

use smithy_rs_tool_common::package::PackageCategory;

use smithy_rs_tool_common::shell;
@@ -113,8 +114,10 @@ pub async fn subcommand_generate_version_manifest( 
        crates,

        release: None,

    };



    versions_manifest.release =

        generate_release_metadata(&versions_manifest, previous_release_versions)?;



    let manifest_file_name = output_location.join("versions.toml");

    info!("Writing {:?}...", manifest_file_name);

    versions_manifest.write_to_file(&manifest_file_name)?;
@@ -125,14 +128,21 @@ fn generate_release_metadata( 
    versions_manifest: &VersionsManifest,

    maybe_previous_release_versions: &Option<PathBuf>,

) -> Result<Option<Release>> {

    if let Some(previous_release_versions) = maybe_previous_release_versions {

    // For release builds we generate a real release section

    if let (Some(previous_release_versions), false) =

        (maybe_previous_release_versions, is_preview_build())

    {

        let old_versions = VersionsManifest::from_file(previous_release_versions)?;

        Ok(Some(Release {

            tag: None,

            crates: find_released_versions(&old_versions, versions_manifest)?,

        }))

    // For preview builds we insert an empty section

    } else {

        Ok(None)

        Ok(Some(Release {

            tag: None,

            crates: BTreeMap::new(),

        }))

    }

}

tools/ci-build/publisher/src/subcommand/tag_versions_manifest.rs

+1 −0
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ pub struct TagVersionsManifestArgs { 
pub fn subcommand_tag_versions_manifest(

    TagVersionsManifestArgs { manifest_path, tag }: &TagVersionsManifestArgs,

) -> Result<()> {

    println!("Tagging manifest at: {manifest_path:#?}");

    let mut manifest = VersionsManifest::from_file(manifest_path)?;

    if let Some(release) = manifest.release.as_mut() {

        release.tag = Some(tag.to_string());

tools/ci-build/sdk-versioner/src/main.rs

+9 −1
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 


use anyhow::{bail, Context, Result};

use clap::Parser;

use smithy_rs_tool_common::ci::{is_in_example_dir, is_preview_build};

use smithy_rs_tool_common::package::{PackageCategory, SDK_PREFIX};

use smithy_rs_tool_common::versions_manifest::VersionsManifest;

use std::ffi::OsStr;
@@ -114,6 +115,13 @@ fn main() -> Result<()> { 
        },

    };



    // In the case of a preview build we avoid updating the examples directories since

    // we only generate the single preview SDK, so most SDKs referred to in the examples

    // will be missing

    if is_preview_build() && is_in_example_dir(&args.crate_paths()[0]) {

        return Ok(());

    }



    let start_time = Instant::now();

    let mut manifest_paths = Vec::new();

    for crate_path in args.crate_paths() {
@@ -236,7 +244,7 @@ fn updated_dependency_value( 
            let dependency_path = sdk_path

                .join(crate_path_name(dependency_name))

                .canonicalize()

                .context("failed to canonicalize dependency path")?;

                .context(format!("failed to canonicalize sdk_path: {sdk_path:#?} with dependency_name: {dependency_name}"))?;

            if let Some(relative_path) = pathdiff::diff_paths(&dependency_path, &crate_path) {

                value["path"] = toml_edit::value(

                    relative_path

tools/ci-build/smithy-rs-tool-common/src/ci.rs

+55 −0
Original line number Diff line number Diff line
@@ -2,9 +2,64 @@ 
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */

use std::ffi::OsStr;

use std::path::Path;



/// Returns `true` if this code is being run in CI

pub fn running_in_ci() -> bool {

    std::env::var("GITHUB_ACTIONS").unwrap_or_default() == "true"

        || std::env::var("SMITHY_RS_DOCKER_BUILD_IMAGE").unwrap_or_default() == "1"

}



/// The `BUILD_TYPE` env var is only set for Codebuild jobs, will always

/// return `false` in other CI environments

pub fn is_preview_build() -> bool {

    let build_type = std::env::var("BUILD_TYPE");



    if let Ok(build_type) = build_type {

        if build_type.eq_ignore_ascii_case("PREVIEW") {

            return true;

        }

    }



    false

}



pub fn is_in_example_dir(manifest_path: impl AsRef<Path>) -> bool {

    let mut path = manifest_path.as_ref();

    // Check if current dir is examples

    if path.ends_with("examples") {

        return true;

    }

    // Examine parent directories until either `examples/` or `aws-sdk-rust/` is found

    while let Some(parent) = path.parent() {

        path = parent;

        if path.file_name() == Some(OsStr::new("examples")) {

            return true;

        } else if path.file_name() == Some(OsStr::new("aws-sdk-rust")) {

            break;

        }

    }

    false

}



#[cfg(test)]

mod tests {

    use super::*;



    #[test]

    fn test_is_in_example_dir() {

        assert!(!is_in_example_dir("aws-sdk-rust/sdk/s3/Cargo.toml"));

        assert!(!is_in_example_dir("aws-sdk-rust/sdk/aws-config/Cargo.toml"));

        assert!(!is_in_example_dir(

            "/path/to/aws-sdk-rust/sdk/aws-config/Cargo.toml"

        ));

        assert!(!is_in_example_dir("sdk/aws-config/Cargo.toml"));

        assert!(is_in_example_dir("examples/foo/Cargo.toml"));

        assert!(is_in_example_dir("examples/foo/bar/Cargo.toml"));

        assert!(is_in_example_dir(

            "aws-sdk-rust/examples/foo/bar/Cargo.toml"

        ));

        assert!(is_in_example_dir("aws-sdk-rust/examples/"));

    }

}