Unverified Commit ada03d4c authored by Russell Cohen's avatar Russell Cohen Committed by GitHub
Browse files

Generate a Docs Landing Page (#853) 

* Generate a Docs Landing Page

This commit refactors a portion of the build into `buildSrc` and adds a job to generate a docs landing page. The generated SDK now contains a `docs.md` that contains top level links to all services.

* Add copyright headers

* Order by human name
parent d742a699

aws/rust-runtime/aws-config/fuzz/Cargo.toml

+4 −1
Original line number Diff line number Diff line
@@ -11,9 +11,12 @@ cargo-fuzz = true 


[dependencies]

libfuzzer-sys = "0.4"

aws-config = { path = ".." }



[dependencies.aws-types]

path = ".."

path = "../../../sdk/build/aws-sdk/sdk/aws-types"







# Prevent this from interfering with workspaces

[workspace]

aws/sdk/build.gradle.kts

+21 −116
Original line number Diff line number Diff line
@@ -3,12 +3,7 @@ 
 * SPDX-License-Identifier: Apache-2.0.

 */



import software.amazon.smithy.aws.traits.ServiceTrait

import software.amazon.smithy.model.Model

import software.amazon.smithy.model.shapes.ServiceShape

import software.amazon.smithy.model.traits.TitleTrait

import java.util.Properties

import kotlin.streams.toList



extra["displayName"] = "Smithy :: Rust :: AWS-SDK"

extra["moduleName"] = "software.amazon.smithy.rust.awssdk"
@@ -83,52 +78,17 @@ fun getProperty(name: String): String? { 
}



// Class and functions for service and protocol membership for SDK generation

data class Membership(val inclusions: Set<String> = emptySet(), val exclusions: Set<String> = emptySet())



fun Membership.isMember(member: String): Boolean = when {

    exclusions.contains(member) -> false

    inclusions.contains(member) -> true

    inclusions.isEmpty() -> true

    else -> false

}



fun parseMembership(rawList: String): Membership {

    val inclusions = mutableSetOf<String>()

    val exclusions = mutableSetOf<String>()



    rawList.split(",").map { it.trim() }.forEach { item ->

        when {

            item.startsWith('-') -> exclusions.add(item.substring(1))

            item.startsWith('+') -> inclusions.add(item.substring(1))

            else -> error("Must specify inclusion (+) or exclusion (-) prefix character to $item.")

        }

    }



    val conflictingMembers = inclusions.intersect(exclusions)

    require(conflictingMembers.isEmpty()) { "$conflictingMembers specified both for inclusion and exclusion in $rawList" }



    return Membership(inclusions, exclusions)

}



data class AwsService(

    val service: String,

    val module: String,

    val moduleDescription: String,

    val modelFile: File,

    val extraConfig: String? = null,

    val extraFiles: List<File>

) {

    fun files(): List<File> = listOf(modelFile) + extraFiles

}



val awsServices: List<AwsService> by lazy { discoverServices() }

val awsServices: List<AwsService> by lazy { discoverServices(loadServiceMembership()) }

val eventStreamAllowList: Set<String> by lazy { eventStreamAllowList() }



fun loadServiceMembership(): Membership {

    val membershipOverride = getProperty("aws.services")?.let { parseMembership(it) }

    println(membershipOverride)

    val fullSdk = parseMembership(getProperty("aws.services.fullsdk") ?: throw kotlin.Exception("never list missing"))

    val tier1 = parseMembership(getProperty("aws.services.smoketest") ?: throw kotlin.Exception("smoketest list missing"))

    val fullSdk =

        parseMembership(getProperty("aws.services.fullsdk") ?: throw kotlin.Exception("full sdk list missing"))

    val tier1 =

        parseMembership(getProperty("aws.services.smoketest") ?: throw kotlin.Exception("smoketest list missing"))

    return membershipOverride ?: if ((getProperty("aws.fullsdk") ?: "") == "true") {

        fullSdk

    } else {
@@ -136,78 +96,13 @@ fun loadServiceMembership(): Membership { 
    }

}



/**

 * Discovers services from the `models` directory

 *

 * Do not invoke this function directly. Use the `awsServices` provider.

 */

fun discoverServices(): List<AwsService> {

    val models = project.file("aws-models")

    val serviceMembership = loadServiceMembership()

    val baseServices = fileTree(models)

        .sortedBy { file -> file.name }

        .mapNotNull { file ->

            val model = Model.assembler().addImport(file.absolutePath).assemble().result.get()

            val services: List<ServiceShape> = model.shapes(ServiceShape::class.java).sorted().toList()

            if (services.size > 1) {

                throw Exception("There must be exactly one service in each aws model file")

            }

            if (services.isEmpty()) {

                logger.info("${file.name} has no services")

                null

            } else {

                val service = services[0]

                val title = service.expectTrait(TitleTrait::class.java).value

                val sdkId = service.expectTrait(ServiceTrait::class.java).sdkId

                    .toLowerCase()

                    .replace(" ", "")

                    // TODO: the smithy models should not include the suffix "service"

                    .removeSuffix("service")

                    .removeSuffix("api")

                val testFile = file.parentFile.resolve("$sdkId-tests.smithy")

                val extras = if (testFile.exists()) {

                    logger.warn("Discovered protocol tests for ${file.name}")

                    listOf(testFile)

                } else {

                    listOf()

                }

                AwsService(

                    service = service.id.toString(),

                    module = sdkId,

                    moduleDescription = "AWS SDK for $title",

                    modelFile = file,

                    extraFiles = extras

                )

            }

        }

    val baseModules = baseServices.map { it.module }.toSet()



    // validate the full exclusion list hits

    serviceMembership.exclusions.forEach { disabledService ->

        check(baseModules.contains(disabledService)) {

            "Service $disabledService was explicitly disabled but no service was generated with that name. Generated:\n ${

            baseModules.joinToString(

                "\n "

            )

            }"

        }

    }

    // validate inclusion list hits

    serviceMembership.inclusions.forEach { service ->

        check(baseModules.contains(service)) { "Service $service was in explicit inclusion list but not generated!" }

    }

    return baseServices.filter {

        serviceMembership.isMember(it.module)

    }

}



fun eventStreamAllowList(): Set<String> {

    val list = getProperty("aws.services.eventstream.allowlist") ?: ""

    return list.split(",").map { it.trim() }.toSet()

}



fun generateSmithyBuild(services: List<AwsService>): String {

    val projections = services.joinToString(",\n") { service ->

    val serviceProjections = services.map { service ->

        val files = service.files().map { extraFile ->

            software.amazon.smithy.utils.StringUtils.escapeJavaString(

                extraFile.absolutePath,
@@ -243,6 +138,7 @@ fun generateSmithyBuild(services: List<AwsService>): String { 
            }

        """.trimIndent()

    }

    val projections = serviceProjections.joinToString(",\n")

    return """

    {

        "version": "1.0",
@@ -263,6 +159,14 @@ task("generateSmithyBuild") { 
    }

}



task("generateDocs") {

    inputs.property("servicelist", awsServices.sortedBy { it.module }.toString())

    outputs.file(outputDir.resolve("docs.md"))

    doLast {

        project.docsLandingPage(awsServices, outputDir)

    }

}



task("relocateServices") {

    description = "relocate AWS services to their final destination"

    doLast {
@@ -413,7 +317,8 @@ task("finalizeSdk") { 
        "relocateServices",

        "relocateRuntime",

        "relocateAwsRuntime",

        "relocateExamples"

        "relocateExamples",

        "generateDocs"

    )

}

buildSrc/build.gradle.kts

0 → 100644
+36 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0.

 */



import java.util.Properties



plugins {

    `kotlin-dsl`

}

repositories {

    maven("https://plugins.gradle.org/m2")

}



// Load properties manually to avoid hard coding smithy version

val props = Properties().apply {

    file("../gradle.properties").inputStream().use { load(it) }

}



val smithyVersion = props["smithyVersion"]



buildscript {

    repositories {

        mavenCentral()

    }

}



dependencies {

    api("software.amazon.smithy:smithy-codegen-core:$smithyVersion")

    implementation("software.amazon.smithy:smithy-utils:$smithyVersion")

    implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion")

    implementation("software.amazon.smithy:smithy-aws-traits:$smithyVersion")

    implementation("software.amazon.smithy:smithy-aws-iam-traits:$smithyVersion")

    implementation("software.amazon.smithy:smithy-aws-cloudformation-traits:$smithyVersion")

    implementation(gradleApi())

}

buildSrc/src/main/kotlin/DocsLandingPage.kt

0 → 100644
+58 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0.

 */



import org.gradle.api.Project

import software.amazon.smithy.utils.CodeWriter

import java.io.File



/**

 * Generate a basic docs landing page into [outputDir]

 *

 * The generated docs will include links to crates.io, docs.rs and GitHub examples for all generated services. The generated docs will

 * be written to `docs.md` in the provided [outputDir].

 */

fun Project.docsLandingPage(awsServices: List<AwsService>, outputDir: File) {

    val project = this

    val writer = CodeWriter()

    with(writer) {

        write("# AWS SDK for Rust")

        write(

            """The AWS SDK for Rust contains one crate for each AWS service, as well as ${docsRs("aws-config")},

            |a crate implementing configuration loading such as credential providers.""".trimMargin()

        )



        writer.write("## AWS Services")

        /* generate a basic markdown table */

        writer.write("| Service | [docs.rs](https://docs.rs) | [crates.io](https://crates.io) | [Usage Examples](https://github.com/awslabs/aws-sdk-rust/tree/main/examples/) |")

        writer.write("| ------- | ------- | --------- | ------ |")

        awsServices.sortedBy { it.humanName }.forEach {

            writer.write(

                "| ${it.humanName} | ${docsRs(it)} | ${cratesIo(it)} | ${

                examples(

                    it,

                    project

                )

                }"

            )

        }

    }

    outputDir.resolve("docs.md").writeText(writer.toString())

}



/**

 * Generate a link to the examples for a given service

 */

private fun examples(service: AwsService, project: Project) = if (with(service) { project.examples() }) {

    "[Link](https://github.com/awslabs/aws-sdk-rust/tree/main/examples/${service.module})"

} else {

    "None yet!"

}



/**

 * Generate a link to the docs

 */

private fun docsRs(service: AwsService) = docsRs(service.crate())

private fun docsRs(crate: String) = "[$crate](https://docs.rs/$crate)"

private fun cratesIo(service: AwsService) = "[${service.crate()}](https://crates.io/crates/${service.crate()})"

buildSrc/src/main/kotlin/ServiceLoader.kt

0 → 100644
+120 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0.

 */



import org.gradle.api.Project

import software.amazon.smithy.aws.traits.ServiceTrait

import software.amazon.smithy.model.Model

import software.amazon.smithy.model.shapes.ServiceShape

import software.amazon.smithy.model.traits.TitleTrait

import java.io.File

import kotlin.streams.toList



/**

 * Discovers services from the `aws-models` directory within the project.

 *

 * Since this function parses all models, it is relatively expensive to call. The result should be cached in a property

 * during build.

 */

fun Project.discoverServices(serviceMembership: Membership): List<AwsService> {

    val models = project.file("aws-models")

    val baseServices = fileTree(models)

        .sortedBy { file -> file.name }

        .mapNotNull { file ->

            val model = Model.assembler().addImport(file.absolutePath).assemble().result.get()

            val services: List<ServiceShape> = model.shapes(ServiceShape::class.java).sorted().toList()

            if (services.size > 1) {

                throw Exception("There must be exactly one service in each aws model file")

            }

            if (services.isEmpty()) {

                logger.info("${file.name} has no services")

                null

            } else {

                val service = services[0]

                val title = service.expectTrait(TitleTrait::class.java).value

                val sdkId = service.expectTrait(ServiceTrait::class.java).sdkId

                    .toLowerCase()

                    .replace(" ", "")

                    // TODO: the smithy models should not include the suffix "service"

                    .removeSuffix("service")

                    .removeSuffix("api")

                val testFile = file.parentFile.resolve("$sdkId-tests.smithy")

                val extras = if (testFile.exists()) {

                    logger.warn("Discovered protocol tests for ${file.name}")

                    listOf(testFile)

                } else {

                    listOf()

                }

                AwsService(

                    service = service.id.toString(),

                    module = sdkId,

                    moduleDescription = "AWS SDK for $title",

                    modelFile = file,

                    extraFiles = extras,

                    humanName = title

                )

            }

        }

    val baseModules = baseServices.map { it.module }.toSet()



    // validate the full exclusion list hits

    serviceMembership.exclusions.forEach { disabledService ->

        check(baseModules.contains(disabledService)) {

            "Service $disabledService was explicitly disabled but no service was generated with that name. Generated:\n ${

            baseModules.joinToString(

                "\n "

            )

            }"

        }

    }

    // validate inclusion list hits

    serviceMembership.inclusions.forEach { service ->

        check(baseModules.contains(service)) { "Service $service was in explicit inclusion list but not generated!" }

    }

    return baseServices.filter {

        serviceMembership.isMember(it.module)

    }

}



data class Membership(val inclusions: Set<String> = emptySet(), val exclusions: Set<String> = emptySet())



data class AwsService(

    val service: String,

    val module: String,

    val moduleDescription: String,

    val modelFile: File,

    val extraConfig: String? = null,

    val extraFiles: List<File>,

    val humanName: String

) {

    fun files(): List<File> = listOf(modelFile) + extraFiles

    fun Project.examples(): Boolean = projectDir.resolve("examples").resolve(module).exists()

}



fun AwsService.crate(): String = "aws-sdk-$module"



fun Membership.isMember(member: String): Boolean = when {

    exclusions.contains(member) -> false

    inclusions.contains(member) -> true

    inclusions.isEmpty() -> true

    else -> false

}



fun parseMembership(rawList: String): Membership {

    val inclusions = mutableSetOf<String>()

    val exclusions = mutableSetOf<String>()



    rawList.split(",").map { it.trim() }.forEach { item ->

        when {

            item.startsWith('-') -> exclusions.add(item.substring(1))

            item.startsWith('+') -> inclusions.add(item.substring(1))

            else -> error("Must specify inclusion (+) or exclusion (-) prefix character to $item.")

        }

    }



    val conflictingMembers = inclusions.intersect(exclusions)

    require(conflictingMembers.isEmpty()) { "$conflictingMembers specified both for inclusion and exclusion in $rawList" }



    return Membership(inclusions, exclusions)

}