title	description	author	publishedDate	date
Build a CI/CD Pipeline with Integration Test	Build a basic ci/cd pipeline with integration test	haimtran	06/23/2022	2022-07-24

Basic CI/CD Pipeline with Integration Test

Introduction

GitHub this shows a basic examle of a ci/cd pipeline for a lambda api: codebuild for unittest, codebuild for integration test, codeploy for deploy the api stack. The api url is passed via system parameter store from deployed pre-product to the integration test.

Application Stack

lambda function

import json

def handler(event, context):
    """
    lambda handler
    """
    return {
        'statusCode': 200,
        'headers': {
            "Access-Control-Allow-Origin": "*",
            "Access-Control-Allow-Headers": "Content-Type",
            "Access-Control-Allow-Methods": "OPTIONS,GET"
        },
        'body': json.dumps({
            'message': f"{event}"
        })
    }

application stack is a lambda backed api

export interface ApplicationProps extends StackProps {
  environment: string;
}

export class ApplicationStack extends Stack {
  public readonly url: CfnOutput;

  constructor(scope: Construct, id: string, props: ApplicationProps) {
    super(scope, id, props);

    // lambda function
    const fn = new aws_lambda.Function(this, "Lambda", {
      functionName: `HelloPipeline${props.environment}`,
      runtime: aws_lambda.Runtime.PYTHON_3_8,
      timeout: Duration.seconds(10),
      code: aws_lambda.Code.fromAsset(path.join(__dirname, "../lambda/")),
      handler: "index.handler",
    });

    // api gateway
    const api = new aws_apigateway.RestApi(this, "ApiGwDemo", {
      restApiName: `ApiGwDemo${props.environment}`,
    });

    // api resource
    const resource = api.root.addResource("book");

    // api method
    resource.addMethod("GET", new aws_apigateway.LambdaIntegration(fn));

    this.url = new CfnOutput(this, `Url${props.environment}`, {
      description: "api url",
      exportName: `Url${props.environment}`,
      value: api.url,
    });
  }

GitHub Connection

// github source
const sourceAction =
  new aws_codepipeline_actions.CodeStarConnectionsSourceAction({
    actionName: "GitHub",
    owner: "entest-hai",
    connectionArn: `arn:aws:codestar-connections:${this.region}:${this.account}:connection/${props.codeStarId}`,
    repo: "cicd-integration-test",
    branch: "master",
    output: sourceOutput,
  });

codecommmit connection

const sourceAction = new aws_codepipeline_actions.CodeCommitSourceAction({
  actionName: "CodeCommit",
  repository: repo,
  branch: "master",
  output: sourceOutput,
  variablesNamespace: "SourceVariables",
});

CodeBuild Unittest

// codebuild unitest
const unittestCodeBuild = new aws_codebuild.PipelineProject(
  this,
  "CodeBuildUnittest",
  {
    environment: {
      buildImage: aws_codebuild.LinuxBuildImage.STANDARD_5_0,
    },
    buildSpec: aws_codebuild.BuildSpec.fromObject({
      version: "0.2",
      phases: {
        install: {
          commands: ["echo $CODE_COMMIT_ID", "pip install -r requirements.txt"],
        },
        build: {
          commands: ["python -m pytest -s -v unittests/test_lambda_logic.py"],
        },
      },
      artifacts: {},
    }),
  }
);

CodeBuild CDK Stacks

// codebuild cdk template
const cdkCodeBuild = new aws_codebuild.PipelineProject(this, "CodeBuildCdk", {
  environment: {
    buildImage: aws_codebuild.LinuxBuildImage.STANDARD_5_0,
  },
  buildSpec: aws_codebuild.BuildSpec.fromObject({
    version: "0.2",
    phases: {
      install: {
        commands: ["npm install"],
      },
      build: {
        commands: ["npm run cdk synth -- -o dist"],
      },
    },
    artifacts: {
      "base-directory": "dist",
      files: ["*.template.json"],
    },
  }),
});

CodeDeploy Preproduct

{
  stageName: "Deploy",
  actions: [
    new aws_codepipeline_actions.CloudFormationCreateUpdateStackAction(
      {
        actionName: "DeployApplication",
        templatePath: cdkBuildOutput.atPath(
          "ApplicationStack.template.json"
        ),
        stackName: "PreProductApplicationStack",
        adminPermissions: true,
      }
    ),
  ],
},

CoceBuild Integration Test

We need to get the API endpoint from the deployed pre-production stack. This can be done by several ways such as aws cloudformation describe stacks or boto3 python code.

// codebuild integration test
const integtestCodeBuild = new aws_codebuild.PipelineProject(
  this,
  "CodeBuildIntegTest",
  {
    role: role,
    environment: {
      buildImage: aws_codebuild.LinuxBuildImage.STANDARD_5_0,
    },
    buildSpec: aws_codebuild.BuildSpec.fromObject({
      version: "0.2",
      phases: {
        install: {
          commands: [
            `SERVICE_URL=$(aws cloudformation describe-stacks --stack-name PreProdApplicationStack --query "Stacks[0].Outputs[?OutputKey=='UrlPreProd'].OutputValue" --output text)`,
            "echo $SERVICE_URL",
            "pip install -r requirements.txt",
          ],
        },
        build: {
          commands: ["python -m pytest -s -v integtests/test_service.py"],
        },
      },
      artifacts: {},
    }),
  }
);

CodeDeploy Product

// deploy preprod
const deployPreProd =
  new aws_codepipeline_actions.CloudFormationCreateUpdateStackAction({
    actionName: "DeployPreProdApplication",
    templatePath: cdkBuildOutput.atPath(
      "PreProdApplicationStack.template.json"
    ),
    stackName: "PreProdApplicationStack",
    adminPermissions: true,
    variablesNamespace: "PreProdVariables",
    outputFileName: "PreProdOutputs",
    output: preProdOutput,
  });

CodePipeline Artifacts

// source output
const sourceOutput = new aws_codepipeline.Artifact("SourceCode");
const unitestCodeBuildOutput = new aws_codepipeline.Artifact(
  "UnittestBuildOutput"
);
const cdkBuildOutput = new aws_codepipeline.Artifact("CdkBuildOutput");

CodePipeline

// pipeline
const pipeline = new aws_codepipeline.Pipeline(this, "DevOpsDemoPipeline", {
  pipelineName: "DevOpsDemoPipeline",
  crossAccountKeys: false,
  stages: [
    {
      stageName: "Source",
      actions: [sourceAction],
    },
    {
      stageName: "Unittest",
      actions: [unittestBuildAction],
    },
    {
      stageName: "BuildTemplate",
      actions: [cdkBuild],
    },
    {
      stageName: "DeployPreProd",
      actions: [deployPreProd],
    },
    {
      stageName: "IntegTest",
      actions: [integtestBuildAction],
    },
    {
      stageName: "DeployProd",
      actions: [deployProd],
    },
  ],
});

Integration Test

option 1) using boto3 to query api url from the PreProdApplication stack. option 2) codebuild run a cli command to query the api url

`SERVICE_URL=$(aws cloudformation describe-stacks --stack-name PreProdApplicationStack --query "Stacks[0].Outputs[?OutputKey=='UrlPreProd'].OutputValue" --output text)`

import boto3
import requests

STACK_NAME = "PreProdApplicationStack"
ENDPOINT = "book"

def query_api_url(stack_name):
    """
    query api url from cloudformation template output
    """

    # cloudformation client
    client = boto3.client('cloudformation')
    # query application stack
    resp = client.describe_stacks(
        StackName=stack_name
    )
    # looking for api url in stack output
    stack_outputs = resp['Stacks'][0]['Outputs']
    for output in stack_outputs:
        if output['OutputKey'] == 'UrlPreProd':
            api_url = output['OutputValue']
            print(f"api url: {api_url}")
    # return api url
    return api_url

then perform a simple test to assert status code 200

def test_200_response():
    # get api url
    api_url = query_api_url(STACK_NAME)
    # send request
    with requests.get(f"{api_url}/{ENDPOINT}") as response:
        print(response.text)
        assert response.status_code == 200
#
if __name__=="__main__":
    test_200_response()

Reference

1. Reinvent 2021: Across account CI/CD pipelines
2. Enhanced CI/CD with AWS CDK CodePipeline
3. Building a Cross account CI/CD Pipeline Workshop