Terraform Multi-Stage Pipelines

Inspired by David's work here: https://github.com/flcdrg/azure-pipelines-each/

Repo Structure

Terraform modules go in src/

Pipelines

variables-production.yml

variables:
  environment: 'production'
  terraform_version: '1.6.3'
  terraform_bucket_name: 'acme-corp-production-terraform'
  aws_service_connection: 'AWS - Acme Corp'
  aws_region: 'ap-southeast-2'

azure-pipelines.yml

parameters:
  - name: environments
    type: object
    default:
      - name: development
      - name: production

trigger: none

stages:
  - stage: Build
    displayName: Build lambda webhook
    dependsOn: []
    jobs:
      - template: terraform-build.yml

  - ${{ each environment in parameters.environments }}:
      - stage: TerraformPlan_${{ environment.name }}
        displayName: Plan ${{ environment.name }}
        dependsOn: Build
        variables:
          - template: variables-${{ environment.name }}.yml
          - group: infrastructure-variables-${{ environment.name }}
        jobs:
          - template: terraform-plan.yml
            parameters:
              environmentName: ${{ environment.name }}

      - stage: TerraformApply_${{ environment.name }}
        displayName: Apply ${{ environment.name }}
        dependsOn: TerraformPlan_${{ environment.name }}
        variables:
          - template: variables-${{ environment.name }}.yml
          - group: infrastructure-variables-${{ environment.name }}
        jobs:
          - template: terraform-apply.yml
            parameters:
              environmentName: ${{ environment.name }}

terraform-build.yml

jobs:
  - job: build
    pool:
      vmImage: ubuntu-latest
    steps:
      - task: NodeTool@0
        displayName: Setup Node
        inputs:
          versionSpec: '19.4.x'

      - task: Bash@3
        displayName: Install dependencies
        inputs:
          targetType: inline
          workingDirectory: src/webhooks
          script: |
            npm install

      - task: Bash@3
        displayName: Build webhook
        inputs:
          targetType: inline
          workingDirectory: src/webhooks
          script: |
            npm run build

terraform-plan.yml

parameters:
  - name: environmentName
    type: string

jobs:
  - job: plan
    variables:
      - template: variables-${{ parameters.environmentName }}.yml
      - group: infrastructure-variables-${{ parameters.environmentName }}
    pool:
      vmImage: ubuntu-latest
    steps:
      - task: JasonBJohnson.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-installer.TerraformInstaller@1
        displayName: install terraform $(terraform_version)
        inputs:
          terraform_version: $(terraform_version)

      - task: JasonBJohnson.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-cli.TerraformCLI@1
        displayName: terraform init
        inputs:
          command: init
          workingDirectory: '$(System.DefaultWorkingDirectory)/src'
          backendType: aws
          backendServiceAws: ${{ variables.aws_service_connection }}
          backendAWSBucket: $(terraform_bucket_name)
          backendAWSKey: 'terraform.tfstate'

      - task: replacetokens@5
        inputs:
          targetFiles: "environments/${{ parameters.environmentName }}.tfvars"
          encoding: 'auto'
          tokenPattern: 'octopus'
          writeBOM: false
          actionOnMissing: 'warn'
          keepToken: false
          actionOnNoFiles: 'continue'
          enableTransforms: false
          enableRecursion: false
          useLegacyPattern: false
          enableTelemetry: true

      - task: Bash@3
        displayName: Install dependencies
        inputs:
          targetType: inline
          workingDirectory: $(System.DefaultWorkingDirectory)
          script: |
            mkdir terraform_plan

      - task: JasonBJohnson.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-cli.TerraformCLI@1
        displayName: terraform plan
        inputs:
          command: plan
          commandOptions: "--var-file=$(System.DefaultWorkingDirectory)/environments/${{ parameters.environmentName }}.tfvars -out=${{ parameters.environmentName }}_plan.tfplan"
          publishPlanResults: "${{ parameters.environmentName }}_plan"
          workingDirectory: '$(System.DefaultWorkingDirectory)/src'
          providerServiceAws: ${{ variables.aws_service_connection }}
          providerAwsRegion: $(aws_region)

      - task: PublishPipelineArtifact@1
        displayName: Publish Plan artifact
        inputs:
          targetPath: '$(System.DefaultWorkingDirectory)/src'
          artifactName: "${{ parameters.environmentName }}_terraform_plan"
          publishLocation: 'pipeline'

terraform-apply.yml

parameters:
  - name: environmentName
    type: string

jobs:
  - deployment: Apply
    condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
    variables:
      - template: variables-${{ parameters.environmentName }}.yml
    pool:
      vmImage: ubuntu-latest
    environment: ${{ parameters.environmentName }}
    strategy:
      runOnce:
        deploy:
          steps:
            - task: JasonBJohnson.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-installer.TerraformInstaller@1
              displayName: Install terraform $(terraform_version)
              inputs:
                terraform_version: $(terraform_version)

            - task: DownloadPipelineArtifact@2
              inputs:
                artifactName: "${{ parameters.environmentName }}_terraform_plan"
                path: "${{ parameters.environmentName }}_terraform_plan"

            - task: Bash@3
              displayName: List files
              inputs:
                targetType: inline
                workingDirectory: $(System.DefaultWorkingDirectory)
                script: |
                  rm -rf ${{ parameters.environmentName }}_terraform_plan/.terraform/
                  find . 

            - task: JasonBJohnson.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-cli.TerraformCLI@1
              displayName: terraform init
              inputs:
                command: init
                workingDirectory: "${{ parameters.environmentName }}_terraform_plan"
                backendType: aws
                backendServiceAws: ${{ variables.aws_service_connection }}
                backendAWSBucket: $(terraform_bucket_name)
                backendAWSKey: 'terraform.tfstate'

            - task: JasonBJohnson.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-cli.TerraformCLI@1
              displayName: terraform apply
              inputs:
                command: apply
                workingDirectory: "${{ parameters.environmentName }}_terraform_plan"
                commandOptions: "${{ parameters.environmentName }}_plan.tfplan"
                providerServiceAws: ${{ variables.aws_service_connection }}
                providerAwsRegion: $(aws_region)