Terraform working with Collection Functions

Terraform Notes

Working with Collection Functions

Here's a little challenge: Gather the structure of existing OU of AWS Organization. We are not going to get all the OU IDs and import then into our Terraform State. We are also not going to just [destroy and] rebuild them. Instead we are going to get it using Data call.

One hurdle was that each call to retrieve OU only returns its immediate child.
https://www.terraform.io/docs/providers/aws/d/organizations_organizational_units.html

So I had to hardcode how deep I wanted to retrieve (AWS limits you to 5 deep for OU). But this is a good illustration of working with Map, List, flatten, and concat.

Few highlights:

When you do a data call as a count loop, Terraform will create a list of 3 attributes

• Terraform ID
• Attribute used as an argument reference (in the case of Org, this is Parent_ID)
• List of the expected attribute reference (in the case of Org, this is Children)

When you flatten a list of attributes that contains another list of attributes, your end result is list of attribute. This is how I picture it:

Finding a attribute's value in a list and then returning another attribute's value from that index is pretty straight forward:

parent_name = [for x in local.level1_ou_flat: x.name if x.id == ou.parent_id][0]

This code says, loop through all the items in local.level1_ou_flat and only keep name where id is  ou.parent_id. The above code snippet works because I know that all the names returned will be the same, so I just keep the first item in the list. 

When you output something from module, this will be returned to main call, so from main you have to explicitly output this value which can be referenced using the module's name that you defined in the main. No, you don't have to call this output, main. Also, output is ONLY invoked when you do Terraform Apply. Terraform Plan, unfortunately does not provide output.

In Module:

output "ou_tree" {
  value = local.all_ou
}

In Main:

output "main"{
  value = module.org_scp.ou_tree
}

main.tf

provider "aws" {
  alias    = "master_east_2"
  version = "~> 2.0"
  region  = "us-east-2"
  access_key = "AAAAAAAAAAAAAAAAAAA"
  secret_key = "zzzzzzzzzzzzzzzzzzz"
}

module "org_scp" {
  source = "./module"
  providers = {
    aws = aws.master_east_2
  }  
}

output "main"{
  value = module.org_scp.ou_tree
}

module/m_org_scp.tf

// this return information about org: https://www.terraform.io/docs/providers/aws/d/organizations_organization.html
data "aws_organizations_organization" "myRoot"{
}

/*
data.aws_organizations_organization.myRoot.roots[0].id
data.aws_organizations_organization.myRoot.roots[0].name
data.aws_organizations_organization.myRoot.roots[0].arn
*/

// root will only have 1 element, but we leave this open-ended for consistency with the remaining levels
data "aws_organizations_organizational_units" "level1_ou"{
    count = length(data.aws_organizations_organization.myRoot.roots)
    parent_id = data.aws_organizations_organization.myRoot.roots[count.index].id
}

/* 
data.aws_organizations_organizational_units.level1_ou[*].parent_id
data.aws_organizations_organizational_units.level1_ou[*].id  <-- don't use this
data.aws_organizations_organizational_units.level1_ou[*].children[*].id
data.aws_organizations_organizational_units.level1_ou[*].children[*].arn
data.aws_organizations_organizational_units.level1_ou[*].children[*].name 
*/

locals {
    level1_ou_flat = flatten([
        for ou in data.aws_organizations_organizational_units.level1_ou : [
            for child in ou.children : {
                parent_id = ou.parent_id
                parent_name = data.aws_organizations_organization.myRoot.roots[0].name
                id = child.id
                name = child.name
                arn = child.arn
                path = "${data.aws_organizations_organization.myRoot.roots[0].name}/${child.name}"
            }
        ]
    ])        
}

/*
local.level1_ou_flat[*].parent_id
local.level1_ou_flat[*].parent_name
local.level1_ou_flat[*].id
local.level1_ou_flat[*].name
local.level1_ou_flat[*].arn
local.level1_ou_flat[*].path
*/

// this returns a list of list of OUs on layer 2
data "aws_organizations_organizational_units" "level2_ou"{
    count = length(local.level1_ou_flat)
    parent_id = local.level1_ou_flat[count.index].id 
}

/* 
data.aws_organizations_organizational_units.level2_ou[*].parent_id
data.aws_organizations_organizational_units.level2_ou[*].id  <-- don't use this
data.aws_organizations_organizational_units.level2_ou[*].children[*].id
data.aws_organizations_organizational_units.level2_ou[*].children[*].arn
data.aws_organizations_organizational_units.level2_ou[*].children[*].name 
*/

// flatten the list of list layer 2
locals {
    level2_ou_flat = flatten([
        for ou in data.aws_organizations_organizational_units.level2_ou : [
            for child in ou.children : {
                parent_id = ou.parent_id
                parent_name = [for x in local.level1_ou_flat: x.name if x.id == ou.parent_id][0]
                id = child.id
                name = child.name
                arn = child.arn
                path = "${[for x in local.level1_ou_flat: x.path if x.id == ou.parent_id][0]}/${child.name}"
            }
        ]
    ])        
}

/*
local.level2_ou_flat[*].parent_id
local.level2_ou_flat[*].parent_name
local.level2_ou_flat[*].id
local.level2_ou_flat[*].name
local.level2_ou_flat[*].arn
local.level2_ou_flat[*].path
*/

// this returns a list of list of OUs on layer 3
data "aws_organizations_organizational_units" "level3_ou"{
    count = length(local.level2_ou_flat)
    parent_id = local.level2_ou_flat[count.index].id
}

// flatten the list of list layer 3
locals {
    level3_ou_flat = flatten([
        for ou in data.aws_organizations_organizational_units.level3_ou : [
            for child in ou.children : {
                parent_id = ou.parent_id
                parent_name = [for x in local.level2_ou_flat: x.name if x.id == ou.parent_id][0]
                id = child.id
                name = child.name
                arn = child.arn
                path = "${[for x in local.level2_ou_flat: x.path if x.id == ou.parent_id][0]}/${child.name}"
            }
        ]
    ])        
}

// this returns a list of list of OUs on layer 4
data "aws_organizations_organizational_units" "level4_ou"{
    count = length(local.level3_ou_flat)
    parent_id = local.level3_ou_flat[count.index].id
}

// flatten the list of list layer 4
locals {
    level4_ou_flat = flatten([
        for ou in data.aws_organizations_organizational_units.level4_ou : [
            for child in ou.children : {
                parent_id = ou.parent_id
                parent_name = [for x in local.level3_ou_flat: x.name if x.id == ou.parent_id][0]
                id = child.id
                name = child.name
                arn = child.arn
                path = "${[for x in local.level3_ou_flat: x.path if x.id == ou.parent_id][0]}/${child.name}"
            }
        ]
    ])        
}

// this returns a list of list of OUs on layer 5
data "aws_organizations_organizational_units" "level5_ou"{
    count = length(local.level4_ou_flat)
    parent_id = local.level4_ou_flat[count.index].id
}

// flatten the list of list layer 5
locals {
    level5_ou_flat = flatten([
        for ou in data.aws_organizations_organizational_units.level5_ou : [
            for child in ou.children : {
                parent_id = ou.parent_id
                parent_name = [for x in local.level4_ou_flat: x.name if x.id == ou.parent_id][0]
                id = child.id
                name = child.name
                arn = child.arn
                path = "${[for x in local.level4_ou_flat: x.path if x.id == ou.parent_id][0]}/${child.name}"
            }
        ]
    ])        
}


locals {
    // AWS Organization currently only supports 1 root
    root_id = data.aws_organizations_organization.myRoot.roots[0].id
    root_accounts = data.aws_organizations_organization.myRoot.accounts[*].id
    all_ou = concat(local.level1_ou_flat,local.level2_ou_flat,local.level3_ou_flat,local.level4_ou_flat,local.level5_ou_flat)

    /*
    local.all_ou[*].parent_id
    local.all_ou[*].parent_name
    local.all_ou[*].id
    local.all_ou[*].name
    local.all_ou[*].arn
    local.all_ou[*].path
    */
}


output "ou_tree" {
  value = local.all_ou
}

output:

Apply complete! Resources: 0 added, 0 changed, 0 destroyed.

Outputs:

main = [
  {
    "arn" = "arn:aws:organizations::111111111111:ou/o-fyteu3rxnm/ou-abcd-wdbh6dlt"
    "id" = "ou-abcd-wdbh6dlt"
    "name" = "QA"
    "parent_id" = "r-abcd"
    "parent_name" = "Root"
    "path" = "Root/QA"
  },
  {
    "arn" = "arn:aws:organizations::111111111111:ou/o-fyteu3rxnm/ou-abcd-zuidgyzl"
    "id" = "ou-abcd-zuidgyzl"
    "name" = "Test"
    "parent_id" = "r-abcd"
    "parent_name" = "Root"
    "path" = "Root/Test"
  },
  {
    "arn" = "arn:aws:organizations::111111111111:ou/o-fyteu3rxnm/ou-abcd-dpde5cp9"
    "id" = "ou-abcd-dpde5cp9"
    "name" = "Prod"
    "parent_id" = "r-abcd"
    "parent_name" = "Root"
    "path" = "Root/Prod"
  },
  {
    "arn" = "arn:aws:organizations::111111111111:ou/o-fyteu3rxnm/ou-abcd-bi34zvmq"
    "id" = "ou-abcd-bi34zvmq"
    "name" = "QA_App"
    "parent_id" = "ou-abcd-wdbh6dlt"
    "parent_name" = "QA"
    "path" = "Root/QA/QA_App"
  },
  {
    "arn" = "arn:aws:organizations::111111111111:ou/o-fyteu3rxnm/ou-abcd-fbj0cnnr"
    "id" = "ou-abcd-fbj0cnnr"
    "name" = "App1"
    "parent_id" = "ou-abcd-zuidgyzl"
    "parent_name" = "Test"
    "path" = "Root/Test/App1"
  },
  {
    "arn" = "arn:aws:organizations::111111111111:ou/o-fyteu3rxnm/ou-abcd-t3ngdv40"
    "id" = "ou-abcd-t3ngdv40"
    "name" = "Prod_App2"
    "parent_id" = "ou-abcd-dpde5cp9"
    "parent_name" = "Prod"
    "path" = "Root/Prod/Prod_App2"
  },
  {
    "arn" = "arn:aws:organizations::111111111111:ou/o-fyteu3rxnm/ou-abcd-qubfzrrr"
    "id" = "ou-abcd-qubfzrrr"
    "name" = "Prod_App"
    "parent_id" = "ou-abcd-dpde5cp9"
    "parent_name" = "Prod"
    "path" = "Root/Prod/Prod_App"
  },
  {
    "arn" = "arn:aws:organizations::111111111111:ou/o-fyteu3rxnm/ou-abcd-x4csr627"
    "id" = "ou-abcd-x4csr627"
    "name" = "SubDeptA"
    "parent_id" = "ou-abcd-fbj0cnnr"
    "parent_name" = "App1"
    "path" = "Root/Test/App1/SubDeptA"
  },
  {
    "arn" = "arn:aws:organizations::111111111111:ou/o-fyteu3rxnm/ou-abcd-cii1t2vv"
    "id" = "ou-abcd-cii1t2vv"
    "name" = "SubDeptB"
    "parent_id" = "ou-abcd-fbj0cnnr"
    "parent_name" = "App1"
    "path" = "Root/Test/App1/SubDeptB"
  },
]