{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Parameters": {
    "VpcCidrBlock": {
      "Type": "String"
    },
    "VpcName": {
      "Type": "String",
      "Default": "ose-on-aws"
    },
    "S3BucketName": {
      "Type": "String"
    },
    "S3User": {
      "Type": "String"
    },
    "MasterApiPort": {
      "Type": "Number"
    },
    "MasterHealthTarget": {
      "Type": "String"
    },
    "Route53HostedZone": {
      "Type": "String"
    },
    "PublicHostedZone": {
      "Type": "String"
    },
    "Region": {
      "Type": "String"
    },
    "MasterClusterPublicHostname": {
      "Type": "String"
    },
    "MasterClusterHostname": {
      "Type": "String"
    },
    "AppWildcardDomain": {
      "Type": "String"
    },
    "SubnetCidrBlocks": {
      "Type": "CommaDelimitedList"
    },
    "KeyName": {
      "Type": "AWS::EC2::KeyPair::KeyName"
    },
    "MasterInstanceType": {
      "Type": "String"
    },
    "AmiId": {
      "Type": "AWS::EC2::Image::Id"
    },
    "BastionUserData": {
      "Type": "String"
    },
    "MasterDockerVolSize": {
      "Type": "String"
    },
    "MasterEtcdVolSize": {
      "Type": "String"
    },
    "MasterUserData": {
      "Type": "String"
    },
    "MasterEtcdVolType": {
      "Type": "String"
    },
    "MasterDockerVolType": {
      "Type": "String"
    },
    "InfraInstanceType": {
      "Type": "String"
    },
    "InfraDockerVolSize": {
      "Type": "String"
    },
    "InfraDockerVolType": {
      "Type": "String"
    },
    "AppNodeInstanceType": {
      "Type": "String"
    },
    "NodeUserData": {
      "Type": "String"
    },
    "NodeDockerVolSize": {
      "Type": "String"
    },
    "NodeDockerVolType": {
      "Type": "String"
    }
  },
  "Conditions": {
    "CreateDhcpOpts": {"Fn::Equals" : [{"Ref" : "Region"}, "us-east-1"]}
  },
  "Resources": {
    "Vpc": {
      "Type": "AWS::EC2::VPC",
      "Properties": {
        "CidrBlock": { "Ref": "VpcCidrBlock" },
        "EnableDnsSupport": "true",
        "EnableDnsHostnames": "true",
        "Tags": [ { "Key": "Name", "Value": { "Ref": "VpcName" } } ]
      }
    },
    "VpcInternetGateway": {
      "Type": "AWS::EC2::InternetGateway",
      "Properties": {}
    },
    "VpcGA": {
      "Type": "AWS::EC2::VPCGatewayAttachment",
      "Properties": {
        "InternetGatewayId": { "Ref": "VpcInternetGateway" },
        "VpcId": { "Ref": "Vpc" }
      }
    },
    "VpcRouteTable": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "VpcId": { "Ref": "Vpc" }
      }
    },
    "PrivateRouteTable": {
      "DependsOn": ["Nat"],
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "VpcId": { "Ref": "Vpc" }
      }
    },
    "VPCRouteInternetGateway": {
      "Type": "AWS::EC2::Route",
      "Properties": {
        "GatewayId": { "Ref": "VpcInternetGateway" },
        "DestinationCidrBlock": "0.0.0.0/0",
        "RouteTableId": { "Ref": "VpcRouteTable" }
      }
    },
    "EIP" : {
       "Type" : "AWS::EC2::EIP",
       "Properties" : {
          "Domain" : "vpc"
      }
    },
    "Route" : {
      "DependsOn": ["Nat"],
      "Type" : "AWS::EC2::Route",
      "Properties" : {
         "RouteTableId" : { "Ref" : "PrivateRouteTable" },
         "DestinationCidrBlock" : "0.0.0.0/0",
         "NatGatewayId" : { "Ref" : "Nat" }
      }
     },
    "PublicSubnet1": {
      "Type": "AWS::EC2::Subnet",
      "DependsOn": ["Vpc"],
      "Properties": {
        "AvailabilityZone": {"Fn::Select": ["0", { "Fn::GetAZs" : "" }]},
        "CidrBlock": {"Fn::Select": ["0", {"Ref": "SubnetCidrBlocks"}]},
        "Tags": [ {"Key": "Name", "Value": "Public_Subnet_1"}, {"Key": "KubernetesCluster", "Value": { "Ref": "AWS::StackName" }}],
        "MapPublicIpOnLaunch": "true",
        "VpcId": { "Ref": "Vpc" }
      }
    },
    "PublicSubnet1RTA": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": { "Ref": "VpcRouteTable" },
        "SubnetId": { "Ref": "PublicSubnet1" }
      }
    },
    "PublicSubnet2": {
      "Type": "AWS::EC2::Subnet",
      "DependsOn": ["Vpc"],
      "Properties": {
        "AvailabilityZone": {"Fn::Select": ["1", { "Fn::GetAZs" : "" }]},
        "CidrBlock": {"Fn::Select": ["1", {"Ref": "SubnetCidrBlocks"}]},
        "Tags": [ {"Key": "Name", "Value": "Public_Subnet_2"}, {"Key": "KubernetesCluster", "Value": { "Ref": "AWS::StackName" }}],
        "MapPublicIpOnLaunch": "true",
        "VpcId": { "Ref": "Vpc" }
      }
    },
    "PublicSubnet2RTA": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": { "Ref": "VpcRouteTable" },
        "SubnetId": { "Ref": "PublicSubnet2" }
      }
    },
    "PublicSubnet3": {
      "Type": "AWS::EC2::Subnet",
      "DependsOn": ["Vpc"],
      "Properties": {
        "AvailabilityZone": {"Fn::Select": ["2", { "Fn::GetAZs" : "" }]},
        "CidrBlock": {"Fn::Select": ["2", {"Ref": "SubnetCidrBlocks"}]},
        "Tags": [ {"Key": "Name", "Value": "Public_Subnet_3"}, {"Key": "KubernetesCluster", "Value": { "Ref": "AWS::StackName" }}],
        "MapPublicIpOnLaunch": "true",
        "VpcId": { "Ref": "Vpc" }
      }
    },
    "PublicSubnet3RTA": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "RouteTableId": { "Ref": "VpcRouteTable" },
        "SubnetId": { "Ref": "PublicSubnet3" }
      }
    },
    "PrivateSubnet1": {
      "DependsOn": ["Nat"],
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": {"Fn::Select": ["0", { "Fn::GetAZs" : "" }]},
        "CidrBlock": {"Fn::Select": ["3", {"Ref": "SubnetCidrBlocks"}]},
        "Tags": [ {"Key": "Name", "Value": "Private_Subnet_1"}, {"Key": "KubernetesCluster", "Value": { "Ref": "AWS::StackName" }}],
        "VpcId": { "Ref": "Vpc" }
      }
    },
    "PrivateSubnet1RTA" : {
      "Type" : "AWS::EC2::SubnetRouteTableAssociation",
      "Properties" : {
         "SubnetId" : { "Ref" : "PrivateSubnet1" },
         "RouteTableId" : { "Ref" : "PrivateRouteTable" }
      }
    },
    "PrivateSubnet2": {
      "DependsOn": ["Nat"],
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": {"Fn::Select": ["1", { "Fn::GetAZs" : "" }]},
        "CidrBlock": {"Fn::Select": ["4", {"Ref": "SubnetCidrBlocks"}]},
        "Tags": [ {"Key": "Name", "Value": "Private_Subnet_2"}, {"Key": "KubernetesCluster", "Value": { "Ref": "AWS::StackName" }}],
        "VpcId": { "Ref": "Vpc" }
      }
    },
    "PrivateSubnet2RTA" : {
      "DependsOn": ["Nat"],
      "Type" : "AWS::EC2::SubnetRouteTableAssociation",
      "Properties" : {
         "SubnetId" : { "Ref" : "PrivateSubnet2" },
         "RouteTableId" : { "Ref" : "PrivateRouteTable" }
      }
    },
    "PrivateSubnet3": {
      "DependsOn": ["Nat"],
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "AvailabilityZone": {"Fn::Select": ["2", { "Fn::GetAZs" : "" }]},
        "CidrBlock": {"Fn::Select": ["5", {"Ref": "SubnetCidrBlocks"}]},
        "Tags": [ {"Key": "Name", "Value": "Private_Subnet_3"}, {"Key": "KubernetesCluster", "Value": { "Ref": "AWS::StackName" }}],
        "VpcId": { "Ref": "Vpc" }
      }
    },
    "PrivateSubnet3RTA" : {
       "Type" : "AWS::EC2::SubnetRouteTableAssociation",
       "Properties" : {
          "SubnetId" : { "Ref" : "PrivateSubnet3" },
          "RouteTableId" : { "Ref" : "PrivateRouteTable" }
      }
    },
    "Nat" : {
      "DependsOn": ["EIP"],
      "Type" : "AWS::EC2::NatGateway",
      "Properties" : {
          "AllocationId" : { "Fn::GetAtt" : ["EIP", "AllocationId"]},
          "SubnetId" : { "Ref" : "PublicSubnet1"}
      }
    },
    "EIP" : {
       "Type" : "AWS::EC2::EIP",
       "Properties" : {
          "Domain" : "vpc"
      }
    },
    "Route" : {
      "DependsOn": ["Nat"],
      "Type" : "AWS::EC2::Route",
      "Properties" : {
         "RouteTableId" : { "Ref" : "PrivateRouteTable" },
         "DestinationCidrBlock" : "0.0.0.0/0",
         "NatGatewayId" : { "Ref" : "Nat" }
      }
     },
    "DHCPOpts" : {
      "Type" : "AWS::EC2::DHCPOptions",
      "Condition": "CreateDhcpOpts",
      "Properties" : {
        "DomainName": "ec2.internal",
        "DomainNameServers": [ "AmazonProvidedDNS" ]
      }
    },
    "AssociateOpts" : {
      "Type" : "AWS::EC2::VPCDHCPOptionsAssociation",
      "Condition": "CreateDhcpOpts",
      "Properties" : {
          "VpcId": { "Ref" : "Vpc" },
          "DhcpOptionsId": { "Ref" : "DHCPOpts" }
      }
    },
    "BastionSg": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "bastion-sg",
        "VpcId": { "Ref": "Vpc" },
        "Tags": [ {"Key": "Name", "Value": "bastion_sg"} ],
        "SecurityGroupIngress": [
          {
            "IpProtocol": "tcp",
            "FromPort": "22",
            "ToPort": "22",
            "CidrIp": "0.0.0.0/0"
          }
        ]
      }
    },
    "EtcdSG": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "etcd",
        "VpcId": { "Ref": "Vpc" },
        "Tags": [ {"Key": "Name", "Value": "ose_etcd_sg"} ]
      }
    },
    "InfraElbSG": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Infra Load Balancer",
        "VpcId": { "Ref": "Vpc" },
        "Tags": [ {"Key": "Name", "Value": "ose_router_sg"} ],
        "SecurityGroupIngress": [
          {
            "IpProtocol": "tcp",
            "FromPort": "80",
            "ToPort": "80",
            "CidrIp": "0.0.0.0/0"
          },
          {
            "IpProtocol": "tcp",
            "FromPort": "443",
            "ToPort": "443",
            "CidrIp": "0.0.0.0/0"
          }
        ]
      }
    },
    "MasterExtElbSG": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Master External Load Balancer",
        "VpcId": { "Ref": "Vpc" },
        "Tags": [ {"Key": "Name", "Value": "ose_elb_master_sg"} ],
        "SecurityGroupIngress": [
          {
            "IpProtocol": "tcp",
            "FromPort": { "Ref": "MasterApiPort" },
            "ToPort": { "Ref": "MasterApiPort" },
            "CidrIp": "0.0.0.0/0"
          }
        ]
      }
    },
    "MasterIntElbSG": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Master Internal Load Balancer",
        "VpcId": { "Ref": "Vpc" },
        "Tags": [ {"Key": "Name", "Value": "ose_internal_elb_master_sg"} ]
      }
    },
    "InfraSG": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Infra",
        "VpcId": { "Ref": "Vpc" },
        "Tags": [ {"Key": "Name", "Value": "ose_infra_node_sg"} ]
      }
    },
    "NodeSG": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Node",
        "VpcId": { "Ref": "Vpc" },
        "Tags": [ {"Key": "Name", "Value": "ose_node_sg"}, {"Key": "KubernetesCluster", "Value": { "Ref": "AWS::StackName" }}]
      }
    },
    "MasterSG": {
      "Type": "AWS::EC2::SecurityGroup",
      "Properties": {
        "GroupDescription": "Master",
        "VpcId": { "Ref": "Vpc" },
        "Tags": [ {"Key": "Name", "Value": "ose_master_sg"} ]
      }
    },
    "InfraElbEgressHTTP": {
      "Type": "AWS::EC2::SecurityGroupEgress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "InfraElbSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "80",
        "ToPort": "80",
        "DestinationSecurityGroupId": { "Fn::GetAtt": [ "InfraSG", "GroupId" ] }
      }
    },
    "ElasticApi": {
      "Type": "AWS::EC2::SecurityGroupEgress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "InfraElbSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "9200",
        "ToPort": "9200",
        "DestinationSecurityGroupId": { "Fn::GetAtt": [ "InfraSG", "GroupId" ] }
      }
    },
    "ElasticCluster": {
      "Type": "AWS::EC2::SecurityGroupEgress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "InfraElbSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "9300",
        "ToPort": "9300",
        "DestinationSecurityGroupId": { "Fn::GetAtt": [ "InfraSG", "GroupId" ] }
      }
    },
    "LoggingCluster": {
      "Type": "AWS::EC2::SecurityGroupEgress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "InfraElbSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "443",
        "ToPort": "443",
        "DestinationSecurityGroupId": { "Fn::GetAtt": [ "InfraSG", "GroupId" ] }
      }
    },
    "MasterExtElbEgressAPI": {
      "Type": "AWS::EC2::SecurityGroupEgress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterExtElbSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": { "Ref": "MasterApiPort" },
        "ToPort": { "Ref": "MasterApiPort" },
        "DestinationSecurityGroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] }
      }
    },
    "MasterIntElbEgressAPI": {
      "Type": "AWS::EC2::SecurityGroupEgress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterIntElbSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": { "Ref": "MasterApiPort" },
        "ToPort": { "Ref": "MasterApiPort" },
        "DestinationSecurityGroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] }
      }
    },
    "MasterIntElbIngressMasters": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterIntElbSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": { "Ref": "MasterApiPort" },
        "ToPort": { "Ref": "MasterApiPort" },
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] }
      }
    },
    "MasterIntElbIngressNodes": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterIntElbSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": { "Ref": "MasterApiPort" },
        "ToPort": { "Ref": "MasterApiPort" },
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "InfraIngressHTTP": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "InfraSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "80",
        "ToPort": "80",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "InfraElbSG", "GroupId" ] }
      }
    },
    "InfraIngressHTTPS": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "InfraSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "443",
        "ToPort": "443",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "InfraElbSG", "GroupId" ] }
      }
    },
    "NodeIngressMasterKubelet": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "10250",
        "ToPort": "10250",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] }
      }
    },
    "GlusterDaemon": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "24007",
        "ToPort": "24007",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "GlusterManagement": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "24008",
        "ToPort": "24008",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "GlusterSsh": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "2222",
        "ToPort": "2222",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "GlusterNfs": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "49152",
        "ToPort": "49664",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "NodeIngressNodeKubelet": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "10250",
        "ToPort": "10250",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "NodeIngressNodeVXLAN": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] },
        "IpProtocol": "udp",
        "FromPort": "4789",
        "ToPort": "4789",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "NodeIngressSsh": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "22",
        "ToPort": "22",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "BastionSg", "GroupId" ] }
      }
    },
    "MasterIngressIntLB": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": { "Ref": "MasterApiPort" },
        "ToPort": { "Ref": "MasterApiPort" },
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "MasterIntElbSG", "GroupId" ] }
      }
    },
    "MasterIngressExtLB": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": { "Ref": "MasterApiPort" },
        "ToPort": { "Ref": "MasterApiPort" },
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "MasterExtElbSG", "GroupId" ] }
      }
    },
    "MasterIngressNodesDNSUDP": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] },
        "IpProtocol": "udp",
        "FromPort": "8053",
        "ToPort": "8053",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "MasterIngressNodesDNSTCP": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "8053",
        "ToPort": "8053",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "MasterIngressNodesAPITCP": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": { "Ref": "MasterApiPort" },
        "ToPort": { "Ref": "MasterApiPort" },
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "LoggingTCP": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "24224",
        "ToPort": "24224",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "LoggingUDP": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] },
        "IpProtocol": "udp",
        "FromPort": "24224",
        "ToPort": "24224",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "NodeSG", "GroupId" ] }
      }
    },
    "MasterIngressMastersAPITCP": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": { "Ref": "MasterApiPort" },
        "ToPort": { "Ref": "MasterApiPort" },
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] }
      }
    },
    "EtcdIngressEtcd": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "EtcdSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "2379",
        "ToPort": "2379",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "EtcdSG", "GroupId" ] }
      }
    },
    "EtcdIngressMasters": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "EtcdSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "2379",
        "ToPort": "2379",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "MasterSG", "GroupId" ] }
      }
    },
    "EtcdIngressEtcdPeer": {
      "Type": "AWS::EC2::SecurityGroupIngress",
      "Properties":{
        "GroupId": { "Fn::GetAtt": [ "EtcdSG", "GroupId" ] },
        "IpProtocol": "tcp",
        "FromPort": "2380",
        "ToPort": "2380",
        "SourceSecurityGroupId": { "Fn::GetAtt": [ "EtcdSG", "GroupId" ] }
      }
    },
    "MasterIntElb": {
      "Type": "AWS::ElasticLoadBalancing::LoadBalancer",
      "Properties": {
        "CrossZone": "true",
        "ConnectionSettings": {"IdleTimeout" : 300},
        "Tags": [ {"Key": "Name", "Value": "ose_internal_master_elb"} ],
        "HealthCheck": {
          "HealthyThreshold" : "2",
          "Interval" : "5",
          "Target" : { "Ref": "MasterHealthTarget" },
          "Timeout" : "2",
          "UnhealthyThreshold" : "2"
        },
        "Listeners":[
          {
            "InstancePort": { "Ref" : "MasterApiPort" },
            "InstanceProtocol": "TCP",
            "LoadBalancerPort": { "Ref" : "MasterApiPort" },
            "Protocol": "TCP"
          }
        ],
        "Scheme": "internal",
        "SecurityGroups": [ { "Ref": "MasterIntElbSG" } ],
        "Subnets": [
          {"Ref": "PrivateSubnet1"},
          {"Ref": "PrivateSubnet2"},
          {"Ref": "PrivateSubnet3"}
            ],
        "Instances": [
          {"Ref": "Master01"},
          {"Ref": "Master02"},
          {"Ref": "Master03"}
            ]
        }
    },
    "MasterExtElb": {
      "Type": "AWS::ElasticLoadBalancing::LoadBalancer",
      "Properties": {
        "CrossZone": "true",
        "ConnectionSettings": {"IdleTimeout" : 300},
        "Tags": [ {"Key": "Name", "Value": "ose_master_elb"} ],
        "HealthCheck": {
          "HealthyThreshold" : "2",
          "Interval" : "5",
          "Target" : { "Ref": "MasterHealthTarget" },
          "Timeout" : "2",
          "UnhealthyThreshold" : "2"
        },
        "Listeners":[
          {
            "InstancePort": { "Ref" : "MasterApiPort" },
            "InstanceProtocol": "TCP",
            "LoadBalancerPort": { "Ref" : "MasterApiPort" },
            "Protocol": "TCP"
          }
        ],
        "SecurityGroups": [{"Ref": "MasterExtElbSG"}],
        "Subnets": [
          {"Ref": "PublicSubnet1"},
          {"Ref": "PublicSubnet2"},
          {"Ref": "PublicSubnet3"}
            ],
        "Instances": [
          {"Ref": "Master01"},
          {"Ref": "Master02"},
          {"Ref": "Master03"}
            ]
      }
    },
    "InfraElb": {
      "Type": "AWS::ElasticLoadBalancing::LoadBalancer",
      "Properties": {
        "CrossZone": "true",
        "Tags": [ {"Key": "Name", "Value": "ose_router_elb"} ],
        "HealthCheck": {
          "HealthyThreshold" : "2",
          "Interval" : "5",
          "Target" : "TCP:443",
          "Timeout" : "2",
          "UnhealthyThreshold" : "2"
        },
        "Listeners":[
          {
            "InstancePort": "443",
            "InstanceProtocol": "TCP",
            "LoadBalancerPort": "443",
            "Protocol": "TCP"
          },
          {
            "InstancePort": "80",
            "InstanceProtocol": "TCP",
            "LoadBalancerPort": "80",
            "Protocol": "TCP"
          }
        ],
        "SecurityGroups": [ { "Ref": "InfraElbSG" } ],
        "Subnets": [
          {"Ref": "PublicSubnet1"},
          {"Ref": "PublicSubnet2"},
          {"Ref": "PublicSubnet3"}
        	],
        "Instances": [
          {"Ref": "InfraNode01"},
          {"Ref": "InfraNode02"},
          {"Ref": "InfraNode03"}
            ]
      }
    },
    "NodePolicy": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": { "Service": [ "ec2.amazonaws.com" ] },
              "Action": [ "sts:AssumeRole" ]
            }
          ]
        },
        "Policies": [
          {
            "PolicyName": "node-describe",
            "PolicyDocument": {
              "Version" : "2012-10-17",
              "Statement": [
                {
                  "Effect": "Allow",
                  "Action": [
                     "ec2:DescribeInstance*"
                  ],
                  "Resource": "*"
                }
              ]
            }
          }
        ]
      }
    },
    "MasterPolicy": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": { "Service": [ "ec2.amazonaws.com" ] },
              "Action": [ "sts:AssumeRole" ]
            }
          ]
        },
        "Policies": [
          {
            "PolicyName": "master-ec2-all",
            "PolicyDocument": {
              "Version" : "2012-10-17",
              "Statement": [
                {
                  "Effect": "Allow",
                  "Action": [
                     "ec2:DescribeVolume*",
                     "ec2:CreateVolume",
                     "ec2:CreateTags",
                     "ec2:DescribeInstance*",
                     "ec2:AttachVolume",
                     "ec2:DetachVolume",
                     "ec2:DeleteVolume",
                     "ec2:DescribeSubnets",
                     "ec2:CreateSecurityGroup",
                     "ec2:DescribeSecurityGroups",
                     "elasticloadbalancing:DescribeTags",
                     "elasticloadbalancing:CreateLoadBalancerListeners",
                     "ec2:DescribeRouteTables",
                     "elasticloadbalancing:ConfigureHealthCheck",
                     "ec2:AuthorizeSecurityGroupIngress",
                     "elasticloadbalancing:DeleteLoadBalancerListeners",
                     "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
                     "elasticloadbalancing:DescribeLoadBalancers",
                     "elasticloadbalancing:CreateLoadBalancer",
                     "elasticloadbalancing:DeleteLoadBalancer",
                     "elasticloadbalancing:ModifyLoadBalancerAttributes",
                     "elasticloadbalancing:DescribeLoadBalancerAttributes"
                  ],
                  "Resource": "*"
                }
              ]
            }
          }
        ]
      }
    },
    "MasterInstanceProfile": {
      "Type": "AWS::IAM::InstanceProfile",
      "DependsOn": "MasterPolicy",
      "Properties": {
        "Roles": [ { "Ref": "MasterPolicy" } ]
      }
    },
    "NodeInstanceProfile": {
      "Type": "AWS::IAM::InstanceProfile",
      "DependsOn": "NodePolicy",
      "Properties": {
        "Roles": [ { "Ref": "NodePolicy" } ]
      }
    },
    "RegistryBucket": {
    "Type": "AWS::S3::Bucket",
    "Properties" : {
       "BucketName": { "Ref": "S3BucketName"}
                   }
     },
    "Route53Records": {
      "Type": "AWS::Route53::RecordSetGroup",
      "DependsOn": [
        "InfraElb",
        "MasterIntElb",
        "Master01",
        "Master02",
        "Master03",
        "Bastion",
        "MasterExtElb"
      ],
      "Properties": {
        "HostedZoneName": { "Ref": "Route53HostedZone" },
        "RecordSets": [
          {
            "Name":  { "Ref": "MasterClusterPublicHostname" },
            "Type": "A",
            "AliasTarget": {
                "HostedZoneId": { "Fn::GetAtt" : ["MasterExtElb", "CanonicalHostedZoneNameID"] },
                "DNSName": { "Fn::GetAtt" : ["MasterExtElb","CanonicalHostedZoneName"] }
            }
          },
          {
            "Name": { "Ref": "MasterClusterHostname" },
            "Type": "A",
            "AliasTarget": {
                "HostedZoneId": { "Fn::GetAtt" : ["MasterIntElb", "CanonicalHostedZoneNameID"] },
                "DNSName": { "Fn::GetAtt" : ["MasterIntElb","DNSName"] }
            }
          },
          {
            "Name": { "Ref": "AppWildcardDomain" },
            "Type": "A",
            "AliasTarget": {
                "HostedZoneId": { "Fn::GetAtt" : ["InfraElb", "CanonicalHostedZoneNameID"] },
                "DNSName": { "Fn::GetAtt" : ["InfraElb","CanonicalHostedZoneName"] }
            }
          },
          {
            "Name": {"Fn::Join": [".", ["ose-master01",{"Ref": "Route53HostedZone"}]]},
            "Type": "A",
			"TTL": "300",
		    "ResourceRecords": [{ "Fn::GetAtt" : ["Master01", "PrivateIp"] }]
          },
          {
            "Name": {"Fn::Join": [".", ["ose-master02",{"Ref": "Route53HostedZone"}]]},
            "Type": "A",
			"TTL": "300",
		    "ResourceRecords": [{ "Fn::GetAtt" : ["Master02", "PrivateIp"] }]
          },
          {
            "Name": {"Fn::Join": [".", ["ose-master03",{"Ref": "Route53HostedZone"}]]},
            "Type": "A",
			"TTL": "300",
		    "ResourceRecords": [{ "Fn::GetAtt" : ["Master03", "PrivateIp"] }]
          },
          {
            "Name": {"Fn::Join": [".", ["ose-infra-node01",{"Ref": "Route53HostedZone"}]]},
            "Type": "A",
			"TTL": "300",
		    "ResourceRecords": [{ "Fn::GetAtt" : ["InfraNode01", "PrivateIp"] }]
          },
          {
            "Name": {"Fn::Join": [".", ["ose-infra-node02",{"Ref": "Route53HostedZone"}]]},
            "Type": "A",
			"TTL": "300",
		    "ResourceRecords": [{ "Fn::GetAtt" : ["InfraNode02", "PrivateIp"] }]
          },
	  {
            "Name": {"Fn::Join": [".", ["ose-infra-node03",{"Ref": "Route53HostedZone"}]]},
            "Type": "A",
			"TTL": "300",
		    "ResourceRecords": [{ "Fn::GetAtt" : ["InfraNode03", "PrivateIp"] }]
          },
{% for idx in range(1, app_node_count|int + 1) %}
          {
            "Name": {"Fn::Join": [".", ["ose-app-node{{ '%02d' % idx }}",{"Ref": "Route53HostedZone"}]]},
            "Type": "A",
             "TTL": "300",
            "ResourceRecords": [{ "Fn::GetAtt" : ["AppNode{{ '%02d' % idx }}", "PrivateIp"] }]
          },
{% endfor %}
          {
            "Name": {"Fn::Join": [".", ["bastion",{"Ref": "Route53HostedZone"}]]},
            "Type": "A",
                        "TTL": "300",
                    "ResourceRecords": [{ "Fn::GetAtt" : ["Bastion", "PublicIp"] }]
          }
        ]
      }
    },
    "Bastion" : {
       "Type" : "AWS::EC2::Instance",
       "Properties" : {
          "ImageId" : {"Ref": "AmiId"},
          "UserData": {"Ref": "BastionUserData"},
          "KeyName" : {"Ref": "KeyName"},
	  "InstanceType": "t2.micro",
          "SecurityGroupIds": [{ "Fn::GetAtt" : ["BastionSg", "GroupId"] }],
          "SubnetId" : {"Ref": "PublicSubnet1"},
          "Tags": [
            { "Key": "Name",
              "Value": {"Fn::Join": [".", ["bastion",{"Ref": "PublicHostedZone"}]]}
            }
          ],
          "BlockDeviceMappings" : [
			          {
            "DeviceName": "/dev/sda1",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "10",
              "VolumeType": "gp2"
                   }
            }
          ]
       }
    },
    "BastionEip" : {
       "Type" : "AWS::EC2::EIP",
        "Properties" : {
        "Domain" : "vpc"
           }
       },
    "BastionEipAssoc" : {
      "DependsOn": ["Bastion"],
      "Type" : "AWS::EC2::EIPAssociation",
      "Properties" : {
        "InstanceId" : {"Ref" : "Bastion"},
        "AllocationId" : { "Fn::GetAtt" : ["BastionEip", "AllocationId"]}
        }
       },
    "Master01" : {
       "Type" : "AWS::EC2::Instance",
       "DependsOn": ["MasterInstanceProfile"],
       "Properties" : {
          "ImageId" : {"Ref": "AmiId"},
          "UserData": {"Ref": "MasterUserData"},
          "KeyName" : {"Ref": "KeyName"},
	      "InstanceType": {"Ref": "MasterInstanceType"},
		  "SecurityGroupIds": [{ "Fn::GetAtt" : ["NodeSG", "GroupId"] }, { "Fn::GetAtt" : ["MasterSG", "GroupId"] }, { "Fn::GetAtt" : ["EtcdSG", "GroupId"] }],
          "SubnetId" : {"Ref": "PrivateSubnet1"},
          "IamInstanceProfile": { "Ref": "MasterInstanceProfile" },
          "Tags": [
            { "Key": "Name",
              "Value": {"Fn::Join": [".", ["ose-master01",{"Ref": "PublicHostedZone"}]]}
            },
            { "Key": "KubernetesCluster",
              "Value": { "Ref": "AWS::StackName" }
            },
            { "Key": "openshift-role",
              "Value": "master"
            }
          ],
          "BlockDeviceMappings" : [
          {
            "DeviceName": "/dev/sda1",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          },
          {
            "DeviceName": "/dev/xvdb",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": {"Ref": "MasterDockerVolSize"},
              "VolumeType": {"Ref": "MasterDockerVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdc",
            "Ebs": {
              "DeleteOnTermination": "false",
              "VolumeSize": {"Ref": "MasterEtcdVolSize"},
              "VolumeType": {"Ref": "MasterEtcdVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdd",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "5",
              "VolumeType": "gp2"
            }
          }
         ]
     }
  },
    "Master02" : {
       "Type" : "AWS::EC2::Instance",
       "DependsOn": ["MasterInstanceProfile"],
       "Properties" : {
          "ImageId" : {"Ref": "AmiId"},
          "UserData": {"Ref": "MasterUserData"},
          "KeyName" : {"Ref": "KeyName"},
          "InstanceType": {"Ref": "MasterInstanceType"},
          "SecurityGroupIds": [{ "Fn::GetAtt" : ["NodeSG", "GroupId"] }, { "Fn::GetAtt" : ["MasterSG", "GroupId"] }, { "Fn::GetAtt" : ["EtcdSG", "GroupId"] }],
          "SubnetId" : {"Ref": "PrivateSubnet2"},
          "IamInstanceProfile": { "Ref": "MasterInstanceProfile" },
          "Tags": [
            { "Key": "Name",
              "Value": {"Fn::Join": [".", ["ose-master02",{"Ref": "PublicHostedZone"}]]}
            },
            { "Key": "KubernetesCluster",
              "Value": { "Ref": "AWS::StackName" }
            },
            { "Key": "openshift-role",
              "Value": "master"
            }
          ],
          "BlockDeviceMappings" : [
          {
            "DeviceName": "/dev/sda1",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          },
          {
            "DeviceName": "/dev/xvdb",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": {"Ref": "MasterDockerVolSize"},
              "VolumeType": {"Ref": "MasterDockerVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdc",
            "Ebs": {
              "DeleteOnTermination": "false",
              "VolumeSize": {"Ref": "MasterEtcdVolSize"},
              "VolumeType": {"Ref": "MasterEtcdVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdd",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "5",
              "VolumeType": "gp2"
            }
          }
         ]
     }
   },
    "Master03" : {
       "Type" : "AWS::EC2::Instance",
       "DependsOn": ["MasterInstanceProfile"],
       "Properties" : {
          "ImageId" : {"Ref": "AmiId"},
          "UserData": {"Ref": "MasterUserData"},
          "KeyName" : {"Ref": "KeyName"},
	  "InstanceType": {"Ref": "MasterInstanceType"},
	  "SecurityGroupIds": [{ "Fn::GetAtt" : ["NodeSG", "GroupId"] }, { "Fn::GetAtt" : ["MasterSG", "GroupId"] }, { "Fn::GetAtt" : ["EtcdSG", "GroupId"] }],
          "SubnetId" : {"Ref": "PrivateSubnet3"},
          "IamInstanceProfile": { "Ref": "MasterInstanceProfile" },
          "Tags": [
            { "Key": "Name",
              "Value": {"Fn::Join": [".", ["ose-master03",{"Ref": "PublicHostedZone"}]]}
            },
            { "Key": "KubernetesCluster",
              "Value": { "Ref": "AWS::StackName" }
            },
            { "Key": "openshift-role",
              "Value": "master"
            }
          ],
          "BlockDeviceMappings" : [
          {
            "DeviceName": "/dev/sda1",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          },
          {
            "DeviceName": "/dev/xvdb",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": {"Ref": "MasterDockerVolSize"},
              "VolumeType": {"Ref": "MasterDockerVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdc",
            "Ebs": {
              "DeleteOnTermination": "false",
              "VolumeSize": {"Ref": "MasterEtcdVolSize"},
              "VolumeType": {"Ref": "MasterEtcdVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdd",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "5",
              "VolumeType": "gp2"
            }
          }
         ]
     }
   },
    "InfraNode01" : {
       "Type" : "AWS::EC2::Instance",
       "DependsOn": ["NodeInstanceProfile"],
       "Properties" : {
          "ImageId" : {"Ref": "AmiId"},
          "UserData": {"Ref": "NodeUserData"},
          "KeyName" : {"Ref": "KeyName"},
	  "InstanceType": {"Ref": "InfraInstanceType"},
          "SecurityGroupIds": [{ "Fn::GetAtt" : ["NodeSG", "GroupId"] }, { "Fn::GetAtt" : ["InfraSG", "GroupId"] }],
          "SubnetId" : {"Ref": "PrivateSubnet1"},
          "IamInstanceProfile": { "Ref": "NodeInstanceProfile" },
          "Tags": [
            { "Key": "Name",
              "Value": {"Fn::Join": [".", ["ose-infra-node01",{"Ref": "PublicHostedZone"}]]}
            },
            { "Key": "KubernetesCluster",
              "Value": { "Ref": "AWS::StackName" }
            },
            { "Key": "openshift-role",
              "Value": "infra"
            }
          ],
          "BlockDeviceMappings" : [
          {
            "DeviceName": "/dev/sda1",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          },
          {
            "DeviceName": "/dev/xvdb",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": {"Ref": "NodeDockerVolSize"},
              "VolumeType": {"Ref": "NodeDockerVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdc",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          }
         ]
     }
    },
    "InfraNode02" : {
       "Type" : "AWS::EC2::Instance",
       "DependsOn": ["NodeInstanceProfile"],
       "Properties" : {
          "ImageId" : {"Ref": "AmiId"},
          "UserData": {"Ref": "NodeUserData"},
          "KeyName" : {"Ref": "KeyName"},
          "InstanceType": {"Ref": "InfraInstanceType"},
          "SecurityGroupIds": [{ "Fn::GetAtt" : ["NodeSG", "GroupId"] }, { "Fn::GetAtt" : ["InfraSG", "GroupId"] }],
          "SubnetId" : {"Ref": "PrivateSubnet2"},
          "IamInstanceProfile": { "Ref": "NodeInstanceProfile" },
          "Tags": [
            { "Key": "Name",
              "Value": {"Fn::Join": [".", ["ose-infra-node02",{"Ref": "PublicHostedZone"}]]}
            },
            { "Key": "KubernetesCluster",
              "Value": { "Ref": "AWS::StackName" }
            },
            { "Key": "openshift-role",
              "Value": "infra"
            }
          ],
          "BlockDeviceMappings" : [
          {
            "DeviceName": "/dev/sda1",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2" 
            }
          },
          {
            "DeviceName": "/dev/xvdb",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": {"Ref": "NodeDockerVolSize"},
              "VolumeType": {"Ref": "NodeDockerVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdc",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          }
         ]
     }
 },
    "InfraNode03" : {
       "Type" : "AWS::EC2::Instance",
       "DependsOn": ["NodeInstanceProfile"],
       "Properties" : {
          "ImageId" : {"Ref": "AmiId"},
          "UserData": {"Ref": "NodeUserData"},
          "KeyName" : {"Ref": "KeyName"},
          "InstanceType": {"Ref": "InfraInstanceType"},
          "SecurityGroupIds": [{ "Fn::GetAtt" : ["NodeSG", "GroupId"] }, { "Fn::GetAtt" : ["InfraSG", "GroupId"] }],
          "SubnetId" : {"Ref": "PrivateSubnet3"},
          "IamInstanceProfile": { "Ref": "NodeInstanceProfile" },
          "Tags": [
            { "Key": "Name",
              "Value": {"Fn::Join": [".", ["ose-infra-node03",{"Ref": "PublicHostedZone"}]]}
            },
            { "Key": "KubernetesCluster",
              "Value": { "Ref": "AWS::StackName" }
            },
            { "Key": "openshift-role",
              "Value": "infra"
            }
          ],
          "BlockDeviceMappings" : [
          {
            "DeviceName": "/dev/sda1",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          },
          {
            "DeviceName": "/dev/xvdb",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": {"Ref": "NodeDockerVolSize"},
              "VolumeType": {"Ref": "NodeDockerVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdc",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          }
         ]
     }
 },
{% set rotator = 1 %}
{% for idx in range(1, app_node_count|int + 1) %}
{% if rotator == 4 %}
  {% set rotator = 1 %}
{% endif %}
    "AppNode{{ '%02d' % idx }}" : {
       "Type" : "AWS::EC2::Instance",
       "DependsOn": ["NodeInstanceProfile"],
       "Properties" : {
          "ImageId" : {"Ref": "AmiId"},
          "UserData": {"Ref": "NodeUserData"},
          "KeyName" : {"Ref": "KeyName"},
          "InstanceType": {"Ref": "AppNodeInstanceType"},
	  "SecurityGroupIds": [{ "Fn::GetAtt" : ["NodeSG", "GroupId"] }],
          "SubnetId" : {"Ref": "PrivateSubnet{{ rotator }}"},
	  "IamInstanceProfile": { "Ref": "NodeInstanceProfile" },
          "Tags": [
          { "Key": "Name",
            "Value": {"Fn::Join": [".", ["ose-app-node{{ '%02d' % idx }}",{"Ref": "PublicHostedZone"}]]}
          },
            { "Key": "KubernetesCluster",
              "Value": { "Ref": "AWS::StackName" }
            },
            { "Key": "openshift-role",
              "Value": "app"
            }
          ],
          "BlockDeviceMappings" : [
          {
            "DeviceName": "/dev/sda1",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          },
          {
            "DeviceName": "/dev/xvdb",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": {"Ref": "NodeDockerVolSize"},
              "VolumeType": {"Ref": "NodeDockerVolType"}
            }
          },
          {
            "DeviceName": "/dev/xvdc",
            "Ebs": {
              "DeleteOnTermination": "true",
              "VolumeSize": "50",
              "VolumeType": "gp2"
            }
          }
         ]
     }
   },
{% set rotator = rotator + 1 %}
{% endfor %}
    "S3UserName" : {
      "Type" : "AWS::IAM::User",
      "Properties" : {
        "Path" : "/",
        "UserName": { "Ref": "S3User" },
        "Policies" : [ {
          "PolicyName" : "accessalls3",
          "PolicyDocument" : {
            "Version": "2012-10-17",
            "Statement" : [ {
              "Effect" : "Allow",
              "Action" : [ "s3:*" ],
              "Resource": "*"
            } ]
          }
        } ]
      }
    },
    "CFNKeys" : {
      "Type" : "AWS::IAM::AccessKey",
      "Properties" : {
        "UserName" : { "Ref": "S3UserName" }
      }
    }
  },
  "Outputs" : {
    "StackVpc" : {
      "Value" : { "Ref" : "Vpc" },
      "Description" : "VPC that was created"
    },
    "InfraLb" : {
      "Value" : { "Ref" : "InfraElb" },
      "Description" : "Infrastructure ELB name"
    },
    "PrivateSubnet1" : {
      "Value" : { "Ref" : "PrivateSubnet1" },
      "Description" : "Private Subnet 1"
    },
    "PrivateSubnet2" : {
      "Value" : { "Ref" : "PrivateSubnet2" },
      "Description" : "Private Subnet 2"
    },
    "PrivateSubnet3" : {
      "Value" : { "Ref" : "PrivateSubnet3" },
      "Description" : "Private Subnet 3"
    },
    "NodeSGId" : {
      "Value" : { "Fn::GetAtt": [ "NodeSG", "GroupId" ]},
      "Description" : "Node SG id"
    },
    "InfraSGId" : {
      "Value" : { "Fn::GetAtt": [ "InfraSG", "GroupId" ]},
      "Description" : "Infra Node SG id"
    },
    "BastionSGId" : {
      "Value" : { "Fn::GetAtt": [ "BastionSg", "GroupId" ]},
      "Description" : "Bastion SG id"
    },
    "S3UserAccessId" : {
      "Value" : { "Ref" : "CFNKeys" },
      "Description" : "AWSAccessKeyId of user"
    },
    "NodeARN" : {
      "Value" : { "Ref": "NodeInstanceProfile" },
      "Description": "ARN for the Node instance profile"
    },
    "S3UserSecretKey" : {
      "Value" : { "Fn::GetAtt" : ["CFNKeys", "SecretAccessKey"]},
      "Description" : "AWSSecretKey of new S3"
    },
    "S3Bucket" : {
      "Value" : { "Ref" : "RegistryBucket"},
      "Description" : "Name of S3 bucket"
    }
 }
}
