---
AWSTemplateFormatVersion: 2010-09-09
Description: 'Master manifest for ADP Data Quality usecase'
Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: General configuration
        Parameters:
          - ADPS3BucketName
          - ADPS3BucketRegion
          - BasePlatformVersion
          - UseCaseVersion
          - DnsZoneType
          - EnableExternalAccess

Parameters:
  ADPS3BucketName:
    Type: String
    AllowedPattern: ^[0-9a-z]+([0-9a-z\.\-]*[0-9a-z])*$
    ConstraintDescription:
      'Bucket name can include numbers, lowercase letters, and hyphens (-).
       It cannot start or end with a hyphen (-).'
    Description:
      'ADP bucket name with cloudformation scripts can include numbers,
      lowercase letters, and hyphens (-). It cannot start or end with a hyphen (-).'
    Default: 'griddynamics-analytical-data-platform-releases'
  ADPS3BucketRegion:
    Type: String
    Description: 'The AWS Region where the ADP S3 bucket (ADPS3BucketName) is hosted.'
    Default: 'us-east-1'
  BasePlatformVersion:
    Type: String
    Description: 'BasePlatform build version.'
    Default: '2023-02-16-marketplace'
  UseCaseVersion:
    Type: String
    Description: 'UseCase build version.'
    Default: '2023-02-16-marketplace'
  EnableExternalAccess:
    Type: String
    Description:
      'Allow or deny access to UI from outside of VPC (disables ELB for EKS-based applications).
      Disabled by default to minimize unauthorized access.'
    AllowedValues: [ "Enabled", "Disabled" ]
    Default: 'Disabled'
  DnsZoneType:
    Type: String
    Description:
      'Should match with ADP Base Platform settings. Choose public if you already delegated responsibility for the subdomain to Route 53. This subdomain
      should be configured in Route53HostedZoneName. Otherwise chooose a private DNS zone.'
    AllowedValues: [ "public", "private" ]
    Default: 'private'

Conditions:
  RunningInAnotherRegion: !Not [!Equals [!Ref ADPS3BucketRegion, !Ref "AWS::Region"]]
  EnableExternalAccessCondition: !And
    - !Equals [!Ref DnsZoneType, 'private']
    - !Equals [!Ref EnableExternalAccess, 'Enabled']

Resources:
  BasePlatformUtils:
    Type: AWS::CloudFormation::Stack
    Properties:
      Tags:
        - Key: 'Name'
          Value: 'BasePlatformUtils'
        - Key: 'Owner'
          Value: !ImportValue ADPPlatformOwner
        - Key: 'Project'
          Value: 'ADP'
      TemplateURL: !Sub 'https://${ADPS3BucketName}.s3.${ADPS3BucketRegion}.${AWS::URLSuffix}/platform/${BasePlatformVersion}/cloudformation/l0/base/utils.yaml'
      Parameters:
        ResourcePrefix: !ImportValue ADPResourcePrefix
        ADPS3BucketName: !Sub
          - '${BucketName}'
          - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName]
        BasePlatformVersion: !Ref BasePlatformVersion

  BasePlatformInputs:
    DependsOn: BasePlatformUtils
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformInputsLambdaArn
      BpStateBucket: !Sub
        - "${ResourcePrefix}-platform-state-${AWS::AccountId}"
        - ResourcePrefix: !ImportValue ADPResourcePrefix

  BasePlatformOutputs:
    DependsOn: BasePlatformInputs
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformOutputsLambdaArn
      BpStateBucket: !Sub
        - "${ResourcePrefix}-platform-state-${AWS::AccountId}"
        - ResourcePrefix: !ImportValue ADPResourcePrefix

  BasePlatformVPCOutputs:
    DependsOn: BasePlatformOutputs
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformOutputsLambdaArn
      StackName: "VPCStack"
      BpStateBucket: !Sub
        - "${ResourcePrefix}-platform-state-${AWS::AccountId}"
        - ResourcePrefix: !ImportValue ADPResourcePrefix

  BasePlatformCloudMapOutputs:
    DependsOn: BasePlatformVPCOutputs
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformOutputsLambdaArn
      StackName: "CloudMapStack"
      BpStateBucket: !Sub
        - "${ResourcePrefix}-platform-state-${AWS::AccountId}"
        - ResourcePrefix: !ImportValue ADPResourcePrefix

  BasePlatformLambdaFunctionsOutputs:
    DependsOn: BasePlatformCloudMapOutputs
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformOutputsLambdaArn
      StackName: "BasePlatformLambdaFunctions"
      BpStateBucket: !Sub
        - "${ResourcePrefix}-platform-state-${AWS::AccountId}"
        - ResourcePrefix: !ImportValue ADPResourcePrefix

  InitDeploymentBucketOutputs:
    Condition: RunningInAnotherRegion
    DependsOn: BasePlatformLambdaFunctionsOutputs
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformOutputsLambdaArn
      StackName: "InitDeploymentBucket"
      BpStateBucket: !Sub
        - "${ResourcePrefix}-platform-state-${AWS::AccountId}"
        - ResourcePrefix: !ImportValue ADPResourcePrefix

  CopyDeploymentFiles:
    Condition: RunningInAnotherRegion
    DependsOn: InitDeploymentBucketOutputs
    Type: Custom::CopyFiles
    Properties:
      ServiceToken: !GetAtt InitDeploymentBucketOutputs.CopyFilesFunctionArn
      DestBucket: !GetAtt InitDeploymentBucketOutputs.FilesBucket
      SourceBucket: !Ref ADPS3BucketName
      Prefix: !Sub 'applications/adp/${UseCaseVersion}/'

  BasePlatformKubernetes:
    DependsOn: BasePlatformLambdaFunctionsOutputs
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "Kubernetes"
      BpReferenceComponentName: "KubernetesStack"

  BasePlatformKubernetesExtra:
    DependsOn: BasePlatformKubernetes
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "KubernetesExtra"
      BpReferenceComponentName: "KubernetesExtraStack"

  BasePlatformElastic:
    DependsOn: BasePlatformKubernetesExtra
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "Elastic"
      BpReferenceComponentName: "ElasticStack"

  BasePlatformKibana:
    DependsOn: BasePlatformElastic
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "Kibana"
      BpReferenceComponentName: "KibanaStack"

  BasePlatformEMR:
    DependsOn: BasePlatformKibana
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "EMR"
      BpReferenceComponentName: "EMRStack"

  BasePlatformAirflow:
    DependsOn: BasePlatformEMR
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "Airflow"
      BpReferenceComponentName: "AirflowStack"

  BasePlatformRedshift:
    DependsOn: BasePlatformAirflow
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "Redshift"
      BpReferenceComponentName: "RedshiftStack"

  BasePlatformZooKeeper:
    DependsOn: BasePlatformRedshift
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "ZooKeeper"
      BpReferenceComponentName: "ZooKeeperStack"

  BasePlatformKafka:
    DependsOn: BasePlatformZooKeeper
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "Kafka"
      BpReferenceComponentName: "KafkaStack"

  BasePlatformCassandra:
    DependsOn: BasePlatformKafka
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "Cassandra"
      BpReferenceComponentName: "CassandraStack"

  BasePlatformAtlas:
    DependsOn:
      - BasePlatformCassandra
      - BasePlatformZooKeeper
      - BasePlatformKafka
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn
      BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName
      BpGdComponent: "dataquality"
      BpAwsComponent: "Atlas"
      BpReferenceComponentName: "AtlasStack"

  GetAirflowSecretExecution:
    DependsOn:
      - BasePlatformLambdaFunctionsOutputs
      - BasePlatformAirflow
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformLambdaFunctionsOutputs.BPGetSecretLambdaArn
      Name: !GetAtt BasePlatformAirflow.AirflowSecret

  GetAtlasSecretExecution:
    DependsOn:
      - BasePlatformLambdaFunctionsOutputs
      - BasePlatformAtlas
    Type: AWS::CloudFormation::CustomResource
    Properties:
      ServiceToken: !GetAtt BasePlatformLambdaFunctionsOutputs.BPGetSecretLambdaArn
      Name: !GetAtt BasePlatformAtlas.AtlasSecret

  AtlasLambdaSecurityGroup:
    DependsOn: BasePlatformAtlas
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Dummy SG for Atlas lambda functions
      VpcId: !GetAtt BasePlatformVPCOutputs.VPCID

  DQAtlasInitRole:
    DependsOn: BasePlatformAtlas
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          - Action: ['sts:AssumeRole']
            Effect: Allow
            Principal:
              Service: [lambda.amazonaws.com]
        Version: '2012-10-17'
      Path: /
      Policies:
        - PolicyName: !Sub
            - ${ResourcePrefix}-DQAtlasInitRole-LambdaRole
            - ResourcePrefix: !ImportValue ADPResourcePrefix
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:PutLogEvents'
                Effect: Allow
                Resource: !Sub "arn:${AWS::Partition}:logs:*:*:*"
              - Action:
                  - "servicediscovery:ListServices"
                  - "servicediscovery:ListOperations"
                  - "servicediscovery:GetOperation"
                  - "servicediscovery:DiscoverInstances"
                  - "servicediscovery:ListNamespaces"
                  - "servicediscovery:GetInstancesHealthStatus"
                  - "servicediscovery:GetNamespace"
                  - "servicediscovery:GetInstance"
                  - "servicediscovery:GetService"
                  - "servicediscovery:ListInstances"
                  - "ec2:CreateNetworkInterface"
                  - "ec2:DescribeNetworkInterfaces"
                  - "ec2:DeleteNetworkInterface"
                Effect: Allow
                Resource: "*"
              - Effect: Allow
                Action: s3:GetObject
                Resource: !Sub
                  - "arn:${AWS::Partition}:s3:::${BucketName}/applications/adp/${UseCaseVersion}/dags/*"
                  - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName]
              - Effect: Allow
                Action: 's3:*'
                Resource:
                  - !Sub
                    - 'arn:${AWS::Partition}:s3:::${BucketName}'
                    - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName]
                  - !Sub
                    - 'arn:${AWS::Partition}:s3:::${BucketName}/*'
                    - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName]
                  - !Sub
                    - 'arn:${AWS::Partition}:s3:::${SyncerDagS3Bucket}'
                    - SyncerDagS3Bucket: !GetAtt BasePlatformAirflow.SyncerDagS3Bucket
                  - !Sub
                    - 'arn:${AWS::Partition}:s3:::${SyncerDagS3Bucket}/*'
                    - SyncerDagS3Bucket: !GetAtt BasePlatformAirflow.SyncerDagS3Bucket

  AtlasInitLambda:
    DependsOn: DQAtlasInitRole
    Type: "AWS::Lambda::Function"
    Properties:
      Handler: data_quality_s3_init.lambda_handler
      MemorySize: 128
      Role: !GetAtt DQAtlasInitRole.Arn
      Runtime: python3.7
      Timeout: 900
      Code:
        S3Bucket: !Sub
          - '${BucketName}'
          - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName]
        S3Key: !Sub 'platform/${BasePlatformVersion}/functions/packages/DataQualityInit/lambda.zip'
      VpcConfig:
        SecurityGroupIds: [!Ref AtlasLambdaSecurityGroup]
        SubnetIds:
          - !GetAtt BasePlatformVPCOutputs.PrivateSubnet1AID
          - !GetAtt BasePlatformVPCOutputs.PrivateSubnet2AID
          - !GetAtt BasePlatformVPCOutputs.PrivateSubnet3AID
      Tags:
        - Key: 'Name'
          Value: 'AtlasInitLambda'
        - Key: 'Owner'
          Value: !ImportValue ADPPlatformOwner
        - Key: 'Project'
          Value: 'ADP'

  AtlasInitLambdaExec:
    DependsOn: AtlasInitLambda
    Type: "Custom::AtlasInitLambda"
    Properties:
      ServiceToken: !GetAtt AtlasInitLambda.Arn
      Namespace: !ImportValue ADPResourcePrefix
      AirflowServiceName: airflow
      AirflowUsername: !GetAtt GetAirflowSecretExecution.username
      AirflowPassword: !GetAtt GetAirflowSecretExecution.password
      TargetBucket: !GetAtt BasePlatformAirflow.SyncerDagS3Bucket
      TargetPrefix: "dags"
      SourceBucket: !Sub
        - '${BucketName}'
        - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName]
      SourcePrefix: !Sub "applications/adp/${UseCaseVersion}/dags/"
      Objects:
        - dq_dags.zip
      AirflowProperties: !Sub
          - |
            {
            "atlas_discovery": {
              "namespace": "${Namespace}",
              "region_name": "${AWS::Region}"
              },
            "atlas_aws_secret_name": "${AtlasAwsSecretName}",
            "atlas_lineage_conf": {
              "atlas_enable_lineage": "True",
              "atlas_create_if_not_exists": "True",
              "atlas_fail_if_lineage_error": "False",
              "atlas_timeout": "600"
              },
            "dq_config": {
              "aws_region_name": "${AWS::Region}",
              "source_data": "s3://adp-rnd-dq-dataset/",
              "s3_bucket_name": "${BucketName}/applications/adp/${UseCaseVersion}/jobs/spark/",
              "redshift_secret_name": "${RedshiftSecretName}",
              "emr_cluster_id": "${EmrClusterId}",
              "atlas_secret_name": "${AtlasSecretName}"
              },
            "dq_retail_data_config": {
              "aws_region_name": "${AWS::Region}",
              "source_data": "s3://adp-rnd-dq-dataset/retail_data/",
              "source_data_sensor_mask": "*.csv",
              "s3_bucket_name": "${BucketName}/applications/adp/${UseCaseVersion}/jobs/spark/",
              "redshift_secret_name": "${RedshiftSecretName}",
              "emr_cluster_id": "${EmrClusterId}",
              "atlas_secret_name": "${AtlasSecretName}",
              "spark_script": "process_retail_data_spark.py",
              "deps_filename": "dependencies.zip",
              "soda_spark_conf_filename": "soda_retail_data_spark_tests.yml",
              "soda_airflow_conf_filename": "soda_retail_data_airflow_tests.yml",
              "redshift_table": "retail_data_processed",
              "redshift_schema": "public",
              "es_connection_id": "elasticsearch_default",
              "es_index": "dq_retail_data",
              "redshift_connection_id": "redshift_default"
              }
            }
          - { Namespace: !ImportValue ADPResourcePrefix,
              AtlasAwsSecretName: !Join ['', [!ImportValue ADPResourcePrefix, '-atlas-secret' ]],
              BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName],
              RedshiftSecretName: !GetAtt BasePlatformRedshift.RedshiftSecret,
              EmrClusterId: !GetAtt BasePlatformEMR.ClusterID,
              AtlasSecretName: !GetAtt BasePlatformAtlas.AtlasSecret }
      AirflowConnections:
        {
        "adp_aws_default": {
            "extra": {
              "region_name": !Ref 'AWS::Region'
            }
            },
        "atlas_default": {
          "conn_type": "http",
          "host": !GetAtt GetAtlasSecretExecution.host,
          "port": !GetAtt GetAtlasSecretExecution.port,
          "login": !GetAtt GetAtlasSecretExecution.username,
          "password": !GetAtt GetAtlasSecretExecution.password
          },
        "elasticsearch_default": {
          "conn_type": "https",
          "host": !GetAtt BasePlatformElastic.ElasticSearchFQDN,
          "port": "443"
          }
        }

  KibanaLambdaSecurityGroup:
    DependsOn: BasePlatformKibana
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: Dummy SG for Kibana lambda functions
      VpcId: !GetAtt BasePlatformVPCOutputs.VPCID

  KibanaInitLambda:
    DependsOn: KibanaLambdaSecurityGroup
    Type: "AWS::Lambda::Function"
    Properties:
      Handler: kibanadashboard.lambda_handler
      MemorySize: 128
      Role: !GetAtt DQAtlasInitRole.Arn
      Runtime: python3.8
      Timeout: 900
      Code:
        S3Bucket: !Sub
          - '${BucketName}'
          - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName]
        S3Key: !Sub 'platform/${BasePlatformVersion}/functions/packages/kibanadashboard/lambda.zip'
      VpcConfig:
        SecurityGroupIds: [!Ref KibanaLambdaSecurityGroup]
        SubnetIds:
          - !GetAtt BasePlatformVPCOutputs.PrivateSubnet1AID
          - !GetAtt BasePlatformVPCOutputs.PrivateSubnet2AID
          - !GetAtt BasePlatformVPCOutputs.PrivateSubnet3AID
      Tags:
        - Key: 'Name'
          Value: 'KibanaInitLambda'
        - Key: 'Owner'
          Value: !ImportValue ADPPlatformOwner
        - Key: 'Project'
          Value: 'ADP'

  KibanaInitLambdaExec:
    DependsOn: KibanaInitLambda
    Type: "Custom::AtlasInitLambda"
    Properties:
      ServiceToken: !GetAtt KibanaInitLambda.Arn
      KibanaHost: !GetAtt BasePlatformKibana.KibanaHost

Outputs:
  AirflowELB:
    Condition: EnableExternalAccessCondition
    Value: !GetAtt BasePlatformAirflow.AirflowELB
    Description: Airflow service ELB address (should be enabled in Base Platform first)

  AirflowFQDN:
    Value: !GetAtt BasePlatformAirflow.AirflowFQDN
    Description: Airflow service DNS address (private zone will not be accessible form outside)

  AirflowSecret:
    Value: !GetAtt BasePlatformAirflow.AirflowSecret
    Description: Airflow service secret

  AtlasELB:
    Condition: EnableExternalAccessCondition
    Value: !GetAtt BasePlatformAtlas.AtlasELB
    Description: Atlas service ELB address (will not work if external access is disabled)

  AtlasFQDN:
    Value: !GetAtt BasePlatformAtlas.AtlasFQDN
    Description: Atlas service DNS address (private zone will not be accessible form outside)

  AtlasSecret:
    Value: !GetAtt BasePlatformAtlas.AtlasSecret
    Description: Atlas service secret

  ElasticSearchELB:
    Condition: EnableExternalAccessCondition
    Value: !GetAtt BasePlatformElastic.ElasticSearchELB
    Description: ElasticSearch service ELB address (should be enabled in Base Platform first)

  ElasticSearchFQDN:
    Value: !GetAtt BasePlatformElastic.ElasticSearchFQDN
    Description: ElasticSearch service DNS address (private zone will not be accessible form outside)

  KibanaELB:
    Condition: EnableExternalAccessCondition
    Value: !GetAtt BasePlatformKibana.KibanaELB
    Description: Kibana service ELB address (should be enabled in Base Platform first)

  KibanaFQDN:
    Value: !GetAtt BasePlatformKibana.KibanaFQDN
    Description: Kibana service DNS address (private zone will not be accessible form outside)
