--- AWSTemplateFormatVersion: 2010-09-09 Description: 'Master manifest for ADP Batch usecase' Metadata: AWS::CloudFormation::Interface: ParameterGroups: - Label: default: User-defined variables Parameters: - BatchInputPath - ItemPropertiesInputPath - Label: default: General configuration Parameters: - ADPS3BucketName - ADPS3BucketRegion - BasePlatformVersion - UseCaseVersion Parameters: ADPS3BucketName: Type: String AllowedPattern: ^[0-9a-z]+([0-9a-z-]*[0-9a-z])*$ ConstraintDescription: 'Bucket name can include numbers, lowercase letters, and hyphens (-). It cannot start or end with a hyphen (-).' Description: 'ADP bucket name with cloudformation scripts can include numbers, lowercase letters, and hyphens (-). It cannot start or end with a hyphen (-).' Default: 'griddynamics-analytical-data-platform-releases' ADPS3BucketRegion: Type: String Description: 'The AWS Region where the ADP S3 bucket (ADPS3BucketName) is hosted.' Default: 'us-east-1' BasePlatformVersion: Type: String Description: 'BasePlatform build version.' Default: '2021-05-19' UseCaseVersion: Type: String Description: 'UseCase build version.' Default: '2021-02-16' BatchInputPath: Type: String Description: 'Input file for aws_analytical_batch_process dag.' Default: 'griddynamics-analytical-data-platform-releases/input/batch/transaction-events.csv' ItemPropertiesInputPath: Type: String Description: 'Input file for aws_item_properties_batch_process dag.' Default: 'griddynamics-analytical-data-platform-releases/input/batch/item_properties/*' Conditions: RunningInAnotherRegion: !Not [!Equals [!Ref ADPS3BucketRegion, !Ref "AWS::Region"]] Resources: BasePlatformUtils: Type: AWS::CloudFormation::Stack Properties: TemplateURL: !Sub 'https://${ADPS3BucketName}.s3.${ADPS3BucketRegion}.${AWS::URLSuffix}/platform/${BasePlatformVersion}/cloudformation/l0/base/utils.yaml' Parameters: ResourcePrefix: !ImportValue ADPResourcePrefix ADPS3BucketName: !Sub - '${BucketName}' - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName] BasePlatformVersion: !Ref BasePlatformVersion BasePlatformVPCOutputs: DependsOn: BasePlatformUtils Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformOutputsLambdaArn StackName: "VPCStack0" BpStateBucket: !Sub - "${ResourcePrefix}-platform-state" - ResourcePrefix: !ImportValue ADPResourcePrefix BasePlatformLambdaFunctionsOutputs: DependsOn: BasePlatformVPCOutputs Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformOutputsLambdaArn StackName: "BasePlatformLambdaFunctions" BpStateBucket: !Sub - "${ResourcePrefix}-platform-state" - ResourcePrefix: !ImportValue ADPResourcePrefix InitDeploymentBucketOutputs: Condition: RunningInAnotherRegion DependsOn: BasePlatformLambdaFunctionsOutputs Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformOutputsLambdaArn StackName: "InitDeploymentBucket" BpStateBucket: !Sub - "${ResourcePrefix}-platform-state" - ResourcePrefix: !ImportValue ADPResourcePrefix BasePlatformCloudMapOutputs: DependsOn: BasePlatformLambdaFunctionsOutputs Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformOutputsLambdaArn StackName: "CloudMapStack" BpStateBucket: !Sub - "${ResourcePrefix}-platform-state" - ResourcePrefix: !ImportValue ADPResourcePrefix CopyDeploymentFiles: Condition: RunningInAnotherRegion DependsOn: InitDeploymentBucketOutputs Type: Custom::CopyFiles Properties: ServiceToken: !GetAtt InitDeploymentBucketOutputs.CopyFilesFunctionArn DestBucket: !GetAtt InitDeploymentBucketOutputs.FilesBucket SourceBucket: !Ref ADPS3BucketName Prefix: !Sub 'applications/adp/${UseCaseVersion}/' BasePlatformPrepareKubernetes: DependsOn: BasePlatformLambdaFunctionsOutputs Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName BpGdComponent: "batch_processing" BpAwsComponent: "PrepareKubernetes" BpReferenceComponentName: "PrepareKubernetesStack" BasePlatformKubernetes: DependsOn: BasePlatformPrepareKubernetes Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName BpGdComponent: "batch_processing" BpAwsComponent: "Kubernetes" BpReferenceComponentName: "KubernetesStack" BasePlatformPostKubernetes: DependsOn: BasePlatformKubernetes Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName BpGdComponent: "batch_processing" BpAwsComponent: "PostKubernetes" BpReferenceComponentName: "PostKubernetesStack" BasePlatformKubernetesExtra: DependsOn: BasePlatformPostKubernetes Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName BpGdComponent: "batch_processing" BpAwsComponent: "KubernetesExtra" BpReferenceComponentName: "KubernetesExtraStack" BasePlatformAirflow: DependsOn: BasePlatformKubernetesExtra Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName BpGdComponent: "batch_processing" BpAwsComponent: "Airflow" BpReferenceComponentName: "AirflowStack" BasePlatformRedshift: DependsOn: BasePlatformAirflow Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName BpGdComponent: "batch_processing" BpAwsComponent: "Redshift" BpReferenceComponentName: "RedshiftStack" BasePlatformEMR: DependsOn: BasePlatformRedshift Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformUtils.Outputs.BasePlatformLambdaOperateArn BpStateBucket: !GetAtt BasePlatformUtils.Outputs.BasePlatformStateBucketName BpGdComponent: "batch_processing" BpAwsComponent: "EMR" BpReferenceComponentName: "EMRStack" BatchTargetBucket: Type: AWS::S3::Bucket DeletionPolicy: Delete Properties: BucketName: !Sub - '${ResourcePrefix}-batch-${StackPrefix}' - { ResourcePrefix: !ImportValue ADPResourcePrefix, StackPrefix: !Select [ 2, !Split [ "/", !Ref "AWS::StackId" ] ] } AccessControl: "Private" EmptyBucketLambdaExec: Type: "Custom::EmptyBucketLambda" Properties: ServiceToken: !GetAtt BasePlatformLambdaFunctionsOutputs.EmptyBucketLambdaArn BucketToEmpty: !Ref BatchTargetBucket DagsCopyExecutionRole: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: [lambda.amazonaws.com] Action: ['sts:AssumeRole'] ManagedPolicyArns: - "arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess" - "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" Policies: - PolicyName: root PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - 'cloudformation:DescribeStackResource' - 'cloudformation:DescribeStackEvents' - 'cloudformation:DescribeStacks' - 'cloudformation:GetTemplateSummary' - 'cloudformation:ListStackResources' - 'ec2:DescribeNetworkInterfaces' - 'ec2:DeleteNetworkInterface' - 'ec2:CreateNetworkInterface' - 'servicediscovery:DeleteService' - 'servicediscovery:RegisterInstance' - 'servicediscovery:DeregisterInstance' - 'servicediscovery:UpdateService' - 'servicediscovery:GetService' - 'servicediscovery:DiscoverInstances' Resource: - '*' - Effect: Allow Action: 'servicediscovery:*' Resource: !Sub - 'arn:aws:servicediscovery:::namespace/${CloudMapNamespaceId}' - CloudMapNamespaceId: !GetAtt BasePlatformCloudMapOutputs.CloudMapNamespaceId - Effect: Allow Action: 's3:*' Resource: - !Sub - 'arn:${AWS::Partition}:s3:::${BucketName}' - BucketName: !If [ RunningInAnotherRegion, !GetAtt InitDeploymentBucketOutputs.FilesBucket, !Ref ADPS3BucketName ] - !Sub - 'arn:${AWS::Partition}:s3:::${BucketName}/*' - BucketName: !If [ RunningInAnotherRegion, !GetAtt InitDeploymentBucketOutputs.FilesBucket, !Ref ADPS3BucketName ] - !Sub - 'arn:${AWS::Partition}:s3:::${SyncerDagS3Bucket}' - SyncerDagS3Bucket: !GetAtt BasePlatformAirflow.SyncerDagS3Bucket - !Sub - 'arn:${AWS::Partition}:s3:::${SyncerDagS3Bucket}/*' - SyncerDagS3Bucket: !GetAtt BasePlatformAirflow.SyncerDagS3Bucket - Effect: Allow Action: - 'cloudformation:UpdateStack' - 'cloudformation:CreateChangeSet' Resource: - !Sub 'arn:aws:cloudformation:*:${AWS::AccountId}:stack/{AWS::StackName}/*' - !Sub 'arn:aws:cloudformation:*:${AWS::AccountId}:stack/kuber-*/*' DagsCopyLambdaSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: GroupDescription: Dummy SG for dags initializer lambda function VpcId: !GetAtt BasePlatformVPCOutputs.VPCID SecurityGroupEgress: - CidrIp: 0.0.0.0/0 FromPort: -1 IpProtocol: '-1' ToPort: -1 DagsCopyLambdaFunction: Type: AWS::Lambda::Function Properties: Code: S3Bucket: !Sub - '${BucketName}' - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName] S3Key: !Sub 'platform/${BasePlatformVersion}/functions/packages/batchInit/lambda.zip' Handler: batch_s3_init.lambda_handler Runtime: python3.7 Timeout: 120 Role: !GetAtt DagsCopyExecutionRole.Arn VpcConfig: SecurityGroupIds: [!Ref DagsCopyLambdaSecurityGroup] SubnetIds: - !GetAtt BasePlatformVPCOutputs.PrivateSubnet1AID - !GetAtt BasePlatformVPCOutputs.PrivateSubnet2AID - !GetAtt BasePlatformVPCOutputs.PrivateSubnet3AID GetSecretExecution: Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt BasePlatformLambdaFunctionsOutputs.BPGetSecretLambdaArn Name: !GetAtt BasePlatformAirflow.AirflowSecret DagsInitLambda: DependsOn: BasePlatformEMR Type: AWS::CloudFormation::CustomResource Properties: ServiceToken: !GetAtt DagsCopyLambdaFunction.Arn AirflowServiceName: "airflow" AirflowUsername: !GetAtt GetSecretExecution.username AirflowPassword: !GetAtt GetSecretExecution.password SourceBucket: !Sub - '${BucketName}' - BucketName: !If [RunningInAnotherRegion, !ImportValue ADPS3BucketAnotherRegion, !Ref ADPS3BucketName] SourcePrefix: !Sub "applications/adp/${UseCaseVersion}/dags/" TargetBucket: !GetAtt BasePlatformAirflow.SyncerDagS3Bucket TargetPrefix: "dags" Namespace: !ImportValue ADPResourcePrefix AirflowProperties: { "discovery": { "aws_conn_id": "adp_aws_default", "class": 'com.griddynamics.platform.aws.batch.Main', "namespace": !ImportValue ADPResourcePrefix, "region": !Ref 'AWS::Region' }, "batch_process": { "class": 'com.griddynamics.platform.aws.batch.ClickstreamHistory', "csv-header": 'true', "err-item-id": '5206', "s3_path_to_jar": !Sub "s3://${ADPS3BucketName}/applications/adp/${UseCaseVersion}/jars/batch-pipeline-app.jar", "source-file-path": !Ref BatchInputPath, "target-table": 'public.adp_batch_history', "target-bucket": !Ref BatchTargetBucket }, "item_properties_process": { "class": 'com.griddynamics.platform.aws.batch.ItemPropertiesLoad', "csv-header": 'true', "s3_path_to_jar": !Sub "s3://${ADPS3BucketName}/applications/adp/${UseCaseVersion}/jars/batch-pipeline-app.jar", "source-file-path": !Ref ItemPropertiesInputPath, "target-table": 'public.item_properties', "target-bucket": !Ref BatchTargetBucket }, "batch_deduplication": { "class": 'com.griddynamics.platform.aws.batch.Deduplication', "datetime-column": 'processingtime', "s3_path_to_jar": !Sub "s3://${ADPS3BucketName}/applications/adp/${UseCaseVersion}/jars/batch-pipeline-app.jar", "target-table": 'public.item_properties', "target-bucket": !Ref BatchTargetBucket, "uniq-column": 'itemId' } } AirflowConnections: { "adp_aws_default": { "extra": { "region_name": !Ref 'AWS::Region' } } } Objects: - batch_dags.zip Outputs: AirflowELB: Value: !GetAtt BasePlatformAirflow.AirflowELB Description: Airflow service ELB address (should be enabled in Base Platform first) AirflowFQDN: Value: !GetAtt BasePlatformAirflow.AirflowFQDN Description: Airflow service DNS address (private zone will not be accessible form outside) AirflowSecret: Value: !GetAtt BasePlatformAirflow.AirflowSecret Description: Airflow service secret BatchTargetBucket: Description: Target bucket for Batch usecase. Value: !Ref BatchTargetBucket