serverless.yml

service: sls-data-pipelines-batch-processing

package:
  individually: true

plugins:
  - serverless-pseudo-parameters
  - serverless-iam-roles-per-function
  - serverless-python-requirements
  - serverless-step-functions

custom:
  lumigo:
    token: ${ssm:/lumigo/token}
  S3Prefix: "destination-parquet/"
  S3BatchResultsPrefix: "transformed-parquet-athena/"
  stage: ${opt:stage, self:provider.stage}
  AthenaWorkGroup: "primary"
  Database: "traffic"
  DataCatalog: "AwsDataCatalog"

provider:
  name: aws
  stage: dev
  runtime: python3.8
  region: eu-west-1
  profile: default
  httpApi:
    cors:
      allowedOrigins:
        - "*"
      allowedMethods:
        - GET
        - OPTIONS
  reservedConcurrency: 1
  memorySize: 512
  versionFunctions: false
  timeout: 30
  iamRoleStatements:
    - Effect: "Allow"
      Action:
        - "logs:CreateLogGroup"
        - "logs:CreateLogStream"
        - "logs:PutLogEvents"
      Resource: "*"
    - Effect: "Allow"
      Action:
        - "states:ListStateMachines"
      Resource: "*"
    - Effect: "Allow"
      Action:
        - "states:StartExecution"
      Resource: "arn:aws:states:#{AWS::Region}:#{AWS::AccountId}:BatchProcessingStateMachine"
  environment:
    LUMIGO_TRACER_TOKEN: ${ssm:/lumigo/token}
    LUMIGO_ORIGINAL_HANDLER: handler.handle

functions:
  RunDataCrawler:
    #    handler: src/RunDataCrawler/handler.handle
    handler: /opt/python/lumigo_tracer._handler
    module: src/RunDataCrawler
    timeout: 30
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:134
    environment:
      CRAWLER_NAME: !Ref DeliveryBucketCrawler
    iamRoleStatementsName: sls-batch-${opt:stage, self:provider.stage}-RunDataCrawler-lambdaRole
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - glue:StartCrawler
        Resource: "*"

  GetCrawlerState:
    #    handler: src/GetCrawlerState/handler.handle
    handler: /opt/python/lumigo_tracer._handler
    module: src/GetCrawlerState
    timeout: 30
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:134
    environment:
      CRAWLER_NAME: !Ref DeliveryBucketCrawler
    iamRoleStatementsName: sls-batch-${opt:stage, self:provider.stage}-GetCrawlerState-lambdaRole
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - glue:GetCrawler
        Resource: "*"

  RunETLInsertAthena:
    #    handler: src/RunETLInsertAthena/handler.handle
    handler: /opt/python/lumigo_tracer._handler
    module: src/RunETLInsertAthena
    timeout: 30
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:134
    environment:
      QUERY: ${file(queries/InsertETL.sql)}
      DB: ${self:custom.Database}
      CATALOG: ${self:custom.DataCatalog}
      WORKGROUP: ${self:custom.AthenaWorkGroup}
      OUTPUT: "s3://sls-data-pipelines-dev-deliverybucket-1hs2feakqz1ca/batch-transformed/temp"
    iamRoleStatementsName: sls-batch-${opt:stage, self:provider.stage}-RunETLInsertAthena-lambdaRole
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - athena:StartQueryExecution
        Resource:
          Fn::Join:
            - ""
            - - "arn:aws:athena:#{AWS::Region}:#{AWS::AccountId}:workgroup/"
              - ${self:custom.AthenaWorkGroup}
      - Effect: "Allow"
        Action:
          - s3:GetBucketLocation
          - s3:GetObject
          - s3:ListBucket
          - s3:ListBucketMultipartUploads
          - s3:ListMultipartUploadParts
          - s3:AbortMultipartUpload
          - s3:PutObject
        Resource:
          - Fn::Join:
              - ""
              - - "arn:aws:s3:::"
                - !ImportValue sls-data-pipelines-dev-TrafficEventsDeliveryBucketName
          - Fn::Join:
              - ""
              - - "arn:aws:s3:::"
                - !ImportValue sls-data-pipelines-dev-TrafficEventsDeliveryBucketName
                - "/*"
      - Effect: "Allow"
        Action:
          - glue:GetTable
          - glue:UpdateTable
          - glue:GetPartition
          - glue:GetPartitions
          - glue:BatchCreatePartition
        Resource: "*"
  #      - Effect: "Allow"
  #        Action:
  #          - glue:GetTable
  #          -
  #        Resource:
  #          - Fn::Join:
  #              - ""
  #              - - "arn:aws:glue:#{AWS::Region}:#{AWS::AccountId}:table/"
  #                - ${self:custom.DataCatalog}
  #                - "/"
  #                - ${self:custom.Database}
  #                - "/"
  #                - ${self:custom.Table}
  #
  #      - Effect: "Allow"
  #        Action:
  #          - glue:UpdateTable
  #          - glue:GetPartition
  #          - glue:GetPartitions
  ##        Resource:
  ##          - Fn::Join:
  ##              - ""
  ##              - - "arn:aws:glue:#{AWS::Region}:#{AWS::AccountId}:database/"
  ##                - ${self:custom.Database}
  ##          - Fn::Join:
  ##              - ""
  ##              - - "arn:aws:glue:#{AWS::Region}:#{AWS::AccountId}:table/"
  ##                - ${self:custom.Database}
  ##                - "/"
  ##                - ${self:custom.Table}

  GetAthenaState:
    #    handler: src/GetAthenaState/handler.handle
    handler: /opt/python/lumigo_tracer._handler
    module: src/GetAthenaState
    timeout: 30
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:134
    iamRoleStatementsName: sls-batch-${opt:stage, self:provider.stage}-GetAthenaState-lambdaRole
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - athena:BatchGetQueryExecution
        Resource:
          Fn::Join:
            - ""
            - - "arn:aws:athena:#{AWS::Region}:#{AWS::AccountId}:workgroup/"
              - ${self:custom.AthenaWorkGroup}

resources:
  - ${file(resources/glue.yml)}

stepFunctions:
  stateMachines: ${file(resources/sm.yml)}
#resources:
#  - Outputs:
#      TrafficEventsDeliveryBucket:
#        Description: "Bucket containing all events for all locations"
#        Value:
#          Fn::ImportValue: "sls-data-pipelines-dev-TrafficEventsDeliveryBucketName"