AWSTemplateFormatVersion : 2010-09-09
Transform: AWS::Serverless-2016-10-31
Description: This template creates the AWS resources required for audio redaction of PII using Amazon Transcribe
Parameters:
  BucketName:
    Description: 'S3 Bucket where input audio is uploaded.'
    Type: String
  OutputBucketName:
    Description: 'S3 Bucket where the Amazon Transcribe output JSON and redacted audio are saved.'
    Type: String
  DirsToCreate:
    Description: "Comma delimited list of directories to create."
    Type: CommaDelimitedList
    Default: 'output_json,redacted_audio'
Resources:
  Bucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    DependsOn:
      - TranscribeProcessingLambdaPermission
    Properties:
      BucketName: !Ref BucketName
      NotificationConfiguration:
        LambdaConfigurations:
          - Event: s3:ObjectCreated:*
            Function: !GetAtt TranscribeProcessingLambdaFunction.Arn
            Filter:
              S3Key:
                Rules:
                - Name: 'suffix'
                  Value: '.mp3'
  OutputBucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    DependsOn:
      - FFmpegLambdaPermission
    Properties:
      BucketName: !Ref OutputBucketName
      NotificationConfiguration:
        LambdaConfigurations:
          - Event: s3:ObjectCreated:*
            Function: !GetAtt FFmpegLambdaFunction.Arn
            Filter:
              S3Key:
                Rules:
                - Name: 'prefix'
                  Value: 'output_json/'
                - Name: 'suffix'
                  Value: '.json'
  S3CustomResource:
    Type: Custom::S3CustomResource
    DeletionPolicy: Retain
    Properties:
      ServiceToken: !GetAtt S3CustomResourceLambdaFunction.Arn
      dirs_to_create: !Ref DirsToCreate
      output_bucket: !Ref OutputBucket
  TranscribeProcessingLambdaPermission  :
    Type: AWS::Lambda::Permission
    Properties:
      Action: 'lambda:InvokeFunction'
      FunctionName: !Ref TranscribeProcessingLambdaFunction
      Principal: s3.amazonaws.com
      SourceArn: !Sub 'arn:aws:s3:::${BucketName}'
      SourceAccount: !Ref AWS::AccountId
  FFmpegLambdaPermission:
    Type: AWS::Lambda::Permission
    Properties:
      Action: 'lambda:InvokeFunction'
      FunctionName: !Ref FFmpegLambdaFunction
      Principal: s3.amazonaws.com
      SourceArn: !Sub 'arn:aws:s3:::${OutputBucketName}'
      SourceAccount: !Ref AWS::AccountId
  AWSLambdaExecutionRole:
     Type: AWS::IAM::Role
     Properties:
       AssumeRolePolicyDocument:
         Statement:
         - Action:
           - sts:AssumeRole
           Effect: Allow
           Principal:
             Service:
             - lambda.amazonaws.com
         Version: '2012-10-17'
       Path: "/"
       Policies:
       - PolicyDocument:
           Statement:
           - Action:
             - logs:CreateLogGroup
             - logs:CreateLogStream
             - logs:PutLogEvents
             Effect: Allow
             Resource: arn:aws:logs:*:*:*
           Version: '2012-10-17'
         PolicyName: !Sub ${AWS::StackName}-${AWS::Region}-AWSLambda-CW
       - PolicyDocument:
           Statement:
           - Action:
             - s3:GetObject
             - s3:DeleteObject
             - s3:PutObject
             Effect: Allow
             Resource:
             - !Sub arn:aws:s3:::${OutputBucket}/*
             - !Sub arn:aws:s3:::${OutputBucket}
           Version: '2012-10-17'
         PolicyName: !Sub ${AWS::StackName}-${AWS::Region}-AWSLambda-S3
       - PolicyDocument:
           Statement:
           - Action:
             - transcribe:StartTranscriptionJob
             - transcribe:GetTranscriptionJob
             Effect: Allow
             Resource:
             - "*"
           Version: '2012-10-17'
         PolicyName: !Sub ${AWS::StackName}-${AWS::Region}-AWSLambda-Transcribe
       RoleName: !Sub ${AWS::StackName}-${AWS::Region}-AWSLambdaExecutionRole
  TranscribeProcessingLambdaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service:
            - lambda.amazonaws.com
          Action:
          - sts:AssumeRole
      Policies:
        - PolicyName: allowLogging
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
            - Effect: Allow
              Action:
              - logs:CreateLogGroup
              - logs:CreateLogStream
              - logs:PutLogEvents
              Resource: arn:aws:logs:*:*:*
        - PolicyName: getAndDeleteObjects
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
            - Effect: Allow
              Action:
              - s3:GetObject
              - s3:DeleteObject
              - s3:PutObject
              Resource:
                - !Sub 'arn:aws:s3:::${OutputBucketName}/*'
                - !Sub 'arn:aws:s3:::${BucketName}/*'
        - PolicyName: allowTranscribe
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
            - Effect: Allow
              Action:
              - transcribe:StartTranscriptionJob
              - transcribe:GetTranscriptionJob
              Resource: "*"
  S3CustomResourceLambdaFunction:
     Type: "AWS::Lambda::Function"
     Properties:
       Description: "S3 Custom Resource management, i.e. prefix creation and deletion."
       FunctionName: !Sub '${AWS::StackName}-${AWS::Region}-lambda'
       Handler: index.handler
       Role: !GetAtt AWSLambdaExecutionRole.Arn
       Timeout: 360
       Runtime: python3.6
       Code:
         ZipFile: |
          import boto3
          import cfnresponse
          def handler(event, context):
              # Init ...
              the_event = event['RequestType']
              print("The event is: ", str(the_event))
              response_data = {}
              s_3 = boto3.client('s3')
              # Retrieve parameters
              output_bucket = event['ResourceProperties']['output_bucket']
              dirs_to_create = event['ResourceProperties']['dirs_to_create']
              try:
                  if the_event in ('Create', 'Update'):
                      print("Requested folders: ", str(dirs_to_create))
                      for dir_name in dirs_to_create:
                          print("Creating: ", str(dir_name))
                          s_3.put_object(Bucket=output_bucket,
                                         Key=(dir_name
                                              + '/'))
                  elif the_event == 'Delete':
                      print("Deleting S3 content...")
                      b_operator = boto3.resource('s3')
                      b_operator.Bucket(str(output_bucket)).objects.all().delete()
                  # Everything OK... send the signal back
                  print("Operation successful!")
                  cfnresponse.send(event,
                                   context,
                                   cfnresponse.SUCCESS,
                                   response_data)
              except Exception as e:
                  print("Operation failed...")
                  print(str(e))
                  response_data['Data'] = str(e)
                  cfnresponse.send(event,
                                   context,
                                   cfnresponse.FAILED,
                                   response_data)
  TranscribeProcessingLambdaFunction:
    Type: AWS::Lambda::Function
    Properties:
      Handler: index.handler
      Role: !GetAtt TranscribeProcessingLambdaExecutionRole.Arn
      Runtime: python3.6
      MemorySize: 512
      Timeout: 900
      Environment:
        Variables:
          JSON_OUTPUT_BUCKET: !Ref OutputBucketName
      Code:
        ZipFile: |
          import json
          import boto3
          import time
          import os
          from random import randint
          def handler(event, context):
            transcribe_client = boto3.client('transcribe')
            req_id = event['Records'][0]['responseElements']['x-amz-request-id']
            s3_bucket = event['Records'][0]['s3']['bucket']['name']
            print('S3 bucket: ', event['Records'][0]['s3']['bucket']['name'])
            print('S3 Output bucket: ', os.environ['JSON_OUTPUT_BUCKET'])
            s3_key = event['Records'][0]['s3']['object']['key']
            print('Audio input file: ',event['Records'][0]['s3']['object']['key'])
            audio_file_uri = 's3://' + s3_bucket + '/' + s3_key
            print('Audio input file URI: ', audio_file_uri)
            transcribe_file(s3_key.split('/')[-1].split('.')[0] + '-' + req_id, audio_file_uri, transcribe_client, os.environ['JSON_OUTPUT_BUCKET'])
          def transcribe_file(job_name, file_uri, transcribe_client, bucket):
            transcribe_client.start_transcription_job(
              TranscriptionJobName=job_name,
              Media={'MediaFileUri': file_uri},
              OutputBucketName=bucket,
              OutputKey='output_json/',
              MediaFormat='mp3',
              ContentRedaction={
                'RedactionType': 'PII',
                'RedactionOutput': 'redacted'
              },
              LanguageCode='en-US'
            )
            max_tries = 60
            while max_tries > 0:
              max_tries -= 1
              job = transcribe_client.get_transcription_job(TranscriptionJobName=job_name)
              job_status = job['TranscriptionJob']['TranscriptionJobStatus']
              if job_status in ['COMPLETED', 'FAILED']:
                print("Job in progress...")
                if job_status == 'COMPLETED':
                  print('COMPLETED')
                break
              else:
                print("Waiting for job completion.... ", job_status)
              time.sleep(10)
  FFmpegLambdaFunction:
    Type: AWS::Lambda::Function
    Properties:
      Handler: index.handler
      Role: !GetAtt TranscribeProcessingLambdaExecutionRole.Arn
      Runtime: python3.6
      MemorySize: 512
      Timeout: 900
      Layers:
        - !Ref FFmpegLayer
      Environment:
        Variables:
          AUDIO_INPUT_BUCKET: !Ref BucketName
      Code:
        ZipFile: |
          import json
          import boto3
          import os
          def handler(event, context):
            BUCKET_NAME = event['Records'][0]['s3']['bucket']['name']
            print('S3 bucket: ', BUCKET_NAME)
            KEY = event['Records'][0]['s3']['object']['key'].split('/')[-1]
            print('Transcribe output file: ', KEY)
            AUDIO = KEY[:-22] + '.mp3'
            AUDIO = AUDIO[9:]
            print('Original audio file: ', AUDIO)
            s3 = boto3.resource('s3')
            s3.Bucket(os.environ['AUDIO_INPUT_BUCKET']).download_file(AUDIO, '/tmp/' + AUDIO)
            s3.Bucket(BUCKET_NAME).download_file('output_json/' + KEY, '/tmp/' + KEY)
            f = open('/tmp/' + KEY)
            transcript = json.load(f)
            items = transcript['results']['items']
            command_str_p1 = "/opt/bin/ffmpeg -i "
            redact_times = ""
            for item in items:
              if item['alternatives'][0]['content'] == '[PII]':
                s = 'volume=enable=\'between(t,' + item['start_time'] + ',' + item['end_time'] + ')\':volume=0' + ", "
                redact_times += s
            redact_times = "\"" + redact_times[:-2] + "\""
            os.system(command_str_p1 + "/tmp/" + AUDIO + " -af " + redact_times + " /tmp/redacted-" + AUDIO)
            s3_client = boto3.client('s3')
            s3_client.upload_file("/tmp/redacted-" + AUDIO, BUCKET_NAME, "redacted_audio/redacted-" + AUDIO)
            os.system('ls -l /tmp')
  FFmpegLayer:
    Type: AWS::Serverless::LayerVersion
    Properties:
      Description: Lambda Layer for FFmpeg
      ContentUri:
        Bucket: awsserverlessrepo-changesets-plntc6bfnfj
        Key:
          Fn::Sub: ${AWS::AccountId}/arn:aws:serverlessrepo:us-east-1:496010403454:applications-ffmpeg-lambda-layer-python3-versions-0.0.3/169dd784-3ff4-4c84-aedf-60c33a241f49
      CompatibleRuntimes:
      - python3.6
      - python3.7
      - python3.8
      RetentionPolicy: Delete