From 37d2c9dbdb80b1f64bc9f8d923029af74e14b3f0 Mon Sep 17 00:00:00 2001 From: *lcOeIaePm0 Date: Wed, 13 Oct 2021 10:52:57 +0900 Subject: [PATCH 1/4] =?UTF-8?q?feat:detaimport=E7=94=A8=E3=81=AELambda=20?= =?UTF-8?q?=E3=82=92=E4=BD=9C=E6=88=90=E3=81=97=E3=81=9F=E3=81=9F=E3=82=81?= =?UTF-8?q?=E3=80=81dataimport.py=E3=82=92=E8=BF=BD=E5=8A=A0=E3=81=97?= =?UTF-8?q?=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...bj-newdwh2021-staging-lambda-dataimport.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 lambda/mbj-newdwh2021-staging-lambda-dataimport.py diff --git a/lambda/mbj-newdwh2021-staging-lambda-dataimport.py b/lambda/mbj-newdwh2021-staging-lambda-dataimport.py new file mode 100644 index 00000000..169b845c --- /dev/null +++ b/lambda/mbj-newdwh2021-staging-lambda-dataimport.py @@ -0,0 +1,59 @@ +import os +from datetime import datetime +from os import path +import boto3 + +ecs_client = boto3.client('ecs') + +CLUSTER_NAME = os.environ['CLUSTER_NAME'] +TASK_NAME = os.environ['TASK_NAME'] +CONTAINER_NAME = os.environ['CONTAINER_NAME'] +SUBNET_ID_AP_NORTHEAST_1A = os.environ['SUBNET_ID_AP_NORTHEAST_1A'] +SUBNET_ID_AP_NORTHEAST_1D = os.environ['SUBNET_ID_AP_NORTHEAST_1D'] +SECURITY_GROUP_ID = os.environ['SECURITY_GROUP_ID'] + + +def lambda_handler(event, context): + print(str(datetime.now()) + 'Info I-1 駆動処理開始') + + # イベント情報を取得する + s3_event = event['Records'][0]['s3'] + event_bucket_name = s3_event['bucket']['name'] + event_object_key = s3_event['object']['key'] + event_file_name = os.path.basename(event_object_key) + event_data_source_name = os.path.dirname(event_object_key).split('/')[0] + print(str(datetime.now()) + 'Info I-2 バケット名：' + event_bucket_name) + print(str(datetime.now()) + 'Info I-3 ファイル名：' + event_file_name) + print(str(datetime.now()) + 'Info I-4 データソース名：' + event_data_source_name) + + # ECSを起動する + response = ecs_client.run_task( + launchType='FARGATE', + cluster=CLUSTER_NAME, + taskDefinition=TASK_NAME, + networkConfiguration={ + 'awsvpcConfiguration': { + 'subnets': [ + SUBNET_ID_AP_NORTHEAST_1A, SUBNET_ID_AP_NORTHEAST_1D, + ], + 'securityGroups': [ + SECURITY_GROUP_ID, + ], + 'assignPublicIp': 'ENABLED', + } + }, + overrides={ + 'containerOverrides': [ + { + 'name': CONTAINER_NAME, + 'environment': [ + {'name': 'FILE_NAME', 'value': event_file_name}, + {'name': 'DATA_SOURCE_NAME', 'value': event_data_source_name}, + ], + }, + ], + }, + ) + print(str(datetime.now()) + 'Info I-5 ECS起動レスポンス：' + str(response)) + + print(str(datetime.now()) + 'Info I-6 駆動処理終了') From 1d8cdace48898c61216a0d75a657b27238fd0ba2 Mon Sep 17 00:00:00 2001 From: *lcOeIaePm0 Date: Wed, 20 Oct 2021 16:13:13 +0900 Subject: [PATCH 2/4] =?UTF-8?q?feat:VPC=E3=82=A8=E3=83=B3=E3=83=89?= =?UTF-8?q?=E3=83=9D=E3=82=A4=E3=83=B3=E3=83=88=E8=BF=BD=E5=8A=A0=E3=81=AE?= =?UTF-8?q?=E3=81=9F=E3=82=81=E3=80=81=E3=82=BB=E3=82=AD=E3=83=A5=E3=83=AA?= =?UTF-8?q?=E3=83=86=E3=82=A3=E3=82=B0=E3=83=AB=E3=83=BC=E3=83=97=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat:引数の追加のため、ECS起動パラメータに値を追加した --- .../mbj-newdwh2021-staging-lambda-dataimport.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lambda/mbj-newdwh2021-staging-lambda-dataimport.py b/lambda/mbj-newdwh2021-staging-lambda-dataimport.py index 169b845c..d2af3b0d 100644 --- a/lambda/mbj-newdwh2021-staging-lambda-dataimport.py +++ b/lambda/mbj-newdwh2021-staging-lambda-dataimport.py @@ -1,6 +1,5 @@ import os from datetime import datetime -from os import path import boto3 ecs_client = boto3.client('ecs') @@ -10,7 +9,9 @@ TASK_NAME = os.environ['TASK_NAME'] CONTAINER_NAME = os.environ['CONTAINER_NAME'] SUBNET_ID_AP_NORTHEAST_1A = os.environ['SUBNET_ID_AP_NORTHEAST_1A'] SUBNET_ID_AP_NORTHEAST_1D = os.environ['SUBNET_ID_AP_NORTHEAST_1D'] -SECURITY_GROUP_ID = os.environ['SECURITY_GROUP_ID'] +SECURITY_GROUP_ID_ECRAPI = os.environ['SECURITY_GROUP_ID_ECRAPI'] +SECURITY_GROUP_ID_ECRDKR = os.environ['SECURITY_GROUP_ID_ECRDKR'] +SECURITY_GROUP_ID_LOGS = os.environ['SECURITY_GROUP_ID_LOGS'] def lambda_handler(event, context): @@ -34,10 +35,13 @@ def lambda_handler(event, context): networkConfiguration={ 'awsvpcConfiguration': { 'subnets': [ - SUBNET_ID_AP_NORTHEAST_1A, SUBNET_ID_AP_NORTHEAST_1D, + SUBNET_ID_AP_NORTHEAST_1A, + SUBNET_ID_AP_NORTHEAST_1D, ], 'securityGroups': [ - SECURITY_GROUP_ID, + SECURITY_GROUP_ID_ECRAPI, + SECURITY_GROUP_ID_ECRDKR, + SECURITY_GROUP_ID_LOGS, ], 'assignPublicIp': 'ENABLED', } @@ -47,8 +51,10 @@ def lambda_handler(event, context): { 'name': CONTAINER_NAME, 'environment': [ - {'name': 'FILE_NAME', 'value': event_file_name}, + {'name': 'BUCKET_NAME', 'value': event_bucket_name}, + {'name': 'TARGET_KEY', 'value': event_object_key}, {'name': 'DATA_SOURCE_NAME', 'value': event_data_source_name}, + {'name': 'FILE_NAME', 'value': event_file_name}, ], }, ], From dcd885de0d649ab401f69bfd2b8b13803c96c0d4 Mon Sep 17 00:00:00 2001 From: *lcOeIaePm0 Date: Thu, 21 Oct 2021 17:28:13 +0900 Subject: [PATCH 3/4] =?UTF-8?q?feat:RDS=E3=81=AB=E6=8E=A5=E7=B6=9A?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=81=9F=E3=82=81=E3=80=81RDS=E7=94=A8?= =?UTF-8?q?=E3=81=AE=E3=82=BB=E3=82=AD=E3=83=A5=E3=83=AA=E3=83=86=E3=82=A3?= =?UTF-8?q?=E3=82=B0=E3=83=AB=E3=83=BC=E3=83=97=E3=82=92=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/mbj-newdwh2021-staging-lambda-dataimport.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lambda/mbj-newdwh2021-staging-lambda-dataimport.py b/lambda/mbj-newdwh2021-staging-lambda-dataimport.py index d2af3b0d..b08a4174 100644 --- a/lambda/mbj-newdwh2021-staging-lambda-dataimport.py +++ b/lambda/mbj-newdwh2021-staging-lambda-dataimport.py @@ -12,6 +12,7 @@ SUBNET_ID_AP_NORTHEAST_1D = os.environ['SUBNET_ID_AP_NORTHEAST_1D'] SECURITY_GROUP_ID_ECRAPI = os.environ['SECURITY_GROUP_ID_ECRAPI'] SECURITY_GROUP_ID_ECRDKR = os.environ['SECURITY_GROUP_ID_ECRDKR'] SECURITY_GROUP_ID_LOGS = os.environ['SECURITY_GROUP_ID_LOGS'] +SECURITY_GROUP_ID_RDS = os.environ['SECURITY_GROUP_ID_RDS'] def lambda_handler(event, context): @@ -42,6 +43,7 @@ def lambda_handler(event, context): SECURITY_GROUP_ID_ECRAPI, SECURITY_GROUP_ID_ECRDKR, SECURITY_GROUP_ID_LOGS, + SECURITY_GROUP_ID_RDS, ], 'assignPublicIp': 'ENABLED', } From a76a97f2fa79ad74efe4717d0891376b6c8d8864 Mon Sep 17 00:00:00 2001 From: *lcOeIaePm0 Date: Thu, 21 Oct 2021 17:29:47 +0900 Subject: [PATCH 4/4] =?UTF-8?q?style:=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E8=A8=98=E3=80=81=E3=83=AD=E3=82=B0=E5=87=BA?= =?UTF-8?q?=E5=8A=9B=E3=81=AE=E8=A8=98=E8=BC=89=E5=A4=89=E6=9B=B4=E3=81=AA?= =?UTF-8?q?=E3=81=A9=E5=AE=9F=E6=96=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...bj-newdwh2021-staging-lambda-dataimport.py | 62 ++++++++++--------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/lambda/mbj-newdwh2021-staging-lambda-dataimport.py b/lambda/mbj-newdwh2021-staging-lambda-dataimport.py index b08a4174..6b5b061d 100644 --- a/lambda/mbj-newdwh2021-staging-lambda-dataimport.py +++ b/lambda/mbj-newdwh2021-staging-lambda-dataimport.py @@ -2,31 +2,33 @@ import os from datetime import datetime import boto3 -ecs_client = boto3.client('ecs') +# 環境変数 +CLUSTER_NAME = os.environ["CLUSTER_NAME"] +TASK_NAME = os.environ["TASK_NAME"] +CONTAINER_NAME = os.environ["CONTAINER_NAME"] +SUBNET_ID_AP_NORTHEAST_1A = os.environ["SUBNET_ID_AP_NORTHEAST_1A"] +SUBNET_ID_AP_NORTHEAST_1D = os.environ["SUBNET_ID_AP_NORTHEAST_1D"] +SECURITY_GROUP_ID_ECRAPI = os.environ["SECURITY_GROUP_ID_ECRAPI"] +SECURITY_GROUP_ID_ECRDKR = os.environ["SECURITY_GROUP_ID_ECRDKR"] +SECURITY_GROUP_ID_LOGS = os.environ["SECURITY_GROUP_ID_LOGS"] +SECURITY_GROUP_ID_RDS = os.environ["SECURITY_GROUP_ID_RDS"] -CLUSTER_NAME = os.environ['CLUSTER_NAME'] -TASK_NAME = os.environ['TASK_NAME'] -CONTAINER_NAME = os.environ['CONTAINER_NAME'] -SUBNET_ID_AP_NORTHEAST_1A = os.environ['SUBNET_ID_AP_NORTHEAST_1A'] -SUBNET_ID_AP_NORTHEAST_1D = os.environ['SUBNET_ID_AP_NORTHEAST_1D'] -SECURITY_GROUP_ID_ECRAPI = os.environ['SECURITY_GROUP_ID_ECRAPI'] -SECURITY_GROUP_ID_ECRDKR = os.environ['SECURITY_GROUP_ID_ECRDKR'] -SECURITY_GROUP_ID_LOGS = os.environ['SECURITY_GROUP_ID_LOGS'] -SECURITY_GROUP_ID_RDS = os.environ['SECURITY_GROUP_ID_RDS'] +# クラス変数 +ecs_client = boto3.client('ecs') def lambda_handler(event, context): - print(str(datetime.now()) + 'Info I-1 駆動処理開始') + print(f'{str(datetime.now())} Info I-1 駆動処理開始') # イベント情報を取得する - s3_event = event['Records'][0]['s3'] - event_bucket_name = s3_event['bucket']['name'] - event_object_key = s3_event['object']['key'] + s3_event = event["Records"][0]["s3"] + event_bucket_name = s3_event["bucket"]["name"] + event_object_key = s3_event["object"]["key"] event_file_name = os.path.basename(event_object_key) event_data_source_name = os.path.dirname(event_object_key).split('/')[0] - print(str(datetime.now()) + 'Info I-2 バケット名：' + event_bucket_name) - print(str(datetime.now()) + 'Info I-3 ファイル名：' + event_file_name) - print(str(datetime.now()) + 'Info I-4 データソース名：' + event_data_source_name) + print(f'{str(datetime.now())} Info I-2 バケット名：{event_bucket_name}') + print(f'{str(datetime.now())} Info I-3 ファイル名：{event_file_name}') + print(f'{str(datetime.now())} Info I-4 データソース名：{event_data_source_name}') # ECSを起動する response = ecs_client.run_task( @@ -34,34 +36,34 @@ def lambda_handler(event, context): cluster=CLUSTER_NAME, taskDefinition=TASK_NAME, networkConfiguration={ - 'awsvpcConfiguration': { - 'subnets': [ + "awsvpcConfiguration": { + "subnets": [ SUBNET_ID_AP_NORTHEAST_1A, SUBNET_ID_AP_NORTHEAST_1D, ], - 'securityGroups': [ + "securityGroups": [ SECURITY_GROUP_ID_ECRAPI, SECURITY_GROUP_ID_ECRDKR, SECURITY_GROUP_ID_LOGS, SECURITY_GROUP_ID_RDS, ], - 'assignPublicIp': 'ENABLED', + "assignPublicIp": 'ENABLED', } }, overrides={ - 'containerOverrides': [ + "containerOverrides": [ { - 'name': CONTAINER_NAME, - 'environment': [ - {'name': 'BUCKET_NAME', 'value': event_bucket_name}, - {'name': 'TARGET_KEY', 'value': event_object_key}, - {'name': 'DATA_SOURCE_NAME', 'value': event_data_source_name}, - {'name': 'FILE_NAME', 'value': event_file_name}, + "name": CONTAINER_NAME, + "environment": [ + {"name": 'BUCKET_NAME', "value": event_bucket_name}, + {"name": 'TARGET_KEY', "value": event_object_key}, + {"name": 'DATA_SOURCE_NAME', "value": event_data_source_name}, + {"name": 'FILE_NAME', "value": event_file_name}, ], }, ], }, ) - print(str(datetime.now()) + 'Info I-5 ECS起動レスポンス：' + str(response)) + print(f'{str(datetime.now())} Info I-5 ECS起動レスポンス：{str(response)}') - print(str(datetime.now()) + 'Info I-6 駆動処理終了') + print(f'{str(datetime.now())} Info I-6 駆動処理終了')