nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: データ登録ECSのリトライを行うため、StepFunctionsを起動するようにしてみたもの

Browse Source 
改良の余地有り
This commit is contained in:
shimoda.m@nds-tyo.co.jp 2022-11-16 11:06:23 +09:00
parent 2d05708226
commit cb6adbb45d
1 changed files with 88 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
lambda/dataimport/dataimport.py
View File

@ -1,8 +1,11 @@
import json
import os
from datetime import datetime
import boto3
# 環境変数
SFN_STATE_MACHINE_ARN=os.environ["SFN_STATE_MACHINE_ARN"]
CLUSTER_NAME = os.environ["CLUSTER_NAME"]
TASK_NAME = os.environ["TASK_NAME"]
CONTAINER_NAME = os.environ["CONTAINER_NAME"]
@ -14,6 +17,7 @@ MODE = os.environ["MODE"]
# クラス変数
ecs_client = boto3.client('ecs')
sfn_client = boto3.client('stepfunctions')
def lambda_handler(event, context):
@ -29,38 +33,92 @@ def lambda_handler(event, context):
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-3 ファイル名：{event_file_name}')
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-4 データソース名：{event_data_source_name}')
# ECSを起動する
response = ecs_client.run_task(
launchType='FARGATE',
cluster=CLUSTER_NAME,
taskDefinition=TASK_NAME,
networkConfiguration={
"awsvpcConfiguration": {
"subnets": [
SUBNET_ID_AP_NORTHEAST_1A,
SUBNET_ID_AP_NORTHEAST_1D,
],
"securityGroups": [
SECURITY_GROUP_ID_ECSALL,
SECURITY_GROUP_ID_ECSDATAIMPORT,
],
# StepFunctionsを起動する
response = sfn_client.start_execution(
stateMachineArn=SFN_STATE_MACHINE_ARN,
input=json.dumps(
{
'inputParams': {
'ecs': {
'Cluster': CLUSTER_NAME,
'TaskDefinition': TASK_NAME,
'network': {
# TODO: 配列もパラメータから組めるなら、StepFunctions側で組み立てる
"Subnets": [
SUBNET_ID_AP_NORTHEAST_1A,
SUBNET_ID_AP_NORTHEAST_1D,
],
"SecurityGroups": [
SECURITY_GROUP_ID_ECSALL,
SECURITY_GROUP_ID_ECSDATAIMPORT,
],
},
'overrides': {
"ContainerName": CONTAINER_NAME,
"Environment": [
# TODO: 環境変数以外でも使いたいので、外出しする
{"Name": 'BUCKET_NAME', "Value": event_bucket_name},
{"Name": 'TARGET_KEY', "Value": event_object_key},
{"Name": 'DATA_SOURCE_NAME', "Value": event_data_source_name},
{"Name": 'FILE_NAME', "Value": event_file_name},
{"Name": 'MODE', "Value": MODE},
],
},
},
}
}
},
overrides={
"containerOverrides": [
{
"name": CONTAINER_NAME,
"environment": [
{"name": 'BUCKET_NAME', "value": event_bucket_name},
{"name": 'TARGET_KEY', "value": event_object_key},
{"name": 'DATA_SOURCE_NAME', "value": event_data_source_name},
{"name": 'FILE_NAME', "value": event_file_name},
{"name": 'MODE', "value": MODE},
],
},
],
},
)
)
# # ECSを起動する
# response = ecs_client.run_task(
# launchType='FARGATE',
# cluster=CLUSTER_NAME,
# taskDefinition=TASK_NAME,
# networkConfiguration={
# "awsvpcConfiguration": {
# "subnets": [
# SUBNET_ID_AP_NORTHEAST_1A,
# SUBNET_ID_AP_NORTHEAST_1D,
# ],
# "securityGroups": [
# SECURITY_GROUP_ID_ECSALL,
# SECURITY_GROUP_ID_ECSDATAIMPORT,
# ],
# }
# },
# overrides={
# "containerOverrides": [
# {
# "name": CONTAINER_NAME,
# "environment": [
# {"name": 'BUCKET_NAME', "value": event_bucket_name},
# {"name": 'TARGET_KEY', "value": event_object_key},
# {"name": 'DATA_SOURCE_NAME', "value": event_data_source_name},
# {"name": 'FILE_NAME', "value": event_file_name},
# {"name": 'MODE', "value": MODE},
# ],
# },
# ],
# },
# )
# TODO: コメントと設計修正
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-5 ECS起動レスポンス：{str(response)}')
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-6 駆動処理終了')
if __name__ == '__main__':
# ローカル実行用
lambda_handler({"Records": [{
"s3": {
"bucket": {
"name": "test-shimoda-bucket"
},
"object": {
"key": "folder/object_key"
}
}
}]}, {})