diff --git a/.gitignore b/.gitignore index 4342f002..dc81048e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,7 @@ node_modules/ # ローカル確認用環境変数ファイル .env # pythonのキャッシュファイル -__pycache__/ \ No newline at end of file +__pycache__/ + +# StepFunctionsステートメント定義変換後のフォルダ +stepfunctions/*/build \ No newline at end of file diff --git a/stepfunctions/TOOLS/convert.conf b/stepfunctions/TOOLS/convert.conf new file mode 100644 index 00000000..3d62ff57 --- /dev/null +++ b/stepfunctions/TOOLS/convert.conf @@ -0,0 +1,24 @@ +# +# 各項目の第1カラムにリプレース前文字列を記載する +# 各項目の第2カラムstagingの情報を記載する +# 各項目の第3カラムにproductの情報を記載する +# 変換前文字列と変換後文字列をカンマで区切る +# + +# AWSアカウントID +{REPLACE_AWS_ACCOUNT_ID},826466435614,826466435614 + +# 環境情報 +{REPLACE_ENV_NAME},staging,product + +# サブネット(PrivateSubnet1) +{REPLACE_SUBNET_PRI_1A},subnet-0a47b12f6899ab19e,subnet-0d9bf8cd421cf2489 + +# サブネット(PrivateSubnet2) +{REPLACE_SUBNET_PRI_1D},subnet-0ecb92c12eb49ebc3,subnet-0595f52cf6fd9b9e7 + +# セキュリティグループ(ecs-all) +{REPLACE_SG_ECS_ALL},sg-051e0fb9925539592,sg-05df4823fc789b0fa + +# セキュリティグループ(ecs-crm-datafetch) +{REPLACE_SG_CRM_DATAFETCH},sg-0b20b7bb1cb1ab886,sg-XXXXXXXXXXXXXXXXX \ No newline at end of file diff --git a/stepfunctions/TOOLS/convert_definition.py b/stepfunctions/TOOLS/convert_definition.py new file mode 100644 index 00000000..cea832e1 --- /dev/null +++ b/stepfunctions/TOOLS/convert_definition.py @@ -0,0 +1,173 @@ +import sys +import csv +import os +import re + +CONF_FOLDER_PATH = '.' +DEFINITION_FOLDER_BASE = '..' +CONVERTED_FOLDER_BASE = 'build' +CONVERT_CONF = 'convert.conf' +CHAR_CODE = 'utf-8' +PRD_NAME = 'product' +STG_NAME = 'staging' +COMMENT_PATTERN = r'(^\s*(#.*|)$)|(^(\s*|)(\r\n|\r|\n)$)' + + +def main(args=None): + try: + """ + args1: StepFunctionsステート名 + args2: 変換先環境名(product or staging) + """ + + # カレントディレクトリ移動 + os.chdir(os.path.dirname(os.path.abspath(__file__))) + + # 引数チェック + state_name, to_env = check_args(args) + + print('引数確認OK') + + # ファイル存在チェック + check_file_exist(state_name, to_env) + + print('ファイル存在確認OK') + + # 変換 + converted_file_name = convert_definition(state_name, to_env) + + print(f'変換が完了しました ファイル名:{converted_file_name}') + + return + + except Exception as e: + raise Exception(e) + + +def check_args(args): + try: + check_length(args) + + check_aws_environment(args) + + return args[1], args[2] + + except Exception as e: + raise Exception(f'引数確認に失敗しました {e}') + + +def check_length(args): + if len(args) != 3: + raise Exception('引数の数が異常です') + + return + + +def check_aws_environment(args): + if args[2] == STG_NAME or args[2] == PRD_NAME: + return + + raise Exception('第2引数が不正です') + + +def check_file_exist(state_name, to_env): + if not os.path.isfile(f'{CONF_FOLDER_PATH}/{CONVERT_CONF}'): + raise Exception('変換定義ファイルが存在しません') + + if not os.path.isfile(f'{DEFINITION_FOLDER_BASE}/{state_name}/{state_name}.json'): + raise Exception('変換元のステートメント定義が存在しません') + + if not os.path.isdir(f'{DEFINITION_FOLDER_BASE}/{state_name}/{CONVERTED_FOLDER_BASE}/{to_env}'): + os.makedirs(f'{DEFINITION_FOLDER_BASE}/{state_name}/{CONVERTED_FOLDER_BASE}/{to_env}', exist_ok=True) + + return + + +def convert_definition(state_name, to_env): + + try: + # 定義フォルダのパス生成 + from_folder = f'{DEFINITION_FOLDER_BASE}/{state_name}' + to_folder = f'{from_folder}/{CONVERTED_FOLDER_BASE}/{to_env}' + + # 変換定義のリスト化 + convert_list = set_convert_list() + + # 変換定義のTOの列番号を確認 + to_attr_num = set_attribute(to_env, convert_list) + + # 読込み、書込みファイルオープン + from_file = open(f'{from_folder}/{state_name}.json', + mode='r', encoding=CHAR_CODE) + to_file = open(f'{to_folder}/{state_name}.json', mode='w', + encoding=CHAR_CODE, newline='\n') + + # 変換元ファイルを1行ずつ読込み + for line in from_file: + # 変換定義を1要素ずつ読込み + for convert_value in convert_list: + # 変換 + line = line.replace( + convert_value[0], convert_value[to_attr_num]) + to_file.write(line) + + # 各ファイルクローズ + from_file.close() + to_file.close() + + return f'{to_folder}/{state_name}.json' + + except Exception as e: + raise Exception(f'変換に失敗しました {e}') + + +def set_convert_list(): + + try: + # 変換定義ファイルオープン + convert_conf_file = open( + f'{CONF_FOLDER_PATH}/{CONVERT_CONF}', mode='r', encoding=CHAR_CODE) + + # ファイル読込み + convert_conf = convert_conf_file.readlines() + + # 変換定義ファイルリスト化 + convert_list = [] + for row in convert_conf: + if re.match(COMMENT_PATTERN, row[0]): + continue + row = row.rstrip('\n') + data = row.split(',') + convert_list.append(data) + + convert_conf_file.close() + + return convert_list + + except Exception as e: + Exception('変換定義ファイルのリスト化に失敗しました') + + +def set_attribute(from_env, convert_list): + """ + 変換定義ファイルのどちらの列がどちらの環境かを特定 + 1列目にどちらかの環境名(product or staging)がある場合、定義内のその1列が対象の環境となる + """ + to_attr_num = None + + for attr in convert_list: + + if not attr[1] or not attr[2]: + raise Exception('変換定義が正しくありません') + + if attr[1] == from_env: + to_attr_num = 1 + + elif attr[2] == from_env: + to_attr_num = 2 + + return to_attr_num + + +if __name__ == '__main__': + main(args=sys.argv) diff --git a/stepfunctions/r-crm-datafetch-state/r-crm-datafetch-state.json b/stepfunctions/r-crm-datafetch-state/r-crm-datafetch-state.json new file mode 100644 index 00000000..f4b22be9 --- /dev/null +++ b/stepfunctions/r-crm-datafetch-state/r-crm-datafetch-state.json @@ -0,0 +1,129 @@ +{ + "Comment": "crm-datafetch root job", + "StartAt": "params", + "States": { + "params": { + "Comment": "パラメータ設定", + "Type": "Pass", + "Parameters": { + "sns": { + "TopicArn": "arn:aws:sns:ap-northeast-1:{REPLACE_AWS_ACCOUNT_ID}:nds-notice-{REPLACE_ENV_NAME}" + }, + "ecs": { + "Cluster": "arn:aws:ecs:ap-northeast-1:{REPLACE_AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-{REPLACE_ENV_NAME}-crm-ecs", + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets": [ + "{REPLACE_SUBNET_PRI_1A}", + "{REPLACE_SUBNET_PRI_1D}" + ], + "SecurityGroups": [ + "{REPLACE_SG_ECS_ALL}", + "{REPLACE_SG_CRM_DATAFETCH}" + ], + "AssignPublicIp": "DISABLED" + } + } + } + }, + "ResultPath": "$.params", + "Next": "crm-datafetch-diff" + }, + "crm-datafetch-diff": { + "Comment": "CRMデータ取得", + "Type": "Task", + "Resource": "arn:aws:states:::ecs:runTask.sync", + "Parameters": { + "Cluster.$": "$.params.ecs.Cluster", + "LaunchType.$": "$.params.ecs.LaunchType", + "TaskDefinition": "arn:aws:ecs:ap-northeast-1:{REPLACE_AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-{REPLACE_ENV_NAME}-task-crm-datafetch", + "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration" + }, + "Retry": [ + { + "ErrorEquals": ["States.ALL"], + "BackoffRate": 2, + "IntervalSeconds": 3, + "MaxAttempts": 3 + } + ], + "Catch": [ + { + "ErrorEquals": ["States.ALL"], + "ResultPath": "$.result", + "Next": "ErrorEnd" + } + ], + "ResultPath": "$.result", + "Next": "Wait" + }, + "Wait": { + "Type": "Wait", + "Seconds": 5, + "Next": "crm-datafetch-all" + }, + "crm-datafetch-all": { + "Comment": "CRMデータ全量取得", + "Type": "Task", + "Resource": "arn:aws:states:::ecs:runTask.sync", + "Parameters": { + "Cluster.$": "$.params.ecs.Cluster", + "LaunchType.$": "$.params.ecs.LaunchType", + "TaskDefinition": "arn:aws:ecs:ap-northeast-1:{REPLACE_AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-{REPLACE_ENV_NAME}-task-crm-datafetch", + "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration", + "Overrides": { + "ContainerOverrides": [ + { + "Name": "mbj-newdwh2021-{REPLACE_ENV_NAME}-container-crm-datafetch", + "Environment": [ + { + "Name": "OBJECT_INFO_FILENAME", + "Value": "crm_object_list_all.json" + } + ] + } + ] + } + }, + "Retry": [ + { + "ErrorEquals": ["States.ALL"], + "BackoffRate": 2, + "IntervalSeconds": 3, + "MaxAttempts": 3 + } + ], + "Catch": [ + { + "ErrorEquals": ["States.ALL"], + "ResultPath": "$.result", + "Next": "ErrorEnd" + } + ], + "ResultPath": "$.result", + "Next": "SuccessNotice" + }, + "SuccessNotice": { + "Comment": "正常終了通知", + "Type": "Task", + "Resource": "arn:aws:states:::sns:publish", + "Parameters": { + "TopicArn.$": "$.params.sns.TopicArn", + "Subject": "StepFunctions正常終了通知", + "Message.$": "States.Format('CRMデータ取得処理が完了しました。\n\n 対象のステートマシン:{} \n 実行ID:{}', $$.StateMachine.Id, $$.Execution.Name)" + }, + "Next": "NormalEnd" + }, + "NormalEnd": { + "Comment": "正常終了", + "Type": "Succeed" + }, + "ErrorEnd": { + "Comment": "異常終了", + "Type": "Fail", + "Error": "StatesError", + "Cause": "StepFunctions ErrorEnd" + } + } +}