feat: StepFunctionsのGit管理用
This commit is contained in:
Y_SAKAI
parent
1da7a46e34
commit
d56c66f96d
5
.gitignore
vendored
5
.gitignore
vendored
@ -4,4 +4,7 @@ node_modules/
|
||||
# ローカル確認用環境変数ファイル
|
||||
.env
|
||||
# pythonのキャッシュファイル
|
||||
__pycache__/
|
||||
__pycache__/
|
||||
|
||||
# StepFunctionsステートメント定義変換後のフォルダ
|
||||
stepfunctions/*/build
|
||||
24
stepfunctions/TOOLS/convert.conf
Normal file
24
stepfunctions/TOOLS/convert.conf
Normal file
@ -0,0 +1,24 @@
|
||||
#
|
||||
# 各項目の第1カラムにリプレース前文字列を記載する
|
||||
# 各項目の第2カラムstagingの情報を記載する
|
||||
# 各項目の第3カラムにproductの情報を記載する
|
||||
# 変換前文字列と変換後文字列をカンマで区切る
|
||||
#
|
||||
|
||||
# AWSアカウントID
|
||||
{REPLACE_AWS_ACCOUNT_ID},826466435614,826466435614
|
||||
|
||||
# 環境情報
|
||||
{REPLACE_ENV_NAME},staging,product
|
||||
|
||||
# サブネット(PrivateSubnet1)
|
||||
{REPLACE_SUBNET_PRI_1A},subnet-0a47b12f6899ab19e,subnet-0d9bf8cd421cf2489
|
||||
|
||||
# サブネット(PrivateSubnet2)
|
||||
{REPLACE_SUBNET_PRI_1D},subnet-0ecb92c12eb49ebc3,subnet-0595f52cf6fd9b9e7
|
||||
|
||||
# セキュリティグループ(ecs-all)
|
||||
{REPLACE_SG_ECS_ALL},sg-051e0fb9925539592,sg-05df4823fc789b0fa
|
||||
|
||||
# セキュリティグループ(ecs-crm-datafetch)
|
||||
{REPLACE_SG_CRM_DATAFETCH},sg-0b20b7bb1cb1ab886,sg-XXXXXXXXXXXXXXXXX
|
||||
173
stepfunctions/TOOLS/convert_definition.py
Normal file
173
stepfunctions/TOOLS/convert_definition.py
Normal file
@ -0,0 +1,173 @@
|
||||
import sys
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
|
||||
CONF_FOLDER_PATH = '.'
|
||||
DEFINITION_FOLDER_BASE = '..'
|
||||
CONVERTED_FOLDER_BASE = 'build'
|
||||
CONVERT_CONF = 'convert.conf'
|
||||
CHAR_CODE = 'utf-8'
|
||||
PRD_NAME = 'product'
|
||||
STG_NAME = 'staging'
|
||||
COMMENT_PATTERN = r'(^\s*(#.*|)$)|(^(\s*|)(\r\n|\r|\n)$)'
|
||||
|
||||
|
||||
def main(args=None):
|
||||
try:
|
||||
"""
|
||||
args1: StepFunctionsステート名
|
||||
args2: 変換先環境名（product or staging）
|
||||
"""
|
||||
|
||||
# カレントディレクトリ移動
|
||||
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
# 引数チェック
|
||||
state_name, to_env = check_args(args)
|
||||
|
||||
print('引数確認OK')
|
||||
|
||||
# ファイル存在チェック
|
||||
check_file_exist(state_name, to_env)
|
||||
|
||||
print('ファイル存在確認OK')
|
||||
|
||||
# 変換
|
||||
converted_file_name = convert_definition(state_name, to_env)
|
||||
|
||||
print(f'変換が完了しました ファイル名：{converted_file_name}')
|
||||
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(e)
|
||||
|
||||
|
||||
def check_args(args):
|
||||
try:
|
||||
check_length(args)
|
||||
|
||||
check_aws_environment(args)
|
||||
|
||||
return args[1], args[2]
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(f'引数確認に失敗しました {e}')
|
||||
|
||||
|
||||
def check_length(args):
|
||||
if len(args) != 3:
|
||||
raise Exception('引数の数が異常です')
|
||||
|
||||
return
|
||||
|
||||
|
||||
def check_aws_environment(args):
|
||||
if args[2] == STG_NAME or args[2] == PRD_NAME:
|
||||
return
|
||||
|
||||
raise Exception('第2引数が不正です')
|
||||
|
||||
|
||||
def check_file_exist(state_name, to_env):
|
||||
if not os.path.isfile(f'{CONF_FOLDER_PATH}/{CONVERT_CONF}'):
|
||||
raise Exception('変換定義ファイルが存在しません')
|
||||
|
||||
if not os.path.isfile(f'{DEFINITION_FOLDER_BASE}/{state_name}/{state_name}.json'):
|
||||
raise Exception('変換元のステートメント定義が存在しません')
|
||||
|
||||
if not os.path.isdir(f'{DEFINITION_FOLDER_BASE}/{state_name}/{CONVERTED_FOLDER_BASE}/{to_env}'):
|
||||
os.makedirs(f'{DEFINITION_FOLDER_BASE}/{state_name}/{CONVERTED_FOLDER_BASE}/{to_env}', exist_ok=True)
|
||||
|
||||
return
|
||||
|
||||
|
||||
def convert_definition(state_name, to_env):
|
||||
|
||||
try:
|
||||
# 定義フォルダのパス生成
|
||||
from_folder = f'{DEFINITION_FOLDER_BASE}/{state_name}'
|
||||
to_folder = f'{from_folder}/{CONVERTED_FOLDER_BASE}/{to_env}'
|
||||
|
||||
# 変換定義のリスト化
|
||||
convert_list = set_convert_list()
|
||||
|
||||
# 変換定義のTOの列番号を確認
|
||||
to_attr_num = set_attribute(to_env, convert_list)
|
||||
|
||||
# 読込み、書込みファイルオープン
|
||||
from_file = open(f'{from_folder}/{state_name}.json',
|
||||
mode='r', encoding=CHAR_CODE)
|
||||
to_file = open(f'{to_folder}/{state_name}.json', mode='w',
|
||||
encoding=CHAR_CODE, newline='\n')
|
||||
|
||||
# 変換元ファイルを1行ずつ読込み
|
||||
for line in from_file:
|
||||
# 変換定義を1要素ずつ読込み
|
||||
for convert_value in convert_list:
|
||||
# 変換
|
||||
line = line.replace(
|
||||
convert_value[0], convert_value[to_attr_num])
|
||||
to_file.write(line)
|
||||
|
||||
# 各ファイルクローズ
|
||||
from_file.close()
|
||||
to_file.close()
|
||||
|
||||
return f'{to_folder}/{state_name}.json'
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(f'変換に失敗しました {e}')
|
||||
|
||||
|
||||
def set_convert_list():
|
||||
|
||||
try:
|
||||
# 変換定義ファイルオープン
|
||||
convert_conf_file = open(
|
||||
f'{CONF_FOLDER_PATH}/{CONVERT_CONF}', mode='r', encoding=CHAR_CODE)
|
||||
|
||||
# ファイル読込み
|
||||
convert_conf = convert_conf_file.readlines()
|
||||
|
||||
# 変換定義ファイルリスト化
|
||||
convert_list = []
|
||||
for row in convert_conf:
|
||||
if re.match(COMMENT_PATTERN, row[0]):
|
||||
continue
|
||||
row = row.rstrip('\n')
|
||||
data = row.split(',')
|
||||
convert_list.append(data)
|
||||
|
||||
convert_conf_file.close()
|
||||
|
||||
return convert_list
|
||||
|
||||
except Exception as e:
|
||||
Exception('変換定義ファイルのリスト化に失敗しました')
|
||||
|
||||
|
||||
def set_attribute(from_env, convert_list):
|
||||
"""
|
||||
変換定義ファイルのどちらの列がどちらの環境かを特定
|
||||
1列目にどちらかの環境名（product or staging）がある場合、定義内のその1列が対象の環境となる
|
||||
"""
|
||||
to_attr_num = None
|
||||
|
||||
for attr in convert_list:
|
||||
|
||||
if not attr[1] or not attr[2]:
|
||||
raise Exception('変換定義が正しくありません')
|
||||
|
||||
if attr[1] == from_env:
|
||||
to_attr_num = 1
|
||||
|
||||
elif attr[2] == from_env:
|
||||
to_attr_num = 2
|
||||
|
||||
return to_attr_num
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(args=sys.argv)
|
||||
129
stepfunctions/r-crm-datafetch-state/r-crm-datafetch-state.json
Normal file
129
stepfunctions/r-crm-datafetch-state/r-crm-datafetch-state.json
Normal file
@ -0,0 +1,129 @@
|
||||
{
|
||||
"Comment": "crm-datafetch root job",
|
||||
"StartAt": "params",
|
||||
"States": {
|
||||
"params": {
|
||||
"Comment": "パラメータ設定",
|
||||
"Type": "Pass",
|
||||
"Parameters": {
|
||||
"sns": {
|
||||
"TopicArn": "arn:aws:sns:ap-northeast-1:{REPLACE_AWS_ACCOUNT_ID}:nds-notice-{REPLACE_ENV_NAME}"
|
||||
},
|
||||
"ecs": {
|
||||
"Cluster": "arn:aws:ecs:ap-northeast-1:{REPLACE_AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-{REPLACE_ENV_NAME}-crm-ecs",
|
||||
"LaunchType": "FARGATE",
|
||||
"NetworkConfiguration": {
|
||||
"AwsvpcConfiguration": {
|
||||
"Subnets": [
|
||||
"{REPLACE_SUBNET_PRI_1A}",
|
||||
"{REPLACE_SUBNET_PRI_1D}"
|
||||
],
|
||||
"SecurityGroups": [
|
||||
"{REPLACE_SG_ECS_ALL}",
|
||||
"{REPLACE_SG_CRM_DATAFETCH}"
|
||||
],
|
||||
"AssignPublicIp": "DISABLED"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"ResultPath": "$.params",
|
||||
"Next": "crm-datafetch-diff"
|
||||
},
|
||||
"crm-datafetch-diff": {
|
||||
"Comment": "CRMデータ取得",
|
||||
"Type": "Task",
|
||||
"Resource": "arn:aws:states:::ecs:runTask.sync",
|
||||
"Parameters": {
|
||||
"Cluster.$": "$.params.ecs.Cluster",
|
||||
"LaunchType.$": "$.params.ecs.LaunchType",
|
||||
"TaskDefinition": "arn:aws:ecs:ap-northeast-1:{REPLACE_AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-{REPLACE_ENV_NAME}-task-crm-datafetch",
|
||||
"NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration"
|
||||
},
|
||||
"Retry": [
|
||||
{
|
||||
"ErrorEquals": ["States.ALL"],
|
||||
"BackoffRate": 2,
|
||||
"IntervalSeconds": 3,
|
||||
"MaxAttempts": 3
|
||||
}
|
||||
],
|
||||
"Catch": [
|
||||
{
|
||||
"ErrorEquals": ["States.ALL"],
|
||||
"ResultPath": "$.result",
|
||||
"Next": "ErrorEnd"
|
||||
}
|
||||
],
|
||||
"ResultPath": "$.result",
|
||||
"Next": "Wait"
|
||||
},
|
||||
"Wait": {
|
||||
"Type": "Wait",
|
||||
"Seconds": 5,
|
||||
"Next": "crm-datafetch-all"
|
||||
},
|
||||
"crm-datafetch-all": {
|
||||
"Comment": "CRMデータ全量取得",
|
||||
"Type": "Task",
|
||||
"Resource": "arn:aws:states:::ecs:runTask.sync",
|
||||
"Parameters": {
|
||||
"Cluster.$": "$.params.ecs.Cluster",
|
||||
"LaunchType.$": "$.params.ecs.LaunchType",
|
||||
"TaskDefinition": "arn:aws:ecs:ap-northeast-1:{REPLACE_AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-{REPLACE_ENV_NAME}-task-crm-datafetch",
|
||||
"NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration",
|
||||
"Overrides": {
|
||||
"ContainerOverrides": [
|
||||
{
|
||||
"Name": "mbj-newdwh2021-{REPLACE_ENV_NAME}-container-crm-datafetch",
|
||||
"Environment": [
|
||||
{
|
||||
"Name": "OBJECT_INFO_FILENAME",
|
||||
"Value": "crm_object_list_all.json"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"Retry": [
|
||||
{
|
||||
"ErrorEquals": ["States.ALL"],
|
||||
"BackoffRate": 2,
|
||||
"IntervalSeconds": 3,
|
||||
"MaxAttempts": 3
|
||||
}
|
||||
],
|
||||
"Catch": [
|
||||
{
|
||||
"ErrorEquals": ["States.ALL"],
|
||||
"ResultPath": "$.result",
|
||||
"Next": "ErrorEnd"
|
||||
}
|
||||
],
|
||||
"ResultPath": "$.result",
|
||||
"Next": "SuccessNotice"
|
||||
},
|
||||
"SuccessNotice": {
|
||||
"Comment": "正常終了通知",
|
||||
"Type": "Task",
|
||||
"Resource": "arn:aws:states:::sns:publish",
|
||||
"Parameters": {
|
||||
"TopicArn.$": "$.params.sns.TopicArn",
|
||||
"Subject": "StepFunctions正常終了通知",
|
||||
"Message.$": "States.Format('CRMデータ取得処理が完了しました。\n\n 対象のステートマシン：{} \n 実行ID：{}', $$.StateMachine.Id, $$.Execution.Name)"
|
||||
},
|
||||
"Next": "NormalEnd"
|
||||
},
|
||||
"NormalEnd": {
|
||||
"Comment": "正常終了",
|
||||
"Type": "Succeed"
|
||||
},
|
||||
"ErrorEnd": {
|
||||
"Comment": "異常終了",
|
||||
"Type": "Fail",
|
||||
"Error": "StatesError",
|
||||
"Cause": "StepFunctions ErrorEnd"
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user