From 2527390a1b6722f5eb948aab34e57b4c0596da5b Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 1 Jul 2022 14:27:17 +0900 Subject: [PATCH 01/10] =?UTF-8?q?refactor:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92git?= =?UTF-8?q?=E5=AF=BE=E8=B1=A1=E5=A4=96=E3=81=AB=E8=A8=AD=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 8 ++++- ecs/dataimport/dataimport/main.py | 49 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e4f6d9fb..3c4e16ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json -lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* \ No newline at end of file +lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* +__pycache__/ +.env +launch.json +settings.json +Pipfile +Pipfile.lock \ No newline at end of file diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 730a7be8..5540f886 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,4 +1,5 @@ from datetime import datetime +from pickle import FALSE import boto3 import pymysql from pymysql.constants import CLIENT @@ -23,6 +24,7 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, + 'importManner': 12, } LINE_FEED_CODE = { 'CR': '\r', @@ -30,6 +32,10 @@ LINE_FEED_CODE = { 'CRLF': '\r\n', } DIRECTORY_SETTINGS = '/settings/' +TRUNCATE_SRC_TABLE_SYMBOL = 'truncate_src_table:' +TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' +TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' +INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート方法に不備がありました。　インポート方法は "{TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT}" のように設定してください' # クラス変数 s3_client = boto3.client('s3') @@ -161,6 +167,18 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ⑦ ロードスキーマのデータを蓄積スキーマにUPSERTする print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-08 - ロードスキーマ（{settings_list[SETTINGS_ITEM["loadSchemaName"]]}）のデータを蓄積スキーマ（{settings_list[SETTINGS_ITEM["storageSchemaName"]]}）に登録します') + + # インポート方法判断 + try: + if truncate_judge(settings_list): + with conn.cursor() as cur: + sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["storageSchemaName"]]}' + cur.execute(sql_truncate) + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} E-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') + + except InvalidConfigException as e: + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-01 - エラー内容：{e}') + error(bucket_name, target_data_source, target_file_name, log_info) # SQL文生成 sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["storageSchemaName"]]} (' @@ -272,3 +290,34 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) + + +def truncate_judge(settings_list): + """TRUNCATE処理対応判定 + Args: + settings_list (list): 個別設定ファイル + Raises: + InvalidConfigException: 個別設定ファイルのインポート方法の設定ミス + Returns: + Bool: Truncate対象の場合True。Truncate対象でない場合False + """ + + # upsert判定 + if len(settings_list) < 13: + return False + if not settings_list[SETTINGS_ITEM["importManner"]]: + return False + + # インポート方法設定チェック + if not settings_list[SETTINGS_ITEM["importManner"]].startswith(TRUNCATE_SRC_TABLE_SYMBOL): + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + import_manner_splitted_list = settings_list[SETTINGS_ITEM["importManner"]].split(':') + if len(import_manner_splitted_list) != 2: + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + if not import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + return True + + +class InvalidConfigException(Exception): + pass From 53fb21f976b030baf3aa4768fa4fbe70cba8b287 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 1 Jul 2022 14:27:17 +0900 Subject: [PATCH 02/10] =?UTF-8?q?Revert=20"refactor:=20=E3=83=AD=E3=83=BC?= =?UTF-8?q?=E3=82=AB=E3=83=AB=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92?= =?UTF-8?q?git=E5=AF=BE=E8=B1=A1=E5=A4=96=E3=81=AB=E8=A8=AD=E5=AE=9A"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit e97c26ac147ec4458e396bdd1ba72ba1e9d72ccb. --- .gitignore | 8 +---- ecs/dataimport/dataimport/main.py | 49 ------------------------------- 2 files changed, 1 insertion(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 3c4e16ef..e4f6d9fb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,4 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json -lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* -__pycache__/ -.env -launch.json -settings.json -Pipfile -Pipfile.lock \ No newline at end of file +lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* \ No newline at end of file diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 5540f886..730a7be8 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,5 +1,4 @@ from datetime import datetime -from pickle import FALSE import boto3 import pymysql from pymysql.constants import CLIENT @@ -24,7 +23,6 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, - 'importManner': 12, } LINE_FEED_CODE = { 'CR': '\r', @@ -32,10 +30,6 @@ LINE_FEED_CODE = { 'CRLF': '\r\n', } DIRECTORY_SETTINGS = '/settings/' -TRUNCATE_SRC_TABLE_SYMBOL = 'truncate_src_table:' -TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' -TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' -INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート方法に不備がありました。　インポート方法は "{TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT}" のように設定してください' # クラス変数 s3_client = boto3.client('s3') @@ -167,18 +161,6 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ⑦ ロードスキーマのデータを蓄積スキーマにUPSERTする print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-08 - ロードスキーマ（{settings_list[SETTINGS_ITEM["loadSchemaName"]]}）のデータを蓄積スキーマ（{settings_list[SETTINGS_ITEM["storageSchemaName"]]}）に登録します') - - # インポート方法判断 - try: - if truncate_judge(settings_list): - with conn.cursor() as cur: - sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["storageSchemaName"]]}' - cur.execute(sql_truncate) - print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} E-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') - - except InvalidConfigException as e: - print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-01 - エラー内容：{e}') - error(bucket_name, target_data_source, target_file_name, log_info) # SQL文生成 sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["storageSchemaName"]]} (' @@ -290,34 +272,3 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) - - -def truncate_judge(settings_list): - """TRUNCATE処理対応判定 - Args: - settings_list (list): 個別設定ファイル - Raises: - InvalidConfigException: 個別設定ファイルのインポート方法の設定ミス - Returns: - Bool: Truncate対象の場合True。Truncate対象でない場合False - """ - - # upsert判定 - if len(settings_list) < 13: - return False - if not settings_list[SETTINGS_ITEM["importManner"]]: - return False - - # インポート方法設定チェック - if not settings_list[SETTINGS_ITEM["importManner"]].startswith(TRUNCATE_SRC_TABLE_SYMBOL): - raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) - import_manner_splitted_list = settings_list[SETTINGS_ITEM["importManner"]].split(':') - if len(import_manner_splitted_list) != 2: - raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) - if not import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: - raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) - return True - - -class InvalidConfigException(Exception): - pass From ca26597a086f84644c1427b4ec34cb68a2e1b3f2 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 1 Jul 2022 14:27:17 +0900 Subject: [PATCH 03/10] =?UTF-8?q?refactor:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92git?= =?UTF-8?q?=E5=AF=BE=E8=B1=A1=E5=A4=96=E3=81=AB=E8=A8=AD=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e4f6d9fb..3c4e16ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json -lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* \ No newline at end of file +lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* +__pycache__/ +.env +launch.json +settings.json +Pipfile +Pipfile.lock \ No newline at end of file From 9057515df95716f634c401c94f1fe0e6454722e7 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 1 Jul 2022 14:27:17 +0900 Subject: [PATCH 04/10] =?UTF-8?q?feat:CRM=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E9=80=A3=E6=90=BA=E3=80=80=E3=83=87=E3=83=BC=E3=82=BF=E5=8F=96?= =?UTF-8?q?=E8=BE=BC=E3=81=AE=E5=85=A8=E4=BB=B6=E7=BD=AE=E3=81=8D=E6=8F=9B?= =?UTF-8?q?=E3=81=88=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 730a7be8..5540f886 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,4 +1,5 @@ from datetime import datetime +from pickle import FALSE import boto3 import pymysql from pymysql.constants import CLIENT @@ -23,6 +24,7 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, + 'importManner': 12, } LINE_FEED_CODE = { 'CR': '\r', @@ -30,6 +32,10 @@ LINE_FEED_CODE = { 'CRLF': '\r\n', } DIRECTORY_SETTINGS = '/settings/' +TRUNCATE_SRC_TABLE_SYMBOL = 'truncate_src_table:' +TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' +TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' +INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート方法に不備がありました。　インポート方法は "{TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT}" のように設定してください' # クラス変数 s3_client = boto3.client('s3') @@ -161,6 +167,18 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ⑦ ロードスキーマのデータを蓄積スキーマにUPSERTする print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-08 - ロードスキーマ（{settings_list[SETTINGS_ITEM["loadSchemaName"]]}）のデータを蓄積スキーマ（{settings_list[SETTINGS_ITEM["storageSchemaName"]]}）に登録します') + + # インポート方法判断 + try: + if truncate_judge(settings_list): + with conn.cursor() as cur: + sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["storageSchemaName"]]}' + cur.execute(sql_truncate) + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} E-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') + + except InvalidConfigException as e: + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-01 - エラー内容：{e}') + error(bucket_name, target_data_source, target_file_name, log_info) # SQL文生成 sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["storageSchemaName"]]} (' @@ -272,3 +290,34 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) + + +def truncate_judge(settings_list): + """TRUNCATE処理対応判定 + Args: + settings_list (list): 個別設定ファイル + Raises: + InvalidConfigException: 個別設定ファイルのインポート方法の設定ミス + Returns: + Bool: Truncate対象の場合True。Truncate対象でない場合False + """ + + # upsert判定 + if len(settings_list) < 13: + return False + if not settings_list[SETTINGS_ITEM["importManner"]]: + return False + + # インポート方法設定チェック + if not settings_list[SETTINGS_ITEM["importManner"]].startswith(TRUNCATE_SRC_TABLE_SYMBOL): + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + import_manner_splitted_list = settings_list[SETTINGS_ITEM["importManner"]].split(':') + if len(import_manner_splitted_list) != 2: + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + if not import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + return True + + +class InvalidConfigException(Exception): + pass From 8fa4f7df8ddcc817c1a0864085daf3326450f21a Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 4 Jul 2022 19:55:55 +0900 Subject: [PATCH 05/10] =?UTF-8?q?refactor:=E3=83=9F=E3=82=B9=E3=81=AB?= =?UTF-8?q?=E3=82=88=E3=82=8B=E9=87=8D=E8=A4=87=E8=A1=8C=E3=81=AE=E5=89=8A?= =?UTF-8?q?=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 5540f886..7e5cfdf9 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -34,7 +34,6 @@ LINE_FEED_CODE = { DIRECTORY_SETTINGS = '/settings/' TRUNCATE_SRC_TABLE_SYMBOL = 'truncate_src_table:' TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' -TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート方法に不備がありました。　インポート方法は "{TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT}" のように設定してください' # クラス変数 From 012a88490f42c2e42c0f6b76c17393d7efc3ecf5 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 7 Jul 2022 02:24:04 +0900 Subject: [PATCH 06/10] =?UTF-8?q?fix:=E3=82=AB=E3=83=B3=E3=83=9E=E5=89=8A?= =?UTF-8?q?=E9=99=A4=E5=87=A6=E7=90=86=E3=81=AE=E8=A8=AD=E5=AE=9A=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E4=BF=AE=E6=AD=A3=E3=81=AB=E4=BC=B4?= =?UTF-8?q?=E3=81=86=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 7e5cfdf9..44d63705 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -24,7 +24,14 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, - 'importManner': 12, + 'reserved0': 12, + 'importManner': 13, + 'reserved1': 14, + 'reserved2': 15, + 'reserved3': 16, + 'reserved4': 17, + 'reserved5': 18, + 'reserved6': 19 } LINE_FEED_CODE = { 'CR': '\r', @@ -80,13 +87,18 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf cur.execute(f'SET time_zone = "+9:00"') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-03 - タイムゾーンを変更しました') - # ④ 個別設定ファイルのロードスキーマのテーブル名に記載されているテーブルをTRUNCATEする + # 個別設定ファイルの読み込み settings_obj = s3_resource.Object(bucket_name, settings_key) settings_response = settings_obj.get() settings_list = [] for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'): settings_list.append(line.rstrip('\n')) + # 予約行挿入のためsetting_listとSETTINGS_ITEMの要素数を揃える + for _ in range(len(SETTINGS_ITEM) - len(settings_list)): + settings_list.append('') + + # ロードスキーマのTRUNCATE with conn.cursor() as cur: sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["loadSchemaName"]]}' cur.execute(sql_truncate) @@ -302,8 +314,6 @@ def truncate_judge(settings_list): """ # upsert判定 - if len(settings_list) < 13: - return False if not settings_list[SETTINGS_ITEM["importManner"]]: return False From b64090f06b183433826481948923a416c86fd880 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 7 Jul 2022 11:32:28 +0900 Subject: [PATCH 07/10] =?UTF-8?q?fix:I-MAIN-20=E3=81=AE=E3=82=A8=E3=83=A9?= =?UTF-8?q?=E3=83=BC=E3=82=B3=E3=83=BC=E3=83=89=E3=81=AE=E3=83=9F=E3=82=B9?= =?UTF-8?q?=E3=81=AE=E4=BF=AE=E6=AD=A3=E3=80=81=20=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=83=9D=E3=83=BC=E3=83=88=E6=96=B9=E6=B3=95=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=AB=E6=8C=87=E5=AE=9A=E3=81=AE=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6=E3=83=9F=E3=82=B9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 44d63705..ff1818bd 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -185,7 +185,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf with conn.cursor() as cur: sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["storageSchemaName"]]}' cur.execute(sql_truncate) - print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} E-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') except InvalidConfigException as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-01 - エラー内容：{e}') @@ -323,7 +323,7 @@ def truncate_judge(settings_list): import_manner_splitted_list = settings_list[SETTINGS_ITEM["importManner"]].split(':') if len(import_manner_splitted_list) != 2: raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) - if not import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: + if import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) return True From d212a101c53dee18927791e7f22c60523f92a96a Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 8 Jul 2022 11:32:18 +0900 Subject: [PATCH 08/10] =?UTF-8?q?refactor:=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index ff1818bd..05e5c353 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,5 +1,4 @@ from datetime import datetime -from pickle import FALSE import boto3 import pymysql from pymysql.constants import CLIENT @@ -87,6 +86,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf cur.execute(f'SET time_zone = "+9:00"') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-03 - タイムゾーンを変更しました') + # ④ 個別設定ファイルのロードスキーマのテーブル名に記載されているテーブルをTRUNCATEする # 個別設定ファイルの読み込み settings_obj = s3_resource.Object(bucket_name, settings_key) settings_response = settings_obj.get() From 48cb17e8fe245452e5408f58d7d1ecd54afcdca0 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 8 Jul 2022 13:03:48 +0900 Subject: [PATCH 09/10] =?UTF-8?q?docs:=20README=E3=82=92=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=E3=80=82=E4=B8=BB=E3=81=AB=E7=92=B0=E5=A2=83=E6=A7=8B=E7=AF=89?= =?UTF-8?q?=E3=81=AB=E9=96=A2=E3=81=99=E3=82=8B=E3=81=93=E3=81=A8=E3=82=92?= =?UTF-8?q?=E8=A8=98=E8=BC=89=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/README.md | 48 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 ecs/dataimport/README.md diff --git a/ecs/dataimport/README.md b/ecs/dataimport/README.md new file mode 100644 index 00000000..8502b98e --- /dev/null +++ b/ecs/dataimport/README.md @@ -0,0 +1,48 @@ +# データ取り込み処理 + +## 概要 + +データ取り込みバケット(`mbj-newdwh2021-<環境名>-data`)に配置されたデータファイルを、設定に基づいてデータベースに登録する処理を行う +処理順序等の詳細は設計書を参照のこと + +## 環境構築 + +### 事前準備 + +- [Wiki - Python環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)の「pipenvの導入」まで完了していること + +### Python仮想環境にパッケージをインストール + +- このドキュメントと同じ階層でコマンドラインを開き、以下のコマンドを実行する + + ```sh + pipenv install -r requirements.txt + ``` + +- 以降、依存モジュールの追加が発生した場合に、`requirements.txt`に追記した上で、上記のコマンドを実行すること + +## ローカルでの実行手順 + +- 当ディレクトリ内に`.vscode/launch.json`を作成し、以下のコードを貼り付ける + - 既にある場合は作成不要 + + ```json + { + "version": "0.2.0", + "configurations": [ + { + "name": "Python: データ取り込み処理", + "type": "python", + "request": "launch", + "program": "ecs/dataimport/controller.py", + "console": "integratedTerminal", + "justMyCode": true, + "envFile": "${workspaceFolder}/.env", + } + ] + } + ``` + +- 当ディレクトリ内に`.env`ファイルを、作成し、環境変数を設定する + - 設定する環境変数は設計書を参照のこと +- F5キーを押し、処理を実行する From 0fa358d53e41dcabd98992dad0122629ef0ca023 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 8 Jul 2022 13:21:51 +0900 Subject: [PATCH 10/10] =?UTF-8?q?refactor:=E4=B8=8B=E7=94=B0=E3=81=95?= =?UTF-8?q?=E3=82=93=E3=81=AE=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC=E6=8C=87?= =?UTF-8?q?=E6=91=98=E5=8F=8D=E6=98=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 5 +-- .vscode/launch.json | 25 +++++++++++ ecs/Dockerfile/Pipfile | 13 ++++++ ecs/Dockerfile/Pipfile.lock | 85 +++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 ecs/Dockerfile/Pipfile create mode 100644 ecs/Dockerfile/Pipfile.lock diff --git a/.gitignore b/.gitignore index 3c4e16ef..15860e8b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,4 @@ lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* __pycache__/ .env -launch.json -settings.json -Pipfile -Pipfile.lock \ No newline at end of file +settings.json \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..753ba49f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,25 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: データ取り込みローカル実行", + "type": "python", + "request": "launch", + // windowsだと\区切りかも + "program": "ecs\\dataimport\\dataimport\\controller.py", + "console": "integratedTerminal", + "justMyCode": true, + "envFile": "${workspaceFolder}/.env" + }, + { + "name": "Python: Attach using Process Id", + "type": "python", + "request": "attach", + "processId": "${command:pickProcess}", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/Dockerfile/Pipfile b/ecs/Dockerfile/Pipfile new file mode 100644 index 00000000..89112f6f --- /dev/null +++ b/ecs/Dockerfile/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +boto3 = "*" +pymysql = "*" + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/ecs/Dockerfile/Pipfile.lock b/ecs/Dockerfile/Pipfile.lock new file mode 100644 index 00000000..12a98015 --- /dev/null +++ b/ecs/Dockerfile/Pipfile.lock @@ -0,0 +1,85 @@ +{ + "_meta": { + "hash": { + "sha256": "7c69c0c237f231fcd67984f1d1b171c2eebfe00ed1877894bbbe77e201862057" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "boto3": { + "hashes": [ + "sha256:490f5e88f5551b33ae3019a37412158b76426d63d1fb910968ade9b6a024e5fe", + "sha256:e284705da36faa668c715ae1f74ebbff4320dbfbe3a733df3a8ab076d1ed1226" + ], + "index": "pypi", + "version": "==1.24.14" + }, + "botocore": { + "hashes": [ + "sha256:bb56fa77b8fa1ec367c2e16dee62d60000451aac5140dcce3ebddc167fd5c593", + "sha256:df1e9b208ff93daac7c645b0b04fb6dccd7f20262eae24d87941727025cbeece" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.14" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "pymysql": { + "hashes": [ + "sha256:41fc3a0c5013d5f039639442321185532e3e2c8924687abe6537de157d403641", + "sha256:816927a350f38d56072aeca5dfb10221fe1dc653745853d30a216637f5d7ad36" + ], + "index": "pypi", + "version": "==1.0.2" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.8.2" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.16.0" + }, + "urllib3": { + "hashes": [ + "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", + "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.9" + } + }, + "develop": {} +}