From 1a1499a1895043bf6ce0b0e109f130cbe61b8dd3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 27 Jan 2023 16:10:27 +0900 Subject: [PATCH 01/18] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=99=BB=E9=8C=B2=E5=87=A6=E7=90=86=20=E5=88=9D=E6=9C=9F?= =?UTF-8?q?=E5=87=A6=E7=90=86=E5=86=85=E3=81=AEresource=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=82=92=E3=82=AF?= =?UTF-8?q?=E3=83=A9=E3=82=A4=E3=82=A2=E3=83=B3=E3=83=88=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/ini.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/ecs/dataimport/dataimport/ini.py b/ecs/dataimport/dataimport/ini.py index 30d2ef93..098e2b2f 100644 --- a/ecs/dataimport/dataimport/ini.py +++ b/ecs/dataimport/dataimport/ini.py @@ -1,12 +1,12 @@ -from datetime import datetime -import boto3 -import io import csv +import io import re import sys -from error import error -from error import error_doing_file_exists +from datetime import datetime + +import boto3 from common import debug_log +from error import error, error_doing_file_exists # 定数 LOG_LEVEL = {"i": 'Info', "e": 'Error'} @@ -17,7 +17,6 @@ DIRECTORY_SETTINGS = '/settings/' # クラス変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') def init(bucket_name, target_key, target_data_source, target_file_name, log_info, mode): @@ -60,8 +59,7 @@ def init(bucket_name, target_key, target_data_source, target_file_name, log_info try: # ③ S3バケット内のtargetディレクトリに、「投入データファイル名.doing」ファイルを作成する - doing_obj = s3_resource.Object(bucket_name, doing_key) - doing_obj.put(Body='') + s3_client.put_object(Bucket=bucket_name, Key=doing_key, Body=b'') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-INI-04 - targetディレクトリに {doing_file_name} を作成しました') # ④ 投入データファイルをS3バケット内のtargetディレクトリから、workディレクトリに移動(コピー削除)する @@ -70,8 +68,7 @@ def init(bucket_name, target_key, target_data_source, target_file_name, log_info 'Key': target_key } work_key = target_data_source + DIRECTORY_WORK + target_file_name - work_obj = s3_resource.Object(bucket_name, work_key) - work_obj.copy(copy_source) + s3_client.copy(CopySource=copy_source, Bucket=bucket_name, Key=work_key) s3_client.delete_object(Bucket=bucket_name, Key=target_key) print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-INI-05 - 投入データ {target_file_name} をworkディレクトリに移動しました') except Exception as e: @@ -122,9 +119,8 @@ def init(bucket_name, target_key, target_data_source, target_file_name, log_info try: # ⑦ 個別設定ファイルを特定する print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-INI-17 - 個別設定ファイルを検索します') - mapping_obj = s3_resource.Object(bucket_name, mapping_key) - mapping_response = mapping_obj.get() - mapping_body = io.TextIOWrapper(io.BytesIO(mapping_response["Body"].read()), encoding='utf-8') + mapping_obj_response = s3_client.get_object(Bucket=bucket_name, Key=mapping_key) + mapping_body = io.TextIOWrapper(io.BytesIO(mapping_obj_response["Body"].read()), encoding='utf-8') settings_file_name = '' for row in csv.reader(mapping_body, delimiter='\t'): if row: From d5c6485a729c042266f636b2122e2490a55f8299 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 27 Jan 2023 16:19:44 +0900 Subject: [PATCH 02/18] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=99=BB=E9=8C=B2=E5=87=A6=E7=90=86=20=E3=83=81=E3=82=A7?= =?UTF-8?q?=E3=83=83=E3=82=AF=E5=87=A6=E7=90=86=E5=86=85=E3=81=AEresource?= =?UTF-8?q?=E3=82=A4=E3=83=B3=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E3=82=92=E3=82=AF=E3=83=A9=E3=82=A4=E3=82=A2=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=82=A4=E3=83=B3=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/chk.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index 7e050357..987a54b0 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -4,7 +4,6 @@ import sys from datetime import datetime import boto3 - from common import convert_quotechar, debug_log from end import end from error import error @@ -41,7 +40,7 @@ LINE_FEED_CODE = { } # クラス変数 -s3_resource = boto3.resource('s3') +s3_client = boto3.client('s3') # チェック例外クラス @@ -74,16 +73,14 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-01 - チェック処理を開始します') # データ読込 - settings_obj = s3_resource.Object(bucket_name, settings_key) - settings_response = settings_obj.get() + settings_obj_response = s3_client.get_object(Bucket=bucket_name, Key=settings_key) settings_list = [] - for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'): + for line in io.TextIOWrapper(io.BytesIO(settings_obj_response["Body"].read()), encoding='utf-8'): settings_list.append(line.rstrip('\n')) work_key = target_data_source + DIRECTORY_WORK + target_file_name - work_obj = s3_resource.Object(bucket_name, work_key) - work_response = work_obj.get() - work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) + work_obj_response = s3_client.get_object(Bucket=bucket_name, Key=work_key) + work_data = io.TextIOWrapper(io.BytesIO(work_obj_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) work_csv_row = [] for i, line in enumerate(csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]])): # ヘッダあり、かつ、1行目の場合 From 419bad2690d10a41e46f2204956de5131b7986cb Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 27 Jan 2023 17:55:09 +0900 Subject: [PATCH 03/18] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=99=BB=E9=8C=B2=E5=87=A6=E7=90=86=20=E7=B5=82=E4=BA=86?= =?UTF-8?q?=E5=87=A6=E7=90=86=E5=86=85=E3=81=AEresource=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=82=92=E3=82=AF?= =?UTF-8?q?=E3=83=A9=E3=82=A4=E3=82=A2=E3=83=B3=E3=83=88=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/end.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/ecs/dataimport/dataimport/end.py b/ecs/dataimport/dataimport/end.py index e4d53482..3b31c56b 100644 --- a/ecs/dataimport/dataimport/end.py +++ b/ecs/dataimport/dataimport/end.py @@ -1,7 +1,8 @@ from datetime import datetime + import boto3 -from error import error from common import debug_log +from error import error # 定数 LOG_LEVEL = {'i': 'Info', 'e': 'Error'} @@ -12,7 +13,6 @@ DIRECTORY_WARNING = '/warning/' # クラス変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') def end(bucket_name, target_data_source, target_file_name, warning_info, log_info, mode): @@ -45,8 +45,7 @@ def end(bucket_name, target_data_source, target_file_name, warning_info, log_inf } done_file_name = f'{datetime.now():%Y%m%d%H%M%S}_{target_file_name}' done_key = target_data_source + DIRECTORY_DONE + done_file_name - done_obj = s3_resource.Object(bucket_name, done_key) - done_obj.copy(copy_source) + s3_client.copy(CopySource=copy_source, Bucket=bucket_name, Key=done_key) s3_client.delete_object(Bucket=bucket_name, Key=work_key) print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-END-02 - workディレクトリの {target_file_name} をdoneディレクトリに移動しました 移動後ファイル名：{done_file_name}') @@ -64,23 +63,20 @@ def end(bucket_name, target_data_source, target_file_name, warning_info, log_inf # warningファイルの作成 warning_file_name = f'{datetime.now():%Y%m%d%H%M%S}_{target_file_name}_war.log' warning_key = target_data_source + DIRECTORY_WARNING + warning_file_name - warning_obj = s3_resource.Object(bucket_name, warning_key) - warning_obj.put(Body=warning_info) + s3_client.put_object(Bucket=bucket_name, Key=warning_key, Body=bytes(warning_info, 'utf-8')) print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-END-06 - warningディレクトリに {warning_file_name} を作成しました') # warning処理結果ファイルの作成 result_warning_file_name = target_file_name + '.warning' result_warning_key = target_data_source + DIRECTORY_TARGET + result_warning_file_name - result_warning_obj = s3_resource.Object(bucket_name, result_warning_key) - result_warning_obj.put(Body='') + s3_client.put_object(Bucket=bucket_name, Key=result_warning_key, Body=b'') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-END-07 - targetディレクトリに {result_warning_file_name} を作成しました') else: # done処理結果ファイルの作成 print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-END-08 - Warning情報は存在しませんでした') result_done_file_name = target_file_name + '.done' result_done_key = target_data_source + DIRECTORY_TARGET + result_done_file_name - result_done_obj = s3_resource.Object(bucket_name, result_done_key) - result_done_obj.put(Body='') + s3_client.put_object(Bucket=bucket_name, Key=result_done_key, Body=b'') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-END-09 - targetディレクトリに {result_done_file_name} を作成しました') # ⑤ 終了処理終了ログを出力する @@ -88,3 +84,17 @@ def end(bucket_name, target_data_source, target_file_name, warning_info, log_inf except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-END-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) + + +# テスト用コード +# 値はよしなに変えてください +# if __name__ == '__main__': +# end( +# bucket_name='バケット名', +# target_data_source='データソース名', +# target_file_name='targetフォルダ内のファイル', +# # warning_info='ワーニング内容', # ワーニングがある場合のテストはこちらを生かす +# warning_info='', +# log_info='Info', +# mode='i' +# ) From 7a86e834dcd5468474612abc6507c9cf5a9b9577 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 27 Jan 2023 17:58:29 +0900 Subject: [PATCH 04/18] =?UTF-8?q?feat:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E3=81=A7=E5=AE=9F=E8=A1=8C=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E5=AE=9F=E8=A1=8C=E7=94=A8=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=89=E3=82=92=E8=BF=BD=E5=8A=A0=E3=80=82=E6=99=AE?= =?UTF-8?q?=E6=AE=B5=E3=81=AF=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=82=A2?= =?UTF-8?q?=E3=82=A6=E3=83=88=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/chk.py | 13 +++++++++++++ ecs/dataimport/dataimport/end.py | 2 +- ecs/dataimport/dataimport/ini.py | 12 ++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index 987a54b0..f6bb97c6 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -145,3 +145,16 @@ def is_empty_file(work_csv_row: list, settings_list: list): return len(work_csv_row[1:]) == 0 return len(work_csv_row) == 0 + + +# ローカル実行用コード +# 値はよしなに変えてください +# if __name__ == '__main__': +# check( +# bucket_name='バケット名', +# target_data_source='データソース名', +# target_file_name='targetフォルダ内のファイル名', +# settings_key='個別設定ファイル名', +# log_info='Info', +# mode='i' +# ) diff --git a/ecs/dataimport/dataimport/end.py b/ecs/dataimport/dataimport/end.py index 3b31c56b..dce8747c 100644 --- a/ecs/dataimport/dataimport/end.py +++ b/ecs/dataimport/dataimport/end.py @@ -86,7 +86,7 @@ def end(bucket_name, target_data_source, target_file_name, warning_info, log_inf error(bucket_name, target_data_source, target_file_name, log_info) -# テスト用コード +# ローカル実行用コード # 値はよしなに変えてください # if __name__ == '__main__': # end( diff --git a/ecs/dataimport/dataimport/ini.py b/ecs/dataimport/dataimport/ini.py index 098e2b2f..0d77ca10 100644 --- a/ecs/dataimport/dataimport/ini.py +++ b/ecs/dataimport/dataimport/ini.py @@ -155,3 +155,15 @@ def init(bucket_name, target_key, target_data_source, target_file_name, log_info except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-INI-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) + +# ローカル実行用コード +# 値はよしなに変えてください +# if __name__ == '__main__': +# init( +# bucket_name='バケット名', +# target_key='データソース名/target/ファイル名', +# target_data_source='データソース名', +# target_file_name='ファイル名', +# log_info='Info', +# mode='i' +# ) From 7a21b22861e957d7a02c38baf5054ae90de68554 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 30 Jan 2023 10:33:19 +0900 Subject: [PATCH 05/18] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=99=BB=E9=8C=B2=E5=87=A6=E7=90=86=20=E3=82=A8=E3=83=A9?= =?UTF-8?q?=E3=83=BC=E5=87=A6=E7=90=86=E5=86=85=E3=81=AEresource=E3=82=A4?= =?UTF-8?q?=E3=83=B3=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=82=92?= =?UTF-8?q?=E3=82=AF=E3=83=A9=E3=82=A4=E3=82=A2=E3=83=B3=E3=83=88=E3=82=A4?= =?UTF-8?q?=E3=83=B3=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/error.py | 41 +++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/ecs/dataimport/dataimport/error.py b/ecs/dataimport/dataimport/error.py index c81eca65..13678d19 100644 --- a/ecs/dataimport/dataimport/error.py +++ b/ecs/dataimport/dataimport/error.py @@ -1,6 +1,7 @@ -from datetime import datetime -import boto3 import sys +from datetime import datetime + +import boto3 # 定数 LOG_LEVEL = {'i': 'Info', 'e': 'Error'} @@ -10,7 +11,6 @@ DIRECTORY_ERROR = '/error/' # クラス変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') def error(bucket_name, target_data_source, target_file_name, log_info): @@ -34,8 +34,7 @@ def error(bucket_name, target_data_source, target_file_name, log_info): } error_file_name = f'{datetime.now():%Y%m%d%H%M%S}_{target_file_name}' error_key = target_data_source + DIRECTORY_ERROR + error_file_name - error_obj = s3_resource.Object(bucket_name, error_key) - error_obj.copy(copy_source) + s3_client.copy(CopySource=copy_source, Bucket=bucket_name, Key=error_key) s3_client.delete_object(Bucket=bucket_name, Key=work_key) print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-ERR-02 - workディレクトリの {target_file_name} をerrorディレクトリに移動しました 移動後ファイル名：{error_file_name}') @@ -48,8 +47,7 @@ def error(bucket_name, target_data_source, target_file_name, log_info): # ④ S3バケット内のtargetディレクトリに、「投入データファイル名.error」ファイルを作成する result_error_file_name = target_file_name + '.error' result_error_key = target_data_source + DIRECTORY_TARGET + result_error_file_name - result_error_obj = s3_resource.Object(bucket_name, result_error_key) - result_error_obj.put(Body='') + s3_client.put_object(Bucket=bucket_name, Key=result_error_key, Body=b'') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-ERR-04 - targetディレクトリに {result_error_file_name} を作成しました') except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-ERR-99 - エラー内容：{e}') @@ -81,16 +79,14 @@ def error_doing_file_exists(bucket_name, target_key, target_data_source, target_ } error_file_name = f'{datetime.now():%Y%m%d%H%M%S}_{target_file_name}' error_key = target_data_source + DIRECTORY_ERROR + error_file_name - error_obj = s3_resource.Object(bucket_name, error_key) - error_obj.copy(copy_source) + s3_client.copy(CopySource=copy_source, Bucket=bucket_name, Key=error_key) s3_client.delete_object(Bucket=bucket_name, Key=target_key) print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-ERR-07 - targetディレクトリの {target_file_name} をerrorディレクトリに移動しました 移動後ファイル名：{error_file_name}') # ③ S3バケット内のtargetディレクトリに、「投入データファイル名.exclusive_error」ファイルを作成する result_error_file_name = target_file_name + '.exclusive_error' result_error_key = target_data_source + DIRECTORY_TARGET + result_error_file_name - result_error_obj = s3_resource.Object(bucket_name, result_error_key) - result_error_obj.put(Body='') + s3_client.put_object(Bucket=bucket_name, Key=result_error_key, Body=b'') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-ERR-08 - targetディレクトリに {result_error_file_name} を作成しました') except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-ERR-99 - エラー内容：{e}') @@ -99,4 +95,25 @@ def error_doing_file_exists(bucket_name, target_key, target_data_source, target_ print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-ERR-09 - doingファイルが存在した時のエラー処理を終了します') # ⑤ 処理を終了する - sys.exit() \ No newline at end of file + sys.exit() + + +# ローカル実行用コード +# 値はよしなに変えてください +# if __name__ == '__main__': + # エラー処理 + # error( + # bucket_name='バケット名', + # target_data_source='データソース名', + # target_file_name='データソース名/target/ファイル名', + # log_info='Info' + # ) + + # doingファイルの有無チェック関数 + # error_doing_file_exists( + # bucket_name='バケット名', + # target_key='投入データのフルパス', + # target_data_source='投入データのディレクトリ名よりデータソースに該当する部分', + # target_file_name='投入データのファイル名', + # log_info='Info' + # ) From b8975ad45334d82baba222b9e2ae95fa50f325c8 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 30 Jan 2023 11:17:41 +0900 Subject: [PATCH 06/18] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=99=BB=E9=8C=B2=E5=87=A6=E7=90=86=20=E3=83=A1=E3=82=A4?= =?UTF-8?q?=E3=83=B3=E5=87=A6=E7=90=86=E5=86=85=E3=81=AEresource=E3=82=A4?= =?UTF-8?q?=E3=83=B3=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=82=92?= =?UTF-8?q?=E3=82=AF=E3=83=A9=E3=82=A4=E3=82=A2=E3=83=B3=E3=83=88=E3=82=A4?= =?UTF-8?q?=E3=83=B3=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index ef90f598..3233472c 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -5,10 +5,9 @@ from datetime import datetime import boto3 import pymysql -from pymysql.constants import CLIENT - from common import convert_quotechar, debug_log from error import error +from pymysql.constants import CLIENT # 定数 DIRECTORY_WORK = '/work/' @@ -47,7 +46,6 @@ INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート # クラス変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') def main(bucket_name, target_data_source, target_file_name, settings_key, db_info, log_info, mode): @@ -91,8 +89,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ④ 個別設定ファイルのロードスキーマのテーブル名に記載されているテーブルをTRUNCATEする # 個別設定ファイルの読み込み - settings_obj = s3_resource.Object(bucket_name, settings_key) - settings_response = settings_obj.get() + settings_response = s3_client.get_object(Bucket=bucket_name, Key=settings_key) settings_list = [] for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'): settings_list.append(line.rstrip('\n')) @@ -110,8 +107,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ⑤ 投入データファイルを１行ごとにループする print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-05 - 投入データ {target_file_name} の読み込みを開始します') work_key = target_data_source + DIRECTORY_WORK + target_file_name - work_obj = s3_resource.Object(bucket_name, work_key) - work_response = work_obj.get() + work_response = s3_client.get_object(Bucket=bucket_name, Key=work_key) work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) process_count = 0 # 処理件数カウンタ @@ -261,10 +257,9 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf try: if ex_sql_file_exists: # 拡張SQLファイルからSQL文生成 - ex_sqls_obj = s3_resource.Object(bucket_name, ex_sql_key) - ex_sql_response = ex_sqls_obj.get() + ex_sql_obj_response = s3_client.get_object(Bucket=bucket_name, Key=ex_sql_key) ex_sql = '' - for line in io.TextIOWrapper(io.BytesIO(ex_sql_response["Body"].read()), encoding='utf-8'): + for line in io.TextIOWrapper(io.BytesIO(ex_sql_obj_response["Body"].read()), encoding='utf-8'): ex_sql = f'{ex_sql} {line.rstrip()}' # トランザクション開始 @@ -358,3 +353,18 @@ def truncate_judge(settings_list): class InvalidConfigException(Exception): pass + + +# ローカル実行用コード +# 値はよしなに変えてください +# if __name__ == '__main__': +# DB_INFO = {"host": '127.0.0.1', "name": 'org02', "pass": 'user', "user": 'user'} +# main( +# bucket_name='バケット名', +# target_data_source='投入データのディレクトリ名よりデータソースに該当する部分', +# target_file_name='投入データのファイル名', +# settings_key='投入データに該当する個別設定ファイルのフルパス', +# db_info=DB_INFO, +# log_info='info', +# mode='i' +# ) From b5a6dde1a51d5253567e0a4cb10dd56e2f0d2c82 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Mon, 13 May 2024 09:43:11 +0900 Subject: [PATCH 07/18] crm-datafech fix --- ecs/crm-datafetch/src/aws/s3.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index 17e4a1f3..ed062fca 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -14,21 +14,21 @@ from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, class S3Resource: def __init__(self, bucket_name: str) -> None: - self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) - self.__s3_bucket = self.__s3_resource.Bucket(bucket_name) + self.__s3_client = boto3.client(AWS_RESOURCE_S3) + self.__s3_bucket = bucket_name def get_object(self, object_key: str) -> str: - response = self.__s3_bucket.Object(object_key).get() + response = self.__s3_client.get_object(Bucket=self.__s3_bucket, Key=object_key) body = response[S3_RESPONSE_BODY].read() return body.decode(S3_CHAR_CODE) def put_object(self, object_key: str, local_file_path: str) -> None: - self.__s3_bucket.upload_file(Key=object_key, Filename=local_file_path) + self.__s3_client.upload_file(Filename=local_file_path, Bucket=self.__bucket_name, Key=object_key) return def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: copy_source = {'Bucket': src_bucket, 'Key': src_key} - self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) + self.__s3_client.copy_object(CopySource=copy_source, Bucket=dest_bucket, Key=dest_key) return From 2471c86c6ca5db0c4199e72deafe52f7d595e081 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Mon, 13 May 2024 16:17:00 +0900 Subject: [PATCH 08/18] sap-data-decrypt-fix --- lambda/sap-data-decrypt/datadecrypt/main.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/lambda/sap-data-decrypt/datadecrypt/main.py b/lambda/sap-data-decrypt/datadecrypt/main.py index 86eeb63f..836f7c18 100644 --- a/lambda/sap-data-decrypt/datadecrypt/main.py +++ b/lambda/sap-data-decrypt/datadecrypt/main.py @@ -37,7 +37,6 @@ TARGET_BUCKET_BY_DATA_SOURCE = { # 変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') sns_client = boto3.client('sns') # logger設定 @@ -125,9 +124,8 @@ def handler(event, context): decrypt_file_key = f'{execute_date}/{decrypt_file_name}' decrypt_bucket_name = TARGET_BUCKET_BY_DATA_SOURCE[s3_event.data_source_name] logger.info(f'I-07-02 復号化ファイル出力　ファイル名：{decrypt_file_name}　出力先：{decrypt_bucket_name}/{decrypt_file_key}') - decrypt_file_obj = s3_resource.Object(decrypt_bucket_name, decrypt_file_key) - decrypt_file_obj.put(Body=decrypt_file) - decrypt_file.close + s3_client.put_object(Bucket=decrypt_bucket_name, Key=decrypt_file_key, Body=decrypt_file) + decrypt_file.close() logger.info('I-07-03 復号化ファイルをS3に出力しました') except Exception as e: logger.exception(f'E-07-01 復号化ファイルのS3出力に失敗しました　エラー内容：{e}') @@ -142,8 +140,7 @@ def handler(event, context): backup_file_key = f'{s3_event.data_source_name}/{execute_date}/{s3_event.file_name}' logger.info( f'I-07-04 PGP暗号化ファイル移動　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{SAP_DATA_BACKUP_BUCKET_NAME}/{backup_file_key}') - backup_file_obj = s3_resource.Object(SAP_DATA_BACKUP_BUCKET_NAME, backup_file_key) - backup_file_obj.copy(copy_source) + s3_client.copy_object(Bucket=SAP_DATA_BACKUP_BUCKET_NAME, Key=backup_file_key, CopySource=copy_source) s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path) logger.info('I-07-05 PGP暗号化ファイルをバックアップ用バケットに移動しました') except Exception as e: @@ -158,8 +155,7 @@ def handler(event, context): import_file_folder = f'{s3_event.data_source_name}{DIRECTORY_TARGET}' import_file_key = f'{import_file_folder}{decrypt_file_name}' logger.info(f'I-07-06 復号化ファイル出力　ファイル名：{decrypt_file_name}　出力先：{s3_event.bucket_name}/{import_file_folder}') - import_file_obj = s3_resource.Object(s3_event.bucket_name, import_file_key) - import_file_obj.copy(copy_source) + s3_client.copy_object(Bucket=s3_event.bucket_name, Key=import_file_key, CopySource=copy_source) logger.info(f'I-07-07 復号化ファイルをS3に出力しました') except Exception as e: logger.exception(f'E-07-03 復号化ファイルのS3出力に失敗しました　エラー内容：{e}') @@ -186,8 +182,7 @@ def create_status_file(s3_event, extension) -> None: try: result_error_file_name = s3_event.file_name + extension result_error_key = s3_event.data_source_name + DIRECTORY_RECV + result_error_file_name - result_error_obj = s3_resource.Object(s3_event.bucket_name, result_error_key) - result_error_obj.put(Body='') + s3_client.put_object(Bucket=s3_event.bucket_name, Key=result_error_key, Body='') logger.error( f'E-ERR-01 recvディレクトリにエラーファイルを作成しました　ファイル名：{result_error_file_name}　出力先：{s3_event.bucket_name}/{result_error_key}') except Exception as e: @@ -204,8 +199,7 @@ def move_encrypt_file(s3_event) -> None: } error_file_name = f'{datetime.datetime.now():%Y%m%d%H%M%S}_{s3_event.file_name}' error_key = s3_event.data_source_name + DIRECTORY_RECV_ERROR + error_file_name - error_obj = s3_resource.Object(s3_event.bucket_name, error_key) - error_obj.copy(copy_source) + s3_client.copy_object(Bucket=s3_event.bucket_name, Key=error_key, CopySource=copy_source) s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path) logger.error( f'E-ERR-02 recv_errorディレクトリにファイルを移動しました　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{s3_event.bucket_name}/{error_key}') From 2ce19984182e3e821aba289ae94fa01919c2526d Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Mon, 13 May 2024 16:29:42 +0900 Subject: [PATCH 09/18] sap-fin-monthly-data-fix --- .../sap-fin-monthly-data-notice-daily.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lambda/sap-fin-monthly-data-notice-daily/sap-fin-monthly-data-notice-daily.py b/lambda/sap-fin-monthly-data-notice-daily/sap-fin-monthly-data-notice-daily.py index ffc15e90..f4cd42b2 100644 --- a/lambda/sap-fin-monthly-data-notice-daily/sap-fin-monthly-data-notice-daily.py +++ b/lambda/sap-fin-monthly-data-notice-daily/sap-fin-monthly-data-notice-daily.py @@ -28,7 +28,6 @@ MAIL_INDENT = '　　' # 変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') sns_client = boto3.client('sns') # logger設定 @@ -69,8 +68,7 @@ def lambda_handler(event, context): # ③ 設定ファイル[SAP_finI/Fファイルネーム設定ファイル（月次）]を読み込む try: logger.info(f'I-03-01 月次I/Fファイルネーム設定ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{RECEIVE_MONTHLY_FILE_NAME_LIST_PATH}') - receive_monthly_file_name_obj = s3_resource.Object(CONFIG_BUCKET_NAME, RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) - receive_monthly_file_name_response = receive_monthly_file_name_obj.get() + receive_monthly_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) logger.info('I-03-02 月次I/Fファイルネーム設定ファイルを読み込みました') except Exception as e: logger.error(f'E-03-01 月次I/Fファイルネーム設定ファイルの読み込みに失敗しました　エラー内容：{e}') From 86945ad25de6fd6f9da0c88e186358ad2f48f807 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Mon, 13 May 2024 17:52:53 +0900 Subject: [PATCH 10/18] sap-fin-check-daily-fix --- .../sap-fin-receive-check-daily.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py b/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py index 0e3379a6..0899fb3c 100644 --- a/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py +++ b/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py @@ -34,7 +34,6 @@ MAIL_INDENT = '　　' # 変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') sns_client = boto3.client('sns') # logger設定 @@ -69,8 +68,7 @@ def lambda_handler(event, context): # 1.設定ファイル[メルク社非営業日設定ファイル]を読み込む try: logger.info(f'I-02-02 非営業日設定ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{NON_BUSINESS_DAY_LIST_PATH}') - non_business_day_obj = s3_resource.Object(CONFIG_BUCKET_NAME, NON_BUSINESS_DAY_LIST_PATH) - non_business_day_response = non_business_day_obj.get() + non_business_day_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=NON_BUSINESS_DAY_LIST_PATH) logger.info('I-02-03 非営業日設定ファイルを読み込みました') except Exception as e: logger.error(f'E-02-01 非営業日設定ファイルの読み込みに失敗しました　エラー内容：{e}') @@ -102,8 +100,7 @@ def lambda_handler(event, context): # ③ 設定ファイル[SAP_finI/Fファイルネーム設定ファイル（日次）]を読み込む try: logger.info(f'I-03-01 日次I/Fファイルネーム設定ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{RECEIVE_DAILY_FILE_NAME_LIST_PATH}') - receive_daily_file_name_obj = s3_resource.Object(CONFIG_BUCKET_NAME, RECEIVE_DAILY_FILE_NAME_LIST_PATH) - receive_daily_file_name_response = receive_daily_file_name_obj.get() + receive_daily_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH) logger.info('I-03-02 日次I/Fファイルネーム設定ファイルを読み込みました') except Exception as e: logger.error(f'E-03-01 日次I/Fファイルネーム設定ファイルの読み込みに失敗しました　エラー内容：{e}') @@ -115,10 +112,10 @@ def lambda_handler(event, context): # 1.SAP保管用バケットの処理稼働日に該当するサブフォルダにあるファイル一覧を取得する logger.info(f'I-04-02 オブジェクトリストの取得　取得先：{CHECK_BUCKET_NAME}/{execute_date}/') object_prefix = f'{execute_date}/' - object_list = s3_resource.Bucket(CHECK_BUCKET_NAME).objects.filter(Prefix=object_prefix) + object_list = s3_client.list_objects_v2(Bucket=CHECK_BUCKET_NAME, Prefix=object_prefix) file_list = [] - for obj in object_list: - obj_key = obj.key.rsplit('/', INDEX_SPLIT_NUM) + for obj in object_list.get('Contents', []): + obj_key = obj['Key'].rsplit('/', INDEX_SPLIT_NUM) file_list.append(obj_key[INDEX_LAST]) # 2.日次I/Fファイルチェック処理 From eb9b70cbb339d3538ea82e7257588462c53dfdc8 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Tue, 14 May 2024 16:22:55 +0900 Subject: [PATCH 11/18] sap-fin-receive-check-monthly-fix --- .../sap-fin-receive-check-monthly.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lambda/sap-fin-receive-check-monthly/sap-fin-receive-check-monthly.py b/lambda/sap-fin-receive-check-monthly/sap-fin-receive-check-monthly.py index e30858c5..25058295 100644 --- a/lambda/sap-fin-receive-check-monthly/sap-fin-receive-check-monthly.py +++ b/lambda/sap-fin-receive-check-monthly/sap-fin-receive-check-monthly.py @@ -34,7 +34,6 @@ MAIL_INDENT = '　　' # 変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') sns_client = boto3.client('sns') # logger設定 @@ -71,8 +70,8 @@ def lambda_handler(event, context): # 1.設定ファイル[チェック処理実施指定日ファイル]を読み込む try: logger.info(f'I-02-02 チェック処理実施指定日ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{MONTHLY_CEHCK_DAY_LIST_PATH}') - monthly_day_obj = s3_resource.Object(CONFIG_BUCKET_NAME, MONTHLY_CEHCK_DAY_LIST_PATH) - monthly_day_response = monthly_day_obj.get() + monthly_day_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=MONTHLY_CEHCK_DAY_LIST_PATH) + logger.info('I-02-03 チェック処理実施指定日ファイルを読み込みました') except Exception as e: logger.error(f'E-02-01 チェック処理実施指定日設定ファイルの読み込みに失敗しました　エラー内容：{e}') @@ -104,8 +103,8 @@ def lambda_handler(event, context): # ③ 設定ファイル[SAP_finI/Fファイルネーム設定ファイル（月次）]を読み込む try: logger.info(f'I-03-01 月次I/Fファイルネーム設定ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{RECEIVE_MONTHLY_FILE_NAME_LIST_PATH}') - receive_monthly_file_name_obj = s3_resource.Object(CONFIG_BUCKET_NAME, RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) - receive_monthly_file_name_response = receive_monthly_file_name_obj.get() + receive_monthly_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) + logger.info('I-03-02 月次I/Fファイルネーム設定ファイルを読み込みました') except Exception as e: logger.error(f'E-03-01 月次I/Fファイルネーム設定ファイルの読み込みに失敗しました　エラー内容：{e}') @@ -117,10 +116,10 @@ def lambda_handler(event, context): # 1.SAP保管用バケットの処理稼働月に該当するサブフォルダにあるファイル一覧を取得する logger.info(f'I-04-02 オブジェクトリストの取得　取得先：{CHECK_BUCKET_NAME}/{execute_month}/') object_prefix = f'{execute_month}/' - object_list = s3_resource.Bucket(CHECK_BUCKET_NAME).objects.filter(Prefix=object_prefix) + object_list = s3_client.list_objects_v2(Bucket=CHECK_BUCKET_NAME, Prefix=object_prefix) file_list = [] - for obj in object_list: - obj_key = obj.key.rsplit('/', INDEX_SPLIT_NUM) + for obj in object_list.get('Contents', []): + obj_key = obj['Key'].rsplit('/', INDEX_SPLIT_NUM) file_list.append(obj_key[INDEX_LAST]) # 2.月次I/Fファイルチェック処理 @@ -128,7 +127,7 @@ def lambda_handler(event, context): logger.info('I-04-04 取得したオブジェクトリストと月次I/Fファイルネーム設定ファイルの突き合わせを開始します') receive_monthly_file_name_body = io.TextIOWrapper(io.BytesIO(receive_monthly_file_name_response["Body"].read()), encoding='utf-8') match_count = 0 - row_count = sum(1 for line in io.BytesIO(receive_monthly_file_name_obj.get()["Body"].read())) + row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH)["Body"].read())) for row in csv.reader(receive_monthly_file_name_body, delimiter='\t'): file_exists = False for file_name in file_list: From d0c37c962ca7f6bb31a4ab50f7879125bcfa1e89 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Tue, 14 May 2024 16:57:17 +0900 Subject: [PATCH 12/18] sap-sup-receive-check-daily-fix --- .../sap-sup-receive-check-daily.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lambda/sap-sup-receive-check-daily/sap-sup-receive-check-daily.py b/lambda/sap-sup-receive-check-daily/sap-sup-receive-check-daily.py index fc728d68..033ae53b 100644 --- a/lambda/sap-sup-receive-check-daily/sap-sup-receive-check-daily.py +++ b/lambda/sap-sup-receive-check-daily/sap-sup-receive-check-daily.py @@ -34,7 +34,6 @@ MAIL_INDENT = '　　' # 変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') sns_client = boto3.client('sns') # logger設定 @@ -69,8 +68,7 @@ def lambda_handler(event, context): # 1.設定ファイル[メルク社非営業日設定ファイル]を読み込む try: logger.info(f'I-02-02 非営業日設定ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{NON_BUSINESS_DAY_LIST_PATH}') - non_business_day_obj = s3_resource.Object(CONFIG_BUCKET_NAME, NON_BUSINESS_DAY_LIST_PATH) - non_business_day_response = non_business_day_obj.get() + non_business_day_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=NON_BUSINESS_DAY_LIST_PATH) logger.info('I-02-03 非営業日設定ファイルを読み込みました') except Exception as e: logger.error(f'E-02-01 非営業日設定ファイルの読み込みに失敗しました　エラー内容：{e}') @@ -102,8 +100,7 @@ def lambda_handler(event, context): # ③ 設定ファイル[SAP_supI/Fファイルネーム設定ファイル（日次）]を読み込む try: logger.info(f'I-03-01 日次I/Fファイルネーム設定ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{RECEIVE_DAILY_FILE_NAME_LIST_PATH}') - receive_daily_file_name_obj = s3_resource.Object(CONFIG_BUCKET_NAME, RECEIVE_DAILY_FILE_NAME_LIST_PATH) - receive_daily_file_name_response = receive_daily_file_name_obj.get() + receive_daily_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH) logger.info('I-03-02 日次I/Fファイルネーム設定ファイルを読み込みました') except Exception as e: logger.error(f'E-03-01 日次I/Fファイルネーム設定ファイルの読み込みに失敗しました　エラー内容：{e}') @@ -115,10 +112,10 @@ def lambda_handler(event, context): # 1.SAP保管用バケットの処理稼働日に該当するサブフォルダにあるファイル一覧を取得する logger.info(f'I-04-02 オブジェクトリストの取得　取得先：{CHECK_BUCKET_NAME}/{execute_date}/') object_prefix = f'{execute_date}/' - object_list = s3_resource.Bucket(CHECK_BUCKET_NAME).objects.filter(Prefix=object_prefix) + object_list = s3_client.list_objects_v2(Bucket=CHECK_BUCKET_NAME, Prefix=object_prefix) file_list = [] - for obj in object_list: - obj_key = obj.key.rsplit('/', INDEX_SPLIT_NUM) + for obj in object_list.get('Contents', []): + obj_key = obj['Key'].rsplit('/', INDEX_SPLIT_NUM) file_list.append(obj_key[INDEX_LAST]) # 2.日次I/Fファイルチェック処理 @@ -126,7 +123,7 @@ def lambda_handler(event, context): logger.info('I-04-04 取得したオブジェクトリストと日次I/Fファイルネーム設定ファイルの突き合わせを開始します') receive_daily_file_name_body = io.TextIOWrapper(io.BytesIO(receive_daily_file_name_response["Body"].read()), encoding='utf-8') match_count = 0 - row_count = sum(1 for line in io.BytesIO(receive_daily_file_name_obj.get()["Body"].read())) + row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH)["Body"].read())) for row in csv.reader(receive_daily_file_name_body, delimiter='\t'): file_exists = False for file_name in file_list: From 29f279702dd509e4eefcf6f986f557aee6721d7d Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Tue, 14 May 2024 17:21:43 +0900 Subject: [PATCH 13/18] sap-sup-monthly-data-fix --- .../sap-sup-monthly-data-notice-daily.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lambda/sap-sup-monthly-data-notice-daily/sap-sup-monthly-data-notice-daily.py b/lambda/sap-sup-monthly-data-notice-daily/sap-sup-monthly-data-notice-daily.py index d9c9ef24..b568544a 100644 --- a/lambda/sap-sup-monthly-data-notice-daily/sap-sup-monthly-data-notice-daily.py +++ b/lambda/sap-sup-monthly-data-notice-daily/sap-sup-monthly-data-notice-daily.py @@ -28,7 +28,6 @@ MAIL_INDENT = '　　' # 変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') sns_client = boto3.client('sns') # logger設定 @@ -69,8 +68,7 @@ def lambda_handler(event, context): # ③ 設定ファイル[SAP_supI/Fファイルネーム設定ファイル（月次）]を読み込む try: logger.info(f'I-03-01 月次I/Fファイルネーム設定ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{RECEIVE_MONTHLY_FILE_NAME_LIST_PATH}') - receive_monthly_file_name_obj = s3_resource.Object(CONFIG_BUCKET_NAME, RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) - receive_monthly_file_name_response = receive_monthly_file_name_obj.get() + receive_monthly_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) logger.info('I-03-02 月次I/Fファイルネーム設定ファイルを読み込みました') except Exception as e: logger.error(f'E-03-01 月次I/Fファイルネーム設定ファイルの読み込みに失敗しました　エラー内容：{e}') From 98da72acb054faf636cb423f14afe36689359a11 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Tue, 14 May 2024 17:35:08 +0900 Subject: [PATCH 14/18] sap-sup-receive-check-monthly-fix --- .../sap-sup-receive-check-monthly.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lambda/sap-sup-receive-check-monthly/sap-sup-receive-check-monthly.py b/lambda/sap-sup-receive-check-monthly/sap-sup-receive-check-monthly.py index 07f391d6..10ffb8f0 100644 --- a/lambda/sap-sup-receive-check-monthly/sap-sup-receive-check-monthly.py +++ b/lambda/sap-sup-receive-check-monthly/sap-sup-receive-check-monthly.py @@ -71,8 +71,7 @@ def lambda_handler(event, context): # 1.設定ファイル[チェック処理実施指定日ファイル]を読み込む try: logger.info(f'I-02-02 チェック処理実施指定日ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{MONTHLY_CEHCK_DAY_LIST_PATH}') - monthly_day_obj = s3_resource.Object(CONFIG_BUCKET_NAME, MONTHLY_CEHCK_DAY_LIST_PATH) - monthly_day_response = monthly_day_obj.get() + monthly_day_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=MONTHLY_CEHCK_DAY_LIST_PATH) logger.info('I-02-03 チェック処理実施指定日ファイルを読み込みました') except Exception as e: logger.error(f'E-02-01 チェック処理実施指定日設定ファイルの読み込みに失敗しました　エラー内容：{e}') @@ -104,8 +103,7 @@ def lambda_handler(event, context): # ③ 設定ファイル[SAP_supI/Fファイルネーム設定ファイル（月次）]を読み込む try: logger.info(f'I-03-01 月次I/Fファイルネーム設定ファイル読込　読込元：{CONFIG_BUCKET_NAME}/{RECEIVE_MONTHLY_FILE_NAME_LIST_PATH}') - receive_monthly_file_name_obj = s3_resource.Object(CONFIG_BUCKET_NAME, RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) - receive_monthly_file_name_response = receive_monthly_file_name_obj.get() + receive_monthly_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) logger.info('I-03-02 月次I/Fファイルネーム設定ファイルを読み込みました') except Exception as e: logger.error(f'E-03-01 月次I/Fファイルネーム設定ファイルの読み込みに失敗しました　エラー内容：{e}') @@ -117,10 +115,10 @@ def lambda_handler(event, context): # 1.SAP保管用バケットの処理稼働月に該当するサブフォルダにあるファイル一覧を取得する logger.info(f'I-04-02 オブジェクトリストの取得　取得先：{CHECK_BUCKET_NAME}/{execute_month}/') object_prefix = f'{execute_month}/' - object_list = s3_resource.Bucket(CHECK_BUCKET_NAME).objects.filter(Prefix=object_prefix) + object_list = s3_client.list_objects_v2(Bucket=CHECK_BUCKET_NAME, Prefix=object_prefix) file_list = [] - for obj in object_list: - obj_key = obj.key.rsplit('/', INDEX_SPLIT_NUM) + for obj in object_list.get('Contents', []): + obj_key = obj['Key'].rsplit('/', INDEX_SPLIT_NUM) file_list.append(obj_key[INDEX_LAST]) # 2.月次I/Fファイルチェック処理 @@ -128,7 +126,7 @@ def lambda_handler(event, context): logger.info('I-04-04 取得したオブジェクトリストと月次I/Fファイルネーム設定ファイルの突き合わせを開始します') receive_monthly_file_name_body = io.TextIOWrapper(io.BytesIO(receive_monthly_file_name_response["Body"].read()), encoding='utf-8') match_count = 0 - row_count = sum(1 for line in io.BytesIO(receive_monthly_file_name_obj.get()["Body"].read())) + row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH)["Body"].read())) for row in csv.reader(receive_monthly_file_name_body, delimiter='\t'): file_exists = False for file_name in file_list: From 29780dab940565917ea6e932c8991cb40b4c7ae2 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Tue, 14 May 2024 17:41:54 +0900 Subject: [PATCH 15/18] check-view-option-fix --- .../check-view-security-option/check-view-option/aws/s3.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lambda/check-view-security-option/check-view-option/aws/s3.py b/lambda/check-view-security-option/check-view-option/aws/s3.py index 55c33c81..26fbe321 100644 --- a/lambda/check-view-security-option/check-view-option/aws/s3.py +++ b/lambda/check-view-security-option/check-view-option/aws/s3.py @@ -6,12 +6,11 @@ from constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY, UTF8 class S3Resource: def __init__(self, bucket_name: str) -> None: - self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) - self.__s3_bucket = self.__s3_resource.Bucket(bucket_name) + self.__s3_client = boto3.client(AWS_RESOURCE_S3) + self.__s3_bucket = bucket_name def get_object(self, object_key: str): - s3_object = self.__s3_bucket.Object(object_key) - response = s3_object.get() + response = self.__s3_client.get_object(Bucket=self.__s3_bucket, Key=object_key) return response[S3_RESPONSE_BODY].read().decode(UTF8) From e55e95a6e914a5004173efa690260bd2fcc44242 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Wed, 15 May 2024 10:13:40 +0900 Subject: [PATCH 16/18] sap-fin-receive-check-daily-fix --- .../sap-fin-receive-check-daily/sap-fin-receive-check-daily.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py b/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py index 0899fb3c..dcad4886 100644 --- a/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py +++ b/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py @@ -123,7 +123,7 @@ def lambda_handler(event, context): logger.info('I-04-04 取得したオブジェクトリストと日次I/Fファイルネーム設定ファイルの突き合わせを開始します') receive_daily_file_name_body = io.TextIOWrapper(io.BytesIO(receive_daily_file_name_response["Body"].read()), encoding='utf-8') match_count = 0 - row_count = sum(1 for line in io.BytesIO(receive_daily_file_name_obj.get()["Body"].read())) + row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH)["Body"].read())) for row in csv.reader(receive_daily_file_name_body, delimiter='\t'): file_exists = False for file_name in file_list: From de1e82273ef81e441bfac9362cc3e52267caef56 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Wed, 15 May 2024 11:19:00 +0900 Subject: [PATCH 17/18] =?UTF-8?q?get=E3=81=A8read=E5=87=A6=E7=90=86?= =?UTF-8?q?=E5=88=86=E3=81=91=E3=81=9F=E3=83=BB=E3=82=A4=E3=83=B3=E3=82=BF?= =?UTF-8?q?=E3=83=BC=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E5=91=BD=E5=90=8D?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/aws/s3.py | 30 +++++++++---------- .../sap-fin-receive-check-daily.py | 3 +- .../sap-fin-receive-check-monthly.py | 3 +- .../sap-sup-receive-check-daily.py | 3 +- .../sap-sup-receive-check-monthly.py | 3 +- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index ed062fca..2501ce94 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -12,7 +12,7 @@ from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, RESPONSE_JSON_BACKUP_FOLDER) -class S3Resource: +class S3Client: def __init__(self, bucket_name: str) -> None: self.__s3_client = boto3.client(AWS_RESOURCE_S3) self.__s3_bucket = bucket_name @@ -33,66 +33,66 @@ class S3Resource: class ConfigBucket: - __s3_resource: S3Resource = None + __s3_client: S3Client = None def __init__(self) -> None: - self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET) + self.__s3_client = S3Client(CRM_CONFIG_BUCKET) def __str__(self) -> str: return CRM_CONFIG_BUCKET def get_object_info_file(self) -> str: - return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') + return self.__s3_client.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') def get_last_fetch_datetime_file(self, file_key: str) -> str: - return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_key}') + return self.__s3_client.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_key}') def put_last_fetch_datetime_file(self, file_key: str, local_file_path: str) -> None: - self.__s3_resource.put_object( + self.__s3_client.put_object( f'{LAST_FETCH_DATE_FOLDER}/{file_key}', local_file_path) return class DataBucket: - __s3_resource: S3Resource = None + __s3_client: S3Client = None def __init__(self) -> None: - self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET) + self.__s3_client = S3Client(IMPORT_DATA_BUCKET) def __str__(self) -> str: return IMPORT_DATA_BUCKET def put_csv(self, file_key: str, local_file_path: str) -> None: object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_key}' - self.__s3_resource.put_object(object_key, local_file_path) + self.__s3_client.put_object(object_key, local_file_path) return def put_csv_from(self, src_bucket: str, src_key: str): dest_filename = src_key.split('/')[-1] - self.__s3_resource.copy(src_bucket, src_key, str(self), f'{CRM_IMPORT_DATA_FOLDER}/{dest_filename}') + self.__s3_client.copy(src_bucket, src_key, str(self), f'{CRM_IMPORT_DATA_FOLDER}/{dest_filename}') return class BackupBucket: - __s3_resource: S3Resource = None + __s3_client: S3Client = None def __init__(self) -> None: - self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET) + self.__s3_client = S3Client(CRM_BACKUP_BUCKET) def __str__(self) -> str: return CRM_BACKUP_BUCKET def put_response_json(self, file_key: str, local_file_path: str) -> None: object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_key}' - self.__s3_resource.put_object(object_key, local_file_path) + self.__s3_client.put_object(object_key, local_file_path) return def put_csv(self, file_key: str, local_file_path: str) -> None: object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_key}' - self.__s3_resource.put_object(object_key, local_file_path) + self.__s3_client.put_object(object_key, local_file_path) return def put_result_json(self, file_key: str, local_file_path: str) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_key}' - self.__s3_resource.put_object(object_key, local_file_path) + self.__s3_client.put_object(object_key, local_file_path) return diff --git a/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py b/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py index dcad4886..08c82708 100644 --- a/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py +++ b/lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py @@ -123,7 +123,8 @@ def lambda_handler(event, context): logger.info('I-04-04 取得したオブジェクトリストと日次I/Fファイルネーム設定ファイルの突き合わせを開始します') receive_daily_file_name_body = io.TextIOWrapper(io.BytesIO(receive_daily_file_name_response["Body"].read()), encoding='utf-8') match_count = 0 - row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH)["Body"].read())) + receive_daily_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH) + row_count = sum(1 for line in io.BytesIO(receive_daily_file_name_response["Body"].read())) for row in csv.reader(receive_daily_file_name_body, delimiter='\t'): file_exists = False for file_name in file_list: diff --git a/lambda/sap-fin-receive-check-monthly/sap-fin-receive-check-monthly.py b/lambda/sap-fin-receive-check-monthly/sap-fin-receive-check-monthly.py index 25058295..d29f8654 100644 --- a/lambda/sap-fin-receive-check-monthly/sap-fin-receive-check-monthly.py +++ b/lambda/sap-fin-receive-check-monthly/sap-fin-receive-check-monthly.py @@ -127,7 +127,8 @@ def lambda_handler(event, context): logger.info('I-04-04 取得したオブジェクトリストと月次I/Fファイルネーム設定ファイルの突き合わせを開始します') receive_monthly_file_name_body = io.TextIOWrapper(io.BytesIO(receive_monthly_file_name_response["Body"].read()), encoding='utf-8') match_count = 0 - row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH)["Body"].read())) + receive_monthly_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) + row_count = sum(1 for line in io.BytesIO(receive_monthly_file_name_response["Body"].read())) for row in csv.reader(receive_monthly_file_name_body, delimiter='\t'): file_exists = False for file_name in file_list: diff --git a/lambda/sap-sup-receive-check-daily/sap-sup-receive-check-daily.py b/lambda/sap-sup-receive-check-daily/sap-sup-receive-check-daily.py index 033ae53b..f4cef00f 100644 --- a/lambda/sap-sup-receive-check-daily/sap-sup-receive-check-daily.py +++ b/lambda/sap-sup-receive-check-daily/sap-sup-receive-check-daily.py @@ -123,7 +123,8 @@ def lambda_handler(event, context): logger.info('I-04-04 取得したオブジェクトリストと日次I/Fファイルネーム設定ファイルの突き合わせを開始します') receive_daily_file_name_body = io.TextIOWrapper(io.BytesIO(receive_daily_file_name_response["Body"].read()), encoding='utf-8') match_count = 0 - row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH)["Body"].read())) + receive_daily_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH) + row_count = sum(1 for line in io.BytesIO(receive_daily_file_name_response["Body"].read())) for row in csv.reader(receive_daily_file_name_body, delimiter='\t'): file_exists = False for file_name in file_list: diff --git a/lambda/sap-sup-receive-check-monthly/sap-sup-receive-check-monthly.py b/lambda/sap-sup-receive-check-monthly/sap-sup-receive-check-monthly.py index 10ffb8f0..9cc9b42b 100644 --- a/lambda/sap-sup-receive-check-monthly/sap-sup-receive-check-monthly.py +++ b/lambda/sap-sup-receive-check-monthly/sap-sup-receive-check-monthly.py @@ -126,7 +126,8 @@ def lambda_handler(event, context): logger.info('I-04-04 取得したオブジェクトリストと月次I/Fファイルネーム設定ファイルの突き合わせを開始します') receive_monthly_file_name_body = io.TextIOWrapper(io.BytesIO(receive_monthly_file_name_response["Body"].read()), encoding='utf-8') match_count = 0 - row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH)["Body"].read())) + receive_monthly_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH) + row_count = sum(1 for line in io.BytesIO(receive_monthly_file_name_response["Body"].read())) for row in csv.reader(receive_monthly_file_name_body, delimiter='\t'): file_exists = False for file_name in file_list: From ea5b8a930d50ff91866bd5e0825c901a8899b69d Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Tue, 21 May 2024 16:09:11 +0900 Subject: [PATCH 18/18] =?UTF-8?q?=E3=82=AF=E3=83=A9=E3=82=B9=E9=96=A2?= =?UTF-8?q?=E6=95=B0=E3=81=AE=E5=A4=89=E6=95=B0=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/aws/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index 2501ce94..580be5c2 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -23,7 +23,7 @@ class S3Client: return body.decode(S3_CHAR_CODE) def put_object(self, object_key: str, local_file_path: str) -> None: - self.__s3_client.upload_file(Filename=local_file_path, Bucket=self.__bucket_name, Key=object_key) + self.__s3_client.upload_file(Filename=local_file_path, Bucket=self.__s3_bucket, Key=object_key) return def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: