From 2b57687697154ff8e368cf98b69d24f3f50e227f Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 14 Apr 2023 11:53:57 +0900 Subject: [PATCH 01/32] =?UTF-8?q?feat:=20=E6=97=A5=E6=AC=A1=E3=83=90?= =?UTF-8?q?=E3=83=83=E3=83=81=E3=81=AE=E5=88=B6=E5=BE=A1=E3=80=81=E6=97=A5?= =?UTF-8?q?=E4=BB=98=E6=9B=B4=E6=96=B0=E9=96=A2=E9=80=A3=E3=81=AE=E5=87=A6?= =?UTF-8?q?=E7=90=86=E8=BF=BD=E5=8A=A0=E3=80=82=E3=82=84=E3=82=89=E3=81=AA?= =?UTF-8?q?=E3=81=84=E5=87=A6=E7=90=86=E3=82=92=E9=96=93=E5=BC=95=E3=81=84?= =?UTF-8?q?=E3=81=9F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/batch_functions.py | 84 +++++++--- .../src/batch/ultmarc/ultmarc_process.py | 3 +- .../src/error/exceptions.py | 7 - ecs/jskult-batch-daily/src/jobctrl_daily.py | 145 ++++++++++-------- .../src/system_var/constants.py | 8 + 5 files changed, 151 insertions(+), 96 deletions(-) create mode 100644 ecs/jskult-batch-daily/src/system_var/constants.py diff --git a/ecs/jskult-batch-daily/src/batch/batch_functions.py b/ecs/jskult-batch-daily/src/batch/batch_functions.py index 0e8a78fa..be2bd565 100644 --- a/ecs/jskult-batch-daily/src/batch/batch_functions.py +++ b/ecs/jskult-batch-daily/src/batch/batch_functions.py @@ -5,48 +5,90 @@ from datetime import datetime from src.db.database import Database from src.error.exceptions import BatchOperationException, DBException +from src.system_var import constants -def get_syor_date() -> str: - """DBから処理日を取得します +def get_batch_statuses() -> tuple[str, str, str]: + """日付テーブルから、以下を取得して返す。 + - 日次バッチ処理中フラグ + - dump取得状況区分 + - 処理日（YYYY/MM/DD） Raises: BatchOperationException: 日付テーブルが取得できないとき、何らかのエラーが発生したとき Returns: - str: hdke_tbl.syor_date + tuple[str, str]: [0]日次バッチ処理中フラグ、dump取得状況区分 """ db = Database.get_instance() - db.connect() - sql = 'SELECT syor_date FROM src05.hdke_tbl' + sql = 'SELECT bch_actf, dump_sts_kbn, src05.get_syor_date() AS syor_date FROM src05.hdke_tbl' try: - syor_date_result = db.execute_select(sql) + db.connect() + hdke_tbl_result = db.execute_select(sql) except DBException as e: raise BatchOperationException(e) db.disconnect() - if len(syor_date_result) == 0: + + if len(hdke_tbl_result) == 0: raise BatchOperationException('日付テーブルが取得できませんでした') - # 必ず一件取れる - syor_date_record = syor_date_result[0] - syor_date_str = syor_date_record['syor_date'] - return syor_date_str + + # 必ず1件取れる + hdke_tbl_record = hdke_tbl_result[0] + batch_processing_flag = hdke_tbl_record['bch_actf'] + dump_status_kbn = hdke_tbl_record['dump_sts_kbn'] + syor_date = hdke_tbl_record['dump_sts_kbn'] + # 処理日を文字列に変換する + syor_date_str = datetime.strftime(syor_date, '%Y/%m/%d') + + return batch_processing_flag, dump_status_kbn, syor_date_str -def get_syor_date_as_date_format() -> str: - """DBから処理日を取得し、yyyy/mm/ddのフォーマットにして返します +def update_batch_processing_flag_in_processing() -> None: + """バッチ処理中フラグを処理中に更新する Raises: - BatchOperationException: 日付テーブルが取得できないとき、何らかのエラーが発生したとき - - Returns: - str: hdke_tbl.syor_dateをyyyy/mm/ddにフォーマットした文字列 + BatchOperationException: DB操作の何らかのエラー """ - syor_date_str = get_syor_date() - syor_date = datetime.strptime(syor_date_str, '%Y%m%d') - return syor_date.strftime('%Y/%m/%d') + db = Database.get_instance() + sql = 'UPDATE src05.hdke_tbl SET bch_actf = :in_processing' + try: + db.connect() + db.execute(sql, {'in_processing': constants.BATCH_ACTF_BATCH_IN_PROCESSING}) + except DBException as e: + raise BatchOperationException(e) + db.disconnect() + + return -def logging_sql(logger, sql): +def update_batch_process_complete() -> None: + """バッチ処理を完了とし、処理日、バッチ処理中フラグ、dump処理状態区分を更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = """\ + UPDATE src05.hdke_tbl + SET + bch_actf = :batch_complete, + dump_sts_kbn = :dump_unprocessed, + syor_date = DATE_FORMAT((src05.get_syor_date() + interval 1 day), '%Y%m%d) -- +1日 + """ + try: + db.connect() + db.execute(sql, { + 'batch_complete': constants.BATCH_ACTF_BATCH_UNPROCESSED, + 'dump_unprocessed': constants.DUMP_STATUS_KBN_UNPROCESSED + }) + except DBException as e: + raise BatchOperationException(e) + db.disconnect() + + return + + +def logging_sql(logger, sql) -> None: """SQL文をデバッグログで出力する Args: diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py index 42bc6af6..21c0c302 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py @@ -6,6 +6,7 @@ from src.batch.ultmarc.datfile import DatFile from src.batch.ultmarc.utmp_tables.ultmarc_table_mapper_factory import \ UltmarcTableMapperFactory from src.db.database import Database +from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger logger = get_logger('アルトマークデータ保管') @@ -52,7 +53,7 @@ def exec_import(): ultmarc_bucket.delete_dat_file(dat_file_name) except Exception as e: logger.exception(e) - raise e + raise BatchOperationException(e) finally: logger.info('アルトマーク取込処理: 終了') diff --git a/ecs/jskult-batch-daily/src/error/exceptions.py b/ecs/jskult-batch-daily/src/error/exceptions.py index c9effa01..055c24f6 100644 --- a/ecs/jskult-batch-daily/src/error/exceptions.py +++ b/ecs/jskult-batch-daily/src/error/exceptions.py @@ -1,6 +1,3 @@ -from tenacity import RetryError - - class MeDaCaException(Exception): pass @@ -11,7 +8,3 @@ class DBException(MeDaCaException): class BatchOperationException(MeDaCaException): pass - - -class MaxRetryExceededException(MeDaCaException, RetryError): - pass diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index f91437b5..3cd862c8 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -1,11 +1,14 @@ from src.batch import jissekiaraigae -from src.batch.batch_functions import get_syor_date_as_date_format +from src.batch.batch_functions import ( + get_batch_statuses, update_batch_process_complete, + update_batch_processing_flag_in_processing) from src.batch.common.batch_config import BatchConfig from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger +from src.system_var import constants -logger = get_logger('日次処理コントロール') # ここを処理IDとかにするといいかもしれない +logger = get_logger('日次処理コントロール') # バッチ共通設定を取得 batch_config = BatchConfig.get_instance() @@ -14,62 +17,71 @@ batch_config = BatchConfig.get_instance() def batch_process(): try: logger.info('日次ジョブ：開始') - # logger.info('S3マウント状況確認') - # logger.error('S3マウントエラー:DWH（異常終了）') - # logger.error('S3マウントエラー:BIO（異常終了）') - # logger.info('データベース接続') # 実際には、ここでつなげているわけではないので、いらないと思う - # logger.error('データベース接続エラー（異常終了）') # 検査例外を捕まえて、共通的に出せばいいと思う try: - logger.info('処理日取得') - syor_date = get_syor_date_as_date_format() + # 日次バッチ処置中フラグ、dump処理状態区分、処理日を取得 + batch_processing_flag, dump_status_kbn, syor_date = get_batch_statuses() except BatchOperationException as e: - logger.error(f'処理日取得エラー（異常終了）{e}') + logger.exception(f'日付テーブル取得（異常終了）{e}') + raise e + + # 日次バッチ処理中の場合、後続の処理は行わない + if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: + logger.error('日次バッチ処理中のため、日次バッチ処理を終了します。') + return 0 + + # dump取得が正常終了していない場合、後続の処理は行わない + if dump_status_kbn != constants.DUMP_STATUS_KBN_COMPLETE: + logger.error('dump取得が正常終了していないため、日次バッチ処理を終了します。') + return 0 # エラーで返せば、StepFunctionsでリトライしてくれるので、どう返すかは要検討 + logger.info(f'処理日＝{syor_date}') # バッチ共通設定に処理日を追加 batch_config.syor_date = syor_date - # 休日判定ファイルを読み込み + + # バッチ処理中に更新 + try: + update_batch_processing_flag_in_processing() + except BatchOperationException as e: + logger.exception(f'処理フラグ更新(未処理→処理中) エラー（異常終了）{e}') + + # 休日判定ファイルを読み込み(ここは、各処理内に押し込むつもり) logger.info('休日判定処理') if True: # 休日判定 logger.info('非営業日かつ月、火、水以外です。') # 分岐 - try: - # 処理中フラグ判定。ここでdumpのフラグも見る - logger.info('処理フラグ更新中') - logger.info('処理フラグ更新終了') - except BatchOperationException as e: - logger.error(f'処理フラグ更新処理エラー（異常終了）{e}') + # try: + # # 処理中フラグ判定。ここでdumpのフラグも見る + # logger.info('処理フラグ更新中') + # logger.info('処理フラグ更新終了') + # except BatchOperationException as e: + # logger.error(f'処理フラグ更新処理エラー（異常終了）{e}') logger.info('日次ジョブ：終了（正常終了）') - try: - logger.info('日次ジョブ処理中判定') - if True: # 処理中判定 - logger.error('処理フラグ処理中（異常終了）') - logger.info('処理中フラグの更新：起動') - logger.info('処理中フラグの更新：終了') - except BatchOperationException as e: - logger.error(f'日次ジョブ処理中エラー（異常終了）{e}') - # ↓ここから、やらない↓ - # logger.info('処理前バックアップ実行') - # logger.info('処理前バックアップ：起動') - # logger.info('処理前バックアップ：終了') - # logger.error('処理前バックアップ処理エラー（異常終了）', $ex->getMessage()) - # ↑ここまで↑ - logger.info('卸在庫データ取込判定') - if True: # 卸在庫日判定 - logger.info('卸在庫データ取込日です') - logger.debug('卸在庫データファイル名： {_PATH_OROSHI_ZAIKO}') - if True: # 卸在庫ファイル存在確認なければ異常終了 - logger.error('卸在庫データ存在確認エラー（異常終了）') - logger.info('卸在庫データ存在確認：取込処理開始') - logger.debug('卸在庫データファイル名作成： {read_filename}') - logger.debug('ファイル移動OK：{_MOVE_OROSHI_ZAIKO}') # S3からダウンロード - logger.debug('ファイル解凍OK：{sprintf(_ZIP_OROSHI_ZAIKO, $read_filename)}') # gunzip -fなので、gzipを使う - logger.debug('ファイル名変更OK： {sprintf(_RENAME_OROSHI_ZAIKO, $read_filename)}') # S3にアップロード - try: - logger.info('卸在庫データ取込：起動') - logger.info('卸在庫データ取込：終了') - except BatchOperationException as e: - logger.error(f'卸在庫データ取込処理エラー（異常終了）{e}') + # try: + # logger.info('日次ジョブ処理中判定') + # if True: # 処理中判定 + # logger.error('処理フラグ処理中（異常終了）') + # logger.info('処理中フラグの更新：起動') + # logger.info('処理中フラグの更新：終了') + # except BatchOperationException as e: + # logger.error(f'日次ジョブ処理中エラー（異常終了）{e}') + # TODO: 以下、卸在庫取り込み処理は、実消化取り込み内で行う + # logger.info('卸在庫データ取込判定') + # if True: # 卸在庫日判定 + # logger.info('卸在庫データ取込日です') + # logger.debug('卸在庫データファイル名： {_PATH_OROSHI_ZAIKO}') + # if True: # 卸在庫ファイル存在確認なければ異常終了 + # logger.error('卸在庫データ存在確認エラー（異常終了）') + # logger.info('卸在庫データ存在確認：取込処理開始') + # logger.debug('卸在庫データファイル名作成： {read_filename}') + # logger.debug('ファイル移動OK：{_MOVE_OROSHI_ZAIKO}') # S3からダウンロード + # logger.debug('ファイル解凍OK：{sprintf(_ZIP_OROSHI_ZAIKO, $read_filename)}') # gunzip -fなので、gzipを使う + # logger.debug('ファイル名変更OK： {sprintf(_RENAME_OROSHI_ZAIKO, $read_filename)}') # S3にアップロード + # try: + # logger.info('卸在庫データ取込：起動') + # logger.info('卸在庫データ取込：終了') + # except BatchOperationException as e: + # logger.error(f'卸在庫データ取込処理エラー（異常終了）{e}') logger.info('日次処理（アルトマーク）') - if True: # アルトマークなければ + if True: # アルトマーク取り込み処理内で実装 logger.info('日次処理（アルトマーク）実行対象日でない為未実行') try: logger.info('アルトマーク取込：起動') @@ -77,7 +89,7 @@ def batch_process(): logger.info('アルトマーク取込：終了') except BatchOperationException as e: logger.error(f'アルトマーク取込処理エラー（異常終了）{e}') - if True: # 休日判定 + if True: # 休日判定、メルク施設マスタ作成内で行いたい try: logger.info('メルク施設マスタ作成') logger.info('メルク施設マスタ作成終了') @@ -88,10 +100,11 @@ def batch_process(): logger.info('DCF施設統合マスタ作成終了') except BatchOperationException as e: logger.error(f'DCF施設統合マスタ作成エラー（異常終了）{e}') - # if False: # ($holiday === FALSE) とにかく毎日動かす - logger.info('V実消化連携データ存在確認') - if True: - logger.error('V実消化連携データ存在確認（異常終了）') + # if False: # ($holiday === FALSE) # DCF施設統合マスタを作成する日は必ず休日なので、V実消化の取り込みは行わない。 + # TODO: データ存在確認は実消化取り込み内で行う + # logger.info('V実消化連携データ存在確認') + # if True: + # logger.error('V実消化連携データ存在確認（異常終了）') logger.info('日次処理（V実消化）') try: @@ -99,28 +112,26 @@ def batch_process(): logger.info('V実消化取込：終了') except BatchOperationException as e: logger.exception(f'V実消化取込処理エラー（異常終了）{e}') + logger.info('日次処理（実績更新）') + # ここも、休日判定を内側で行う try: + # ここで、生物由来ロット分解と並行処理 logger.info('実績更新：起動') jissekiaraigae.batch_process() + logger.info('生物由来ロット分解：起動') logger.info('実績更新：終了') + logger.info('生物由来ロット分解：終了') except BatchOperationException as e: logger.exception(f'実績更新処理エラー（異常終了）{e}') - # ↓以下、ファイルのバックアップ以外はやらない↓ - # logger.info('処理後バックアップ実行') - # logger.info('処理後バックアップ：起動') - # logger.info('処理後バックアップ：終了') - # logger.error('処理後バックアップ処理エラー（異常終了）', $ex->getMessage()) - # ↑ここまでやらない↑ - logger.info('処理中フラグの更新：非処理中') - try: - logger.info('処理中フラグの更新：起動') - logger.info('処理中フラグの更新：終了') - except BatchOperationException as e: - logger.exception(f'処理中フラグ更新エラー（異常終了）{e}') - logger.info('ワークディレクトリクリーニング') - logger.info('日次ジョブ：終了（正常終了）') + # バッチ処理完了とし、処理日、バッチ処置中フラグ、dump取得状態区分を更新 + try: + update_batch_process_complete() + except BatchOperationException as e: + logger.exception(f'日次バッチ完了処理 エラー（異常終了）{e}') + + logger.info('日次ジョブ：終了（正常終了）') return 0 except Exception as e: raise e diff --git a/ecs/jskult-batch-daily/src/system_var/constants.py b/ecs/jskult-batch-daily/src/system_var/constants.py new file mode 100644 index 00000000..5eafaedd --- /dev/null +++ b/ecs/jskult-batch-daily/src/system_var/constants.py @@ -0,0 +1,8 @@ +# バッチ処理中フラグ：未処理 +BATCH_ACTF_BATCH_UNPROCESSED = '0' +# バッチ処理中フラグ：処理中 +BATCH_ACTF_BATCH_IN_PROCESSING = '1' +# dump取得状態区分：未処理 +DUMP_STATUS_KBN_UNPROCESSED = '0' +# dump取得状態区分：dump取得正常終了 +DUMP_STATUS_KBN_COMPLETE = '2' From d00bf2699eb02d33ecbbc721c3f3ca221cf34aaa Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 14 Apr 2023 13:08:49 +0900 Subject: [PATCH 02/32] =?UTF-8?q?fix:=20=E3=83=90=E3=82=B0=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/batch_functions.py | 4 ++-- ecs/jskult-batch-daily/src/jobctrl_daily.py | 16 ++++++++++++---- .../src/system_var/constants.py | 3 +++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/batch_functions.py b/ecs/jskult-batch-daily/src/batch/batch_functions.py index be2bd565..255d7729 100644 --- a/ecs/jskult-batch-daily/src/batch/batch_functions.py +++ b/ecs/jskult-batch-daily/src/batch/batch_functions.py @@ -36,7 +36,7 @@ def get_batch_statuses() -> tuple[str, str, str]: hdke_tbl_record = hdke_tbl_result[0] batch_processing_flag = hdke_tbl_record['bch_actf'] dump_status_kbn = hdke_tbl_record['dump_sts_kbn'] - syor_date = hdke_tbl_record['dump_sts_kbn'] + syor_date = hdke_tbl_record['syor_date'] # 処理日を文字列に変換する syor_date_str = datetime.strftime(syor_date, '%Y/%m/%d') @@ -73,7 +73,7 @@ def update_batch_process_complete() -> None: SET bch_actf = :batch_complete, dump_sts_kbn = :dump_unprocessed, - syor_date = DATE_FORMAT((src05.get_syor_date() + interval 1 day), '%Y%m%d) -- +1日 + syor_date = DATE_FORMAT((src05.get_syor_date() + interval 1 day), '%Y%m%d') -- +1日 """ try: db.connect() diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 3cd862c8..385586f4 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -22,17 +22,17 @@ def batch_process(): batch_processing_flag, dump_status_kbn, syor_date = get_batch_statuses() except BatchOperationException as e: logger.exception(f'日付テーブル取得（異常終了）{e}') - raise e + return constants.BATCH_EXIT_CODE_SUCCESS # 日次バッチ処理中の場合、後続の処理は行わない if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: logger.error('日次バッチ処理中のため、日次バッチ処理を終了します。') - return 0 + return constants.BATCH_EXIT_CODE_SUCCESS # dump取得が正常終了していない場合、後続の処理は行わない if dump_status_kbn != constants.DUMP_STATUS_KBN_COMPLETE: logger.error('dump取得が正常終了していないため、日次バッチ処理を終了します。') - return 0 # エラーで返せば、StepFunctionsでリトライしてくれるので、どう返すかは要検討 + return constants.BATCH_EXIT_CODE_SUCCESS # TODO: エラーで返せば、StepFunctionsでリトライしてくれるので、どう返すかは要検討 logger.info(f'処理日＝{syor_date}') # バッチ共通設定に処理日を追加 @@ -43,6 +43,7 @@ def batch_process(): update_batch_processing_flag_in_processing() except BatchOperationException as e: logger.exception(f'処理フラグ更新(未処理→処理中) エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS # 休日判定ファイルを読み込み(ここは、各処理内に押し込むつもり) logger.info('休日判定処理') @@ -89,17 +90,20 @@ def batch_process(): logger.info('アルトマーク取込：終了') except BatchOperationException as e: logger.error(f'アルトマーク取込処理エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS if True: # 休日判定、メルク施設マスタ作成内で行いたい try: logger.info('メルク施設マスタ作成') logger.info('メルク施設マスタ作成終了') except BatchOperationException as e: logger.error(f'メルク施設マスタ作成エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS try: logger.info('DCF施設統合マスタ作成') logger.info('DCF施設統合マスタ作成終了') except BatchOperationException as e: logger.error(f'DCF施設統合マスタ作成エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS # if False: # ($holiday === FALSE) # DCF施設統合マスタを作成する日は必ず休日なので、V実消化の取り込みは行わない。 # TODO: データ存在確認は実消化取り込み内で行う # logger.info('V実消化連携データ存在確認') @@ -112,6 +116,7 @@ def batch_process(): logger.info('V実消化取込：終了') except BatchOperationException as e: logger.exception(f'V実消化取込処理エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS logger.info('日次処理（実績更新）') # ここも、休日判定を内側で行う @@ -124,14 +129,17 @@ def batch_process(): logger.info('生物由来ロット分解：終了') except BatchOperationException as e: logger.exception(f'実績更新処理エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS # バッチ処理完了とし、処理日、バッチ処置中フラグ、dump取得状態区分を更新 try: update_batch_process_complete() except BatchOperationException as e: logger.exception(f'日次バッチ完了処理 エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS logger.info('日次ジョブ：終了（正常終了）') - return 0 + return constants.BATCH_EXIT_CODE_SUCCESS except Exception as e: + logger.exception(f'日次バッチ処理中に想定外のエラーが発生しました {e}') raise e diff --git a/ecs/jskult-batch-daily/src/system_var/constants.py b/ecs/jskult-batch-daily/src/system_var/constants.py index 5eafaedd..d5538a80 100644 --- a/ecs/jskult-batch-daily/src/system_var/constants.py +++ b/ecs/jskult-batch-daily/src/system_var/constants.py @@ -1,3 +1,6 @@ +# バッチ正常終了コード +BATCH_EXIT_CODE_SUCCESS = 0 + # バッチ処理中フラグ：未処理 BATCH_ACTF_BATCH_UNPROCESSED = '0' # バッチ処理中フラグ：処理中 From 7ce58fac4fc5978fec69b4f3650d6ae85369e739 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 14 Apr 2023 13:15:55 +0900 Subject: [PATCH 03/32] =?UTF-8?q?feat:=20=E3=83=95=E3=82=A9=E3=83=AB?= =?UTF-8?q?=E3=83=80=E6=A7=8B=E6=88=90=E3=80=81=E9=96=A2=E6=95=B0=E5=90=8D?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/entrypoint.py | 4 ++-- .../src/batch/{datachange => laundering}/__init__.py | 0 .../emp_chg_inst_laundering.py} | 2 +- .../{jissekiaraigae.py => laundering/sales_laundering.py} | 6 +++--- ecs/jskult-batch-daily/src/jobctrl_daily.py | 8 +++++--- 5 files changed, 11 insertions(+), 9 deletions(-) rename ecs/jskult-batch-daily/src/batch/{datachange => laundering}/__init__.py (100%) rename ecs/jskult-batch-daily/src/batch/{datachange/emp_chg_inst_lau.py => laundering/emp_chg_inst_laundering.py} (99%) rename ecs/jskult-batch-daily/src/batch/{jissekiaraigae.py => laundering/sales_laundering.py} (64%) diff --git a/ecs/jskult-batch-daily/entrypoint.py b/ecs/jskult-batch-daily/entrypoint.py index 43a997dc..472efd9f 100644 --- a/ecs/jskult-batch-daily/entrypoint.py +++ b/ecs/jskult-batch-daily/entrypoint.py @@ -1,9 +1,9 @@ """実消化&アルトマーク 日次バッチのエントリーポイント""" -from src.jobctrl_daily import batch_process +from src import jobctrl_daily if __name__ == '__main__': try: - exit(batch_process()) + exit(jobctrl_daily.exec()) except Exception: # エラーが起きても、正常系のコードで返す。 # エラーが起きた事実はbatch_process内でログを出す。 diff --git a/ecs/jskult-batch-daily/src/batch/datachange/__init__.py b/ecs/jskult-batch-daily/src/batch/laundering/__init__.py similarity index 100% rename from ecs/jskult-batch-daily/src/batch/datachange/__init__.py rename to ecs/jskult-batch-daily/src/batch/laundering/__init__.py diff --git a/ecs/jskult-batch-daily/src/batch/datachange/emp_chg_inst_lau.py b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py similarity index 99% rename from ecs/jskult-batch-daily/src/batch/datachange/emp_chg_inst_lau.py rename to ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py index 64a6a363..c8a28baf 100644 --- a/ecs/jskult-batch-daily/src/batch/datachange/emp_chg_inst_lau.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py @@ -9,7 +9,7 @@ logger = get_logger('48-施設担当者マスタ洗替') batch_config = BatchConfig.get_instance() -def batch_process(): +def exec(): db = Database.get_instance() db.connect() logger.info('##########################') diff --git a/ecs/jskult-batch-daily/src/batch/jissekiaraigae.py b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py similarity index 64% rename from ecs/jskult-batch-daily/src/batch/jissekiaraigae.py rename to ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py index 7f5d9fcf..c1884b7d 100644 --- a/ecs/jskult-batch-daily/src/batch/jissekiaraigae.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py @@ -1,11 +1,11 @@ -from src.batch.datachange import emp_chg_inst_lau +from src.batch.laundering import emp_chg_inst_laundering from src.logging.get_logger import get_logger logger = get_logger('実績洗替') -def batch_process(): +def exec(): """実績洗替処理""" logger.info('Start Jisseki Araigae Batch PGM.') # 施設担当者洗替 - emp_chg_inst_lau.batch_process() + emp_chg_inst_laundering.exec() diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 385586f4..c56fe8ba 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -1,8 +1,10 @@ -from src.batch import jissekiaraigae +"""実消化&アルトマーク 日次バッチ処理""" + from src.batch.batch_functions import ( get_batch_statuses, update_batch_process_complete, update_batch_processing_flag_in_processing) from src.batch.common.batch_config import BatchConfig +from src.batch.laundering import sales_laundering from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger @@ -14,7 +16,7 @@ logger = get_logger('日次処理コントロール') batch_config = BatchConfig.get_instance() -def batch_process(): +def exec(): try: logger.info('日次ジョブ：開始') try: @@ -123,7 +125,7 @@ def batch_process(): try: # ここで、生物由来ロット分解と並行処理 logger.info('実績更新：起動') - jissekiaraigae.batch_process() + sales_laundering.exec() logger.info('生物由来ロット分解：起動') logger.info('実績更新：終了') logger.info('生物由来ロット分解：終了') From e36285c6f2d1370c57c143238a37973ee46ba03d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 14 Apr 2023 13:18:43 +0900 Subject: [PATCH 04/32] =?UTF-8?q?style:=20=E5=9E=8B=E3=82=A2=E3=83=8E?= =?UTF-8?q?=E3=83=86=E3=83=BC=E3=82=B7=E3=83=A7=E3=83=B3=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/batch/batch_functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/batch_functions.py b/ecs/jskult-batch-daily/src/batch/batch_functions.py index 255d7729..6c113664 100644 --- a/ecs/jskult-batch-daily/src/batch/batch_functions.py +++ b/ecs/jskult-batch-daily/src/batch/batch_functions.py @@ -1,5 +1,5 @@ """バッチ処理の共通関数""" - +import logging import textwrap from datetime import datetime @@ -88,11 +88,11 @@ def update_batch_process_complete() -> None: return -def logging_sql(logger, sql) -> None: +def logging_sql(logger: logging.Logger, sql: str) -> None: """SQL文をデバッグログで出力する Args: logger (logging.Logger): ロガー sql (str): SQL文 """ - logger.debug(f'\n{"-"*15}\n{textwrap.dedent(sql)[1:-1]}\n{"-"*15}') + logger.debug(f'\n{"-" * 15}\n{textwrap.dedent(sql)[1:-1]}\n{"-" * 15}') From d3d385c214d8f2920f8433423f4a9579591fff3d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 14 Apr 2023 17:42:02 +0900 Subject: [PATCH 05/32] =?UTF-8?q?feat:=20=E4=BC=91=E6=97=A5=E5=88=A4?= =?UTF-8?q?=E5=AE=9A=E8=AB=B8=E3=80=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/.env.example | 3 + ecs/jskult-batch-daily/src/aws/s3.py | 14 +++ .../src/batch/common/batch_config.py | 10 -- .../src/batch/common/batch_context.py | 39 ++++++++ .../src/batch/common/calendar_file.py | 32 +++++++ .../laundering/emp_chg_inst_laundering.py | 6 +- .../src/batch/laundering/mst_inst_create.py | 3 + .../src/batch/ultmarc/ultmarc_process.py | 31 +++++-- .../table_mapper/ultmarc_table_mapper.py | 6 +- ecs/jskult-batch-daily/src/jobctrl_daily.py | 93 ++++++++++++++----- .../src/system_var/constants.py | 8 ++ .../src/system_var/environment.py | 3 + .../com_alma/test_com_alma_mapper.py | 12 +-- .../test_com_dr_wrkplace_mapper.py | 12 +-- .../com_hamtec/test_com_hamtec_mapper.py | 12 +-- 15 files changed, 218 insertions(+), 66 deletions(-) delete mode 100644 ecs/jskult-batch-daily/src/batch/common/batch_config.py create mode 100644 ecs/jskult-batch-daily/src/batch/common/batch_context.py create mode 100644 ecs/jskult-batch-daily/src/batch/common/calendar_file.py create mode 100644 ecs/jskult-batch-daily/src/batch/laundering/mst_inst_create.py diff --git a/ecs/jskult-batch-daily/.env.example b/ecs/jskult-batch-daily/.env.example index 91ae5f74..a7c88968 100644 --- a/ecs/jskult-batch-daily/.env.example +++ b/ecs/jskult-batch-daily/.env.example @@ -8,3 +8,6 @@ ULTMARC_DATA_BUCKET=**************** ULTMARC_DATA_FOLDER=recv ULTMARC_BACKUP_BUCKET=**************** ULTMARC_BACKUP_FOLDER=ultmarc +JSKULT_CONFIG_BUCKET=********************** +JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar +JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME=jskult_holiday_list.txt diff --git a/ecs/jskult-batch-daily/src/aws/s3.py b/ecs/jskult-batch-daily/src/aws/s3.py index 43460f8d..aa235224 100644 --- a/ecs/jskult-batch-daily/src/aws/s3.py +++ b/ecs/jskult-batch-daily/src/aws/s3.py @@ -76,6 +76,20 @@ class UltmarcBucket(S3Bucket): self._s3_client.delete_file(self._bucket_name, dat_file_key) +class ConfigBucket(S3Bucket): + _bucket_name = environment.JSKULT_CONFIG_BUCKET + + def download_holiday_list(self): + # 一時ファイルとして保存する + temporary_dir = tempfile.mkdtemp() + temporary_file_path = path.join(temporary_dir, environment.JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME) + holiday_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME}' + with open(temporary_file_path, mode='wb') as f: + self._s3_client.download_file(self._bucket_name, holiday_list_key, f) + f.seek(0) + return temporary_file_path + + class JskUltBackupBucket(S3Bucket): _bucket_name = environment.ULTMARC_BACKUP_BUCKET diff --git a/ecs/jskult-batch-daily/src/batch/common/batch_config.py b/ecs/jskult-batch-daily/src/batch/common/batch_config.py deleted file mode 100644 index e1db0417..00000000 --- a/ecs/jskult-batch-daily/src/batch/common/batch_config.py +++ /dev/null @@ -1,10 +0,0 @@ -class BatchConfig: - # 処理日(yyyy/mm/dd形式) - syor_date: str - __instance = None - - @classmethod - def get_instance(cls): - if cls.__instance is None: - cls.__instance = cls() - return cls.__instance diff --git a/ecs/jskult-batch-daily/src/batch/common/batch_context.py b/ecs/jskult-batch-daily/src/batch/common/batch_context.py new file mode 100644 index 00000000..3b3ac157 --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/common/batch_context.py @@ -0,0 +1,39 @@ +class BatchContext: + __instance = None + __syor_date: str # 処理日(yyyy/mm/dd形式) + __is_not_business_day: bool # 日次バッチ起動日フラグ + __is_ultmarc_imported: bool # アルトマーク取込実施済フラグ + + def __init__(self) -> None: + self.__is_not_business_day = False + self.__is_ultmarc_imported = False + + @classmethod + def get_instance(cls): + if cls.__instance is None: + cls.__instance = cls() + return cls.__instance + + @property + def syor_date(self): + return self.__syor_date + + @syor_date.setter + def syor_date(self, syor_date_str: str): + self.__syor_date = syor_date_str + + @property + def is_not_business_day(self): + return self.__is_not_business_day + + @is_not_business_day.setter + def is_not_business_day(self, flag: bool): + self.__is_not_business_day = flag + + @property + def is_ultmarc_imported(self): + return self.__is_ultmarc_imported + + @is_ultmarc_imported.setter + def is_ultmarc_imported(self, flag: bool): + self.__is_ultmarc_imported = flag diff --git a/ecs/jskult-batch-daily/src/batch/common/calendar_file.py b/ecs/jskult-batch-daily/src/batch/common/calendar_file.py new file mode 100644 index 00000000..b456f03c --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/common/calendar_file.py @@ -0,0 +1,32 @@ +from src.system_var import constants + + +class CalendarFile: + """カレンダーファイル""" + + __calendar_file_lines: list[str] + + def __init__(self, calendar_file_path): + with open(calendar_file_path) as f: + self.__calendar_file_lines: list[str] = f.readlines() + + def compare_date(self, date_str: str) -> bool: + """与えられた日付がカレンダーファイル内に含まれているかどうか + カレンダーファイル内の日付はyyyy/mm/ddで書かれている前提 + コメント（#）が含まれている行は無視される + + Args: + date_str (str): yyyy/mm/dd文字列 + + Returns: + bool: 含まれていればTrue + """ + for calendar_date in self.__calendar_file_lines: + # コメント行が含まれている場合はスキップ + if constants.CALENDAR_COMMENT_SYMBOL in calendar_date: + continue + + if date_str in calendar_date: + return True + + return False diff --git a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py index c8a28baf..6e611bc9 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py @@ -1,12 +1,12 @@ from src.batch.batch_functions import logging_sql -from src.batch.common.batch_config import BatchConfig +from src.batch.common.batch_context import BatchContext from src.db.database import Database from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger from src.time.elapsed_time import ElapsedTime logger = get_logger('48-施設担当者マスタ洗替') -batch_config = BatchConfig.get_instance() +batch_context = BatchContext.get_instance() def exec(): @@ -15,7 +15,7 @@ def exec(): logger.info('##########################') logger.info('START Changing Employee in charge of institution PGM.') # 業務日付を取得 - syor_date = batch_config.syor_date + syor_date = batch_context.syor_date # `emp_chg_inst_lau`をTruncate truncate_emp_chg_inst_lau(db) # emp_chg_inst から、`emp_chg_inst_lau`へInsert diff --git a/ecs/jskult-batch-daily/src/batch/laundering/mst_inst_create.py b/ecs/jskult-batch-daily/src/batch/laundering/mst_inst_create.py new file mode 100644 index 00000000..a2e52df4 --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/laundering/mst_inst_create.py @@ -0,0 +1,3 @@ +def exec(): + """メルク施設マスタ作成""" + pass diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py index 21c0c302..4c331b56 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py @@ -1,17 +1,20 @@ -"""アルトマークデータ保管""" +"""アルトマークデータ処理""" + +from datetime import datetime from src.aws.s3 import UltmarcBucket -from src.batch.common.batch_config import BatchConfig +from src.batch.common.batch_context import BatchContext from src.batch.ultmarc.datfile import DatFile from src.batch.ultmarc.utmp_tables.ultmarc_table_mapper_factory import \ UltmarcTableMapperFactory from src.db.database import Database from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger +from src.system_var import constants -logger = get_logger('アルトマークデータ保管') +logger = get_logger('アルトマークデータ処理') ultmarc_bucket = UltmarcBucket() -batch_config = BatchConfig.get_instance() +batch_context = BatchContext.get_instance() def exec_import(): @@ -23,13 +26,16 @@ def exec_import(): # ファイルがない場合は処理せず、正常終了とする if len(dat_file_list) == 0: - logger.info('ファイルがないため、アルトマーク取込処理をスキップします') + logger.info('取込ファイルがないため、アルトマーク取込処理をスキップします') + # アルトマークデータ受信予定曜日である月曜日は、保守ユーザーに通知する + if datetime.strptime(batch_context.syor_date, '%Y/%m/%d').weekday() == constants.WEEKDAY_MONDAY: + logger.info('[NOTICE]アルトマークデータの受信がありませんでした') return # ファイルが複数ある場合はエラーとする if len(dat_file_list) > 1: logger.error(f'複数の取込ファイルがあるため、異常終了 ファイル一覧:{dat_file_list}') - return + raise BatchOperationException() # ファイルの件数は必ず1件になる dat_file_info = dat_file_list[0] @@ -42,7 +48,7 @@ def exec_import(): logger.info(f"{dat_file_name}を取り込みます") # ファイルをバックアップ # 現行は、jobctrl_dailyの先頭でやっている - ultmarc_bucket.backup_dat_file(dat_file_name, batch_config.syor_date) + ultmarc_bucket.backup_dat_file(dat_file_name, batch_context.syor_date) # datファイルをダウンロード local_file_path = ultmarc_bucket.download_dat_file(dat_file_name) dat_file = DatFile.from_path(local_file_path) @@ -51,11 +57,18 @@ def exec_import(): # 処理後、ファイルをS3から削除する logger.info(f'取り込み処理が完了したため、datファイルを削除。ファイル名={dat_file_name}') ultmarc_bucket.delete_dat_file(dat_file_name) + # アルトマーク取込済をマーク + batch_context.is_ultmarc_imported = True + logger.info('アルトマーク取込処理: 終了') except Exception as e: logger.exception(e) raise BatchOperationException(e) - finally: - logger.info('アルトマーク取込処理: 終了') + + +def exec_export(): + # 取り込み日に限らず、毎日実行する + """V実消化用施設・薬局薬店データ作成処理""" + pass def _import_to_ultmarc_table(dat_file: DatFile): diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/utmp_tables/table_mapper/ultmarc_table_mapper.py b/ecs/jskult-batch-daily/src/batch/ultmarc/utmp_tables/table_mapper/ultmarc_table_mapper.py index 5a62d171..7aa1f311 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/utmp_tables/table_mapper/ultmarc_table_mapper.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/utmp_tables/table_mapper/ultmarc_table_mapper.py @@ -1,12 +1,12 @@ from abc import ABCMeta, abstractmethod from datetime import datetime -from src.batch.common.batch_config import BatchConfig +from src.batch.common.batch_context import BatchContext from src.batch.ultmarc.utmp_tables.tables.ultmarc_table import UltmarcTable from src.db.database import Database # 処理日を使用するために、configを使用 -batch_config = BatchConfig.get_instance() +batch_context = BatchContext.get_instance() class UltmarcTableMapper(metaclass=ABCMeta): @@ -32,7 +32,7 @@ class UltmarcTableMapper(metaclass=ABCMeta): 'execute_date_str_ymd': execute_date_str_ymd, 'execute_datetime': execute_datetime, # バッチ共通設定から処理日を取得 - 'syor_date': batch_config.syor_date + 'syor_date': batch_context.syor_date } @abstractmethod diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index c56fe8ba..72395a0c 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -1,9 +1,11 @@ """実消化&アルトマーク 日次バッチ処理""" +from src.aws.s3 import ConfigBucket from src.batch.batch_functions import ( get_batch_statuses, update_batch_process_complete, update_batch_processing_flag_in_processing) -from src.batch.common.batch_config import BatchConfig +from src.batch.common.batch_context import BatchContext +from src.batch.common.calendar_file import CalendarFile from src.batch.laundering import sales_laundering from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException @@ -13,7 +15,7 @@ from src.system_var import constants logger = get_logger('日次処理コントロール') # バッチ共通設定を取得 -batch_config = BatchConfig.get_instance() +batch_context = BatchContext.get_instance() def exec(): @@ -32,13 +34,34 @@ def exec(): return constants.BATCH_EXIT_CODE_SUCCESS # dump取得が正常終了していない場合、後続の処理は行わない - if dump_status_kbn != constants.DUMP_STATUS_KBN_COMPLETE: + if dump_status_kbn != constants.DUMP_STATUS_KBN_COMPLETE and dump_status_kbn != constants.DUMP_STATUS_KBN_DUMP_SKIP: logger.error('dump取得が正常終了していないため、日次バッチ処理を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS # TODO: エラーで返せば、StepFunctionsでリトライしてくれるので、どう返すかは要検討 logger.info(f'処理日＝{syor_date}') # バッチ共通設定に処理日を追加 - batch_config.syor_date = syor_date + batch_context.syor_date = syor_date + + # 稼働日かかどうかを設定 + # 非営業日ファイルをダウンロードして判定 + holiday_list_file_path = ConfigBucket().download_holiday_list() + holiday_calendar = CalendarFile(holiday_list_file_path) + # 非稼働日かどうかを設定 + batch_context.is_not_business_day = not holiday_calendar.compare_date(syor_date) + # for row in io.TextIOWrapper(io.BytesIO(non_business_day_response["Body"].read()), encoding='utf-8'): + # if row[INDEX_ROW_COMMENT_SYMBOL] == ROW_COMMENT_SYMBOL: + # continue + # non_date = row.rstrip('\n') + # # 日付妥当性判定 + # try: + # datetime.datetime.strptime(non_date, "%Y/%m/%d") + # except Exception as e: + # raise e + # none_business_day_list.append(non_date) + + # if execute_date in none_business_day_list: + # logger.info('I-02-05 本日は非営業日のため、チェック処理をスキップします') + # return # バッチ処理中に更新 try: @@ -84,29 +107,42 @@ def exec(): # except BatchOperationException as e: # logger.error(f'卸在庫データ取込処理エラー（異常終了）{e}') logger.info('日次処理（アルトマーク）') - if True: # アルトマーク取り込み処理内で実装 - logger.info('日次処理（アルトマーク）実行対象日でない為未実行') + # if True: # アルトマーク取り込み処理内で実装 + # logger.info('日次処理（アルトマーク）実行対象日でない為未実行') + try: logger.info('アルトマーク取込：起動') ultmarc_process.exec_import() logger.info('アルトマーク取込：終了') except BatchOperationException as e: - logger.error(f'アルトマーク取込処理エラー（異常終了）{e}') + logger.exception(f'アルトマーク取込処理エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS - if True: # 休日判定、メルク施設マスタ作成内で行いたい - try: - logger.info('メルク施設マスタ作成') - logger.info('メルク施設マスタ作成終了') - except BatchOperationException as e: - logger.error(f'メルク施設マスタ作成エラー（異常終了）{e}') - return constants.BATCH_EXIT_CODE_SUCCESS - try: - logger.info('DCF施設統合マスタ作成') - logger.info('DCF施設統合マスタ作成終了') - except BatchOperationException as e: - logger.error(f'DCF施設統合マスタ作成エラー（異常終了）{e}') - return constants.BATCH_EXIT_CODE_SUCCESS - # if False: # ($holiday === FALSE) # DCF施設統合マスタを作成する日は必ず休日なので、V実消化の取り込みは行わない。 + + try: + logger.info('V実消化用施設・薬局薬店データ作成処理：起動') + ultmarc_process.exec_export() + logger.info('V実消化用施設・薬局薬店データ作成処理：終了') + except BatchOperationException as e: + logger.exception(f'V実消化用施設・薬局薬店データ作成処理エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # TODO: 現行では休日も、メルク施設マスタと施設統合マスタの作成を行っている。 + # 実績洗い替えの中でもメルク施設マスタと施設統合マスタの作成は行っているが、他の処理は動かしたくない。 + # つまり、休日だったら、メルク施設マスタと施設統合マスタ以外の作成は行わない、とすれば、シンプルになる？ + # if True: # 休日判定、メルク施設マスタ作成内で行いたい + # try: + # logger.info('メルク施設マスタ作成') + # logger.info('メルク施設マスタ作成終了') + # except BatchOperationException as e: + # logger.error(f'メルク施設マスタ作成エラー（異常終了）{e}') + # return constants.BATCH_EXIT_CODE_SUCCESS + # try: + # logger.info('DCF施設統合マスタ作成') + # logger.info('DCF施設統合マスタ作成終了') + # except BatchOperationException as e: + # logger.error(f'DCF施設統合マスタ作成エラー（異常終了）{e}') + # return constants.BATCH_EXIT_CODE_SUCCESS + # if False: # ($holiday === FALSE) # V実消化非稼働日は、V実消化の取り込みは行わない。 # TODO: データ存在確認は実消化取り込み内で行う # logger.info('V実消化連携データ存在確認') # if True: @@ -120,8 +156,18 @@ def exec(): logger.exception(f'V実消化取込処理エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS + # 非稼働日や、月火水に限らず、必ず実施 + logger.info('メルク施設マスタ作成') + try: + logger.info('メルク施設マスタ作成：起動') + logger.info('メルク施設マスタ作成：終了') + except BatchOperationException as e: + logger.exception(f'メルク施設マスタ作成 エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + logger.info('日次処理（実績更新）') - # ここも、休日判定を内側で行う + # ここも、休日判定を実績更新内で行う + # 休日であっても、施設統合マスタの作成だけは行う。 try: # ここで、生物由来ロット分解と並行処理 logger.info('実績更新：起動') @@ -140,7 +186,8 @@ def exec(): logger.exception(f'日次バッチ完了処理 エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS - logger.info('日次ジョブ：終了（正常終了）') + # 正常終了を保守ユーザーに通知 + logger.info('[NOTICE]日次ジョブ：終了（正常終了）') return constants.BATCH_EXIT_CODE_SUCCESS except Exception as e: logger.exception(f'日次バッチ処理中に想定外のエラーが発生しました {e}') diff --git a/ecs/jskult-batch-daily/src/system_var/constants.py b/ecs/jskult-batch-daily/src/system_var/constants.py index d5538a80..26e1c1a8 100644 --- a/ecs/jskult-batch-daily/src/system_var/constants.py +++ b/ecs/jskult-batch-daily/src/system_var/constants.py @@ -9,3 +9,11 @@ BATCH_ACTF_BATCH_IN_PROCESSING = '1' DUMP_STATUS_KBN_UNPROCESSED = '0' # dump取得状態区分：dump取得正常終了 DUMP_STATUS_KBN_COMPLETE = '2' +# dump取得状態区分：dump取得スキップ +DUMP_STATUS_KBN_DUMP_SKIP = '3' + +# カレンダーファイルのコメントシンボル +CALENDAR_COMMENT_SYMBOL = '#' + +# 月曜日(datetime.weekday()で月曜日を表す数字) +WEEKDAY_MONDAY = 0 diff --git a/ecs/jskult-batch-daily/src/system_var/environment.py b/ecs/jskult-batch-daily/src/system_var/environment.py index e7248191..650826ee 100644 --- a/ecs/jskult-batch-daily/src/system_var/environment.py +++ b/ecs/jskult-batch-daily/src/system_var/environment.py @@ -12,6 +12,9 @@ ULTMARC_DATA_BUCKET = os.environ['ULTMARC_DATA_BUCKET'] ULTMARC_DATA_FOLDER = os.environ['ULTMARC_DATA_FOLDER'] ULTMARC_BACKUP_BUCKET = os.environ['ULTMARC_BACKUP_BUCKET'] ULTMARC_BACKUP_FOLDER = os.environ['ULTMARC_BACKUP_FOLDER'] +JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] +JSKULT_CONFIG_CALENDAR_FOLDER = os.environ['JSKULT_CONFIG_CALENDAR_FOLDER'] +JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME = os.environ['JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME'] # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') diff --git a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/test_com_alma_mapper.py b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/test_com_alma_mapper.py index 238f8a3d..8390096f 100644 --- a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/test_com_alma_mapper.py +++ b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/test_com_alma_mapper.py @@ -3,7 +3,7 @@ from datetime import datetime import pytest -from src.batch.common.batch_config import BatchConfig +from src.batch.common.batch_context import BatchContext from src.batch.ultmarc.utmp_tables.table_mapper.concrete import com_alma_mapper from src.db.database import Database from tests.testing_utility import (assert_table_results, @@ -18,13 +18,13 @@ class TestComAlmaMapper: """レイアウト区分004: COM_出身校""" db: Database - batch_config: BatchConfig + batch_context: BatchContext test_file_path: str = path.dirname(__file__) @pytest.fixture(autouse=True, scope='function') def pre_test(self, database: Database): """テスト実行前後処理""" - self.batch_config = BatchConfig.get_instance() + self.batch_context = BatchContext.get_instance() # setup self.db = database self.db.connect() @@ -50,7 +50,7 @@ class TestComAlmaMapper: # Arrange # 処理日設定 - self.batch_config.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') + self.batch_context.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') # テスト用のCSVを読み込む test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_alma_insert.csv')) # 一旦全データをDBから削除 @@ -95,7 +95,7 @@ class TestComAlmaMapper: # Arrange # 処理日設定 - self.batch_config.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') + self.batch_context.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') # テスト用のCSVを読み込む test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_alma_update.csv')) # 一旦全データをDBから削除 @@ -152,7 +152,7 @@ class TestComAlmaMapper: # Arrange # 処理日設定 - self.batch_config.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') + self.batch_context.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') # テスト用のCSVを読み込む test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_alma_delete.csv')) # 一旦全データをDBから削除 diff --git a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_dr_wrkplace/test_com_dr_wrkplace_mapper.py b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_dr_wrkplace/test_com_dr_wrkplace_mapper.py index f838dabf..89e65b54 100644 --- a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_dr_wrkplace/test_com_dr_wrkplace_mapper.py +++ b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_dr_wrkplace/test_com_dr_wrkplace_mapper.py @@ -2,7 +2,7 @@ import os.path as path import pytest -from src.batch.common.batch_config import BatchConfig +from src.batch.common.batch_context import BatchContext from src.batch.ultmarc.utmp_tables.table_mapper.concrete import \ com_dr_wrkplace_mapper from src.db.database import Database @@ -18,13 +18,13 @@ class TestComDrWrkplaceMapper: """COM_DCF医師勤務先""" db: Database - batch_config: BatchConfig + batch_context: BatchContext test_file_path: str = path.dirname(__file__) @ pytest.fixture(autouse=True, scope='function') def pre_test(self, database: Database): """テスト実行前後処理""" - self.batch_config = BatchConfig.get_instance() + self.batch_context = BatchContext.get_instance() # setup self.db = database @@ -53,7 +53,7 @@ class TestComDrWrkplaceMapper: # Arrange # 処理日設定 # 適用開始日と同値になる - self.batch_config.syor_date = '2020/02/22' + self.batch_context.syor_date = '2020/02/22' # テスト用のCSVを読み込む test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_dr_wrkplace_insert.csv')) # 一旦全データをDBから削除 @@ -147,7 +147,7 @@ class TestComDrWrkplaceMapper: # Arrange # 処理日設定 # 適用開始日と同値、適用終了日の+1日になる - self.batch_config.syor_date = '2020/02/23' + self.batch_context.syor_date = '2020/02/23' # テスト用のCSVを読み込む test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_dr_wrkplace_update.csv')) # 一旦全データをDBから削除 @@ -264,7 +264,7 @@ class TestComDrWrkplaceMapper: # Arrange # 処理日設定 - self.batch_config.syor_date = '2020/02/24' + self.batch_context.syor_date = '2020/02/24' # テスト用のCSVを読み込む test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_dr_wrkplace_delete.csv')) # 一旦全データをDBから削除 diff --git a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_hamtec/test_com_hamtec_mapper.py b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_hamtec/test_com_hamtec_mapper.py index 08c773f9..c7434739 100644 --- a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_hamtec/test_com_hamtec_mapper.py +++ b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_hamtec/test_com_hamtec_mapper.py @@ -3,7 +3,7 @@ from datetime import datetime import pytest -from src.batch.common.batch_config import BatchConfig +from src.batch.common.batch_context import BatchContext from src.batch.ultmarc.utmp_tables.table_mapper.concrete import \ com_hamtec_mapper from src.db.database import Database @@ -19,13 +19,13 @@ class TestComHamtecMapper: """レイアウト区分021: COM_高度先進医療""" db: Database - batch_config: BatchConfig + batch_context: BatchContext test_file_path: str = path.dirname(__file__) @pytest.fixture(autouse=True, scope='function') def pre_test(self, database: Database): """テスト実行前後処理""" - self.batch_config = BatchConfig.get_instance() + self.batch_context = BatchContext.get_instance() # setup self.db = database self.db.connect() @@ -51,7 +51,7 @@ class TestComHamtecMapper: # Arrange # 処理日設定 - self.batch_config.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') + self.batch_context.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') # テスト用のCSVを読み込む test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_hamtec_insert.csv')) # 一旦全データをDBから削除 @@ -96,7 +96,7 @@ class TestComHamtecMapper: # Arrange # 処理日設定 - self.batch_config.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') + self.batch_context.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') # テスト用のCSVを読み込む test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_hamtec_update.csv')) # 一旦全データをDBから削除 @@ -153,7 +153,7 @@ class TestComHamtecMapper: # Arrange # 処理日設定 - self.batch_config.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') + self.batch_context.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') # テスト用のCSVを読み込む test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_hamtec_delete.csv')) # 一旦全データをDBから削除 From 60b95a2474b61a403f23503b7ec4fa97aa1767d0 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 14 Apr 2023 17:42:15 +0900 Subject: [PATCH 06/32] =?UTF-8?q?feat:=20=E5=AE=9F=E6=B6=88=E5=8C=96?= =?UTF-8?q?=E9=9D=9E=E7=A8=BC=E5=83=8D=E6=97=A5=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jskult/calendar/jskult_holiday_list.txt | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 s3/config/jskult/calendar/jskult_holiday_list.txt diff --git a/s3/config/jskult/calendar/jskult_holiday_list.txt b/s3/config/jskult/calendar/jskult_holiday_list.txt new file mode 100644 index 00000000..f154ef79 --- /dev/null +++ b/s3/config/jskult/calendar/jskult_holiday_list.txt @@ -0,0 +1,131 @@ +2022/12/03 +2022/12/04 +2022/12/10 +2022/12/11 +2022/12/17 +2022/12/18 +2022/12/24 +2022/12/25 +2022/12/29 +2022/12/30 +2022/12/31 +2023/01/01 +2023/01/02 +2023/01/03 +2023/01/07 +2023/01/08 +2023/01/09 +2023/01/14 +2023/01/15 +2023/01/21 +2023/01/22 +2023/01/28 +2023/01/29 +2023/02/04 +2023/02/05 +2023/02/11 +2023/02/12 +2023/02/18 +2023/02/19 +2023/02/23 +2023/02/25 +2023/02/26 +2023/03/04 +2023/03/05 +2023/03/11 +2023/03/12 +2023/03/18 +2023/03/19 +2023/03/21 +2023/03/25 +2023/03/26 +2023/04/01 +2023/04/02 +2023/04/08 +2023/04/09 +2023/04/15 +2023/04/16 +2023/04/22 +2023/04/23 +2023/04/29 +2023/04/30 +2023/05/03 +2023/05/04 +2023/05/05 +2023/05/06 +2023/05/07 +2023/05/13 +2023/05/14 +2023/05/20 +2023/05/21 +2023/05/27 +2023/05/28 +2023/06/03 +2023/06/04 +2023/06/10 +2023/06/11 +2023/06/17 +2023/06/18 +2023/06/24 +2023/06/25 +2023/07/01 +2023/07/02 +2023/07/08 +2023/07/09 +2023/07/15 +2023/07/16 +2023/07/17 +2023/07/22 +2023/07/23 +2023/07/29 +2023/07/30 +2023/08/05 +2023/08/06 +2023/08/11 +2023/08/12 +2023/08/13 +2023/08/19 +2023/08/20 +2023/08/26 +2023/08/27 +2023/09/02 +2023/09/03 +2023/09/09 +2023/09/10 +2023/09/16 +2023/09/17 +2023/09/18 +2023/09/23 +2023/09/24 +2023/09/30 +2023/10/01 +2023/10/07 +2023/10/08 +2023/10/09 +2023/10/14 +2023/10/15 +2023/10/21 +2023/10/22 +2023/10/28 +2023/10/29 +2023/11/03 +2023/11/04 +2023/11/05 +2023/11/11 +2023/11/12 +2023/11/18 +2023/11/19 +2023/11/23 +2023/11/25 +2023/11/26 +2023/12/02 +2023/12/03 +2023/12/09 +2023/12/10 +2023/12/16 +2023/12/17 +2023/12/23 +2023/12/24 +2023/12/29 +2023/12/30 +2023/12/31 \ No newline at end of file From 8c73f153d6dd9112d6e07968912b00587d8931b8 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 14 Apr 2023 17:46:13 +0900 Subject: [PATCH 07/32] =?UTF-8?q?feat:=20=E4=B8=8D=E8=A6=81=E3=81=AA?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=81=AA=E3=81=A9=E3=82=92?= =?UTF-8?q?=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/jobctrl_daily.py | 83 +-------------------- 1 file changed, 2 insertions(+), 81 deletions(-) diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 72395a0c..94e8a835 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -34,7 +34,7 @@ def exec(): return constants.BATCH_EXIT_CODE_SUCCESS # dump取得が正常終了していない場合、後続の処理は行わない - if dump_status_kbn != constants.DUMP_STATUS_KBN_COMPLETE and dump_status_kbn != constants.DUMP_STATUS_KBN_DUMP_SKIP: + if dump_status_kbn not in [constants.DUMP_STATUS_KBN_COMPLETE, constants.DUMP_STATUS_KBN_DUMP_SKIP]: logger.error('dump取得が正常終了していないため、日次バッチ処理を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS # TODO: エラーで返せば、StepFunctionsでリトライしてくれるので、どう返すかは要検討 @@ -42,26 +42,10 @@ def exec(): # バッチ共通設定に処理日を追加 batch_context.syor_date = syor_date - # 稼働日かかどうかを設定 - # 非営業日ファイルをダウンロードして判定 + # 稼働日かかどうかを、非営業日ファイルをダウンロードして判定 holiday_list_file_path = ConfigBucket().download_holiday_list() holiday_calendar = CalendarFile(holiday_list_file_path) - # 非稼働日かどうかを設定 batch_context.is_not_business_day = not holiday_calendar.compare_date(syor_date) - # for row in io.TextIOWrapper(io.BytesIO(non_business_day_response["Body"].read()), encoding='utf-8'): - # if row[INDEX_ROW_COMMENT_SYMBOL] == ROW_COMMENT_SYMBOL: - # continue - # non_date = row.rstrip('\n') - # # 日付妥当性判定 - # try: - # datetime.datetime.strptime(non_date, "%Y/%m/%d") - # except Exception as e: - # raise e - # none_business_day_list.append(non_date) - - # if execute_date in none_business_day_list: - # logger.info('I-02-05 本日は非営業日のため、チェック処理をスキップします') - # return # バッチ処理中に更新 try: @@ -70,46 +54,7 @@ def exec(): logger.exception(f'処理フラグ更新(未処理→処理中) エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS - # 休日判定ファイルを読み込み(ここは、各処理内に押し込むつもり) - logger.info('休日判定処理') - if True: # 休日判定 - logger.info('非営業日かつ月、火、水以外です。') # 分岐 - # try: - # # 処理中フラグ判定。ここでdumpのフラグも見る - # logger.info('処理フラグ更新中') - # logger.info('処理フラグ更新終了') - # except BatchOperationException as e: - # logger.error(f'処理フラグ更新処理エラー（異常終了）{e}') - logger.info('日次ジョブ：終了（正常終了）') - # try: - # logger.info('日次ジョブ処理中判定') - # if True: # 処理中判定 - # logger.error('処理フラグ処理中（異常終了）') - # logger.info('処理中フラグの更新：起動') - # logger.info('処理中フラグの更新：終了') - # except BatchOperationException as e: - # logger.error(f'日次ジョブ処理中エラー（異常終了）{e}') - # TODO: 以下、卸在庫取り込み処理は、実消化取り込み内で行う - # logger.info('卸在庫データ取込判定') - # if True: # 卸在庫日判定 - # logger.info('卸在庫データ取込日です') - # logger.debug('卸在庫データファイル名： {_PATH_OROSHI_ZAIKO}') - # if True: # 卸在庫ファイル存在確認なければ異常終了 - # logger.error('卸在庫データ存在確認エラー（異常終了）') - # logger.info('卸在庫データ存在確認：取込処理開始') - # logger.debug('卸在庫データファイル名作成： {read_filename}') - # logger.debug('ファイル移動OK：{_MOVE_OROSHI_ZAIKO}') # S3からダウンロード - # logger.debug('ファイル解凍OK：{sprintf(_ZIP_OROSHI_ZAIKO, $read_filename)}') # gunzip -fなので、gzipを使う - # logger.debug('ファイル名変更OK： {sprintf(_RENAME_OROSHI_ZAIKO, $read_filename)}') # S3にアップロード - # try: - # logger.info('卸在庫データ取込：起動') - # logger.info('卸在庫データ取込：終了') - # except BatchOperationException as e: - # logger.error(f'卸在庫データ取込処理エラー（異常終了）{e}') logger.info('日次処理（アルトマーク）') - # if True: # アルトマーク取り込み処理内で実装 - # logger.info('日次処理（アルトマーク）実行対象日でない為未実行') - try: logger.info('アルトマーク取込：起動') ultmarc_process.exec_import() @@ -126,28 +71,6 @@ def exec(): logger.exception(f'V実消化用施設・薬局薬店データ作成処理エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS - # TODO: 現行では休日も、メルク施設マスタと施設統合マスタの作成を行っている。 - # 実績洗い替えの中でもメルク施設マスタと施設統合マスタの作成は行っているが、他の処理は動かしたくない。 - # つまり、休日だったら、メルク施設マスタと施設統合マスタ以外の作成は行わない、とすれば、シンプルになる？ - # if True: # 休日判定、メルク施設マスタ作成内で行いたい - # try: - # logger.info('メルク施設マスタ作成') - # logger.info('メルク施設マスタ作成終了') - # except BatchOperationException as e: - # logger.error(f'メルク施設マスタ作成エラー（異常終了）{e}') - # return constants.BATCH_EXIT_CODE_SUCCESS - # try: - # logger.info('DCF施設統合マスタ作成') - # logger.info('DCF施設統合マスタ作成終了') - # except BatchOperationException as e: - # logger.error(f'DCF施設統合マスタ作成エラー（異常終了）{e}') - # return constants.BATCH_EXIT_CODE_SUCCESS - # if False: # ($holiday === FALSE) # V実消化非稼働日は、V実消化の取り込みは行わない。 - # TODO: データ存在確認は実消化取り込み内で行う - # logger.info('V実消化連携データ存在確認') - # if True: - # logger.error('V実消化連携データ存在確認（異常終了）') - logger.info('日次処理（V実消化）') try: logger.info('V実消化取込：起動') @@ -166,8 +89,6 @@ def exec(): return constants.BATCH_EXIT_CODE_SUCCESS logger.info('日次処理（実績更新）') - # ここも、休日判定を実績更新内で行う - # 休日であっても、施設統合マスタの作成だけは行う。 try: # ここで、生物由来ロット分解と並行処理 logger.info('実績更新：起動') From 834bc503dd9e8b019eb78f6ad765b666d23e418e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 14 Apr 2023 18:21:52 +0900 Subject: [PATCH 08/32] =?UTF-8?q?feat:=20=E5=88=B6=E5=BE=A1=E5=AF=BE?= =?UTF-8?q?=E8=B1=A1=E3=81=AE=E5=87=A6=E7=90=86=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/laundering/create_dcf_inst_merge.py | 15 ++++++++++ .../src/batch/laundering/create_mst_inst.py | 21 ++++++++++++++ .../src/batch/laundering/mst_inst_create.py | 3 -- .../src/batch/laundering/sales_laundering.py | 10 ++++++- .../src/batch/ultmarc/ultmarc_process.py | 5 ++-- ecs/jskult-batch-daily/src/jobctrl_daily.py | 29 ++++++++++++------- 6 files changed, 67 insertions(+), 16 deletions(-) create mode 100644 ecs/jskult-batch-daily/src/batch/laundering/create_dcf_inst_merge.py create mode 100644 ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py delete mode 100644 ecs/jskult-batch-daily/src/batch/laundering/mst_inst_create.py diff --git a/ecs/jskult-batch-daily/src/batch/laundering/create_dcf_inst_merge.py b/ecs/jskult-batch-daily/src/batch/laundering/create_dcf_inst_merge.py new file mode 100644 index 00000000..974aa329 --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/laundering/create_dcf_inst_merge.py @@ -0,0 +1,15 @@ +from src.batch.common.batch_context import BatchContext +from src.logging.get_logger import get_logger + +batch_context = BatchContext.get_instance() +logger = get_logger('DCF施設統合マスタ作成') + + +def exec(): + """DCF施設統合マスタ作成""" + + # アルトマーク取込が行われていない場合は処理をスキップする + if not batch_context.is_ultmarc_imported: + logger.info('アルトマーク取込が行われていないため、DCF施設統合マスタ作成処理をスキップします。') + + pass diff --git a/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py b/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py new file mode 100644 index 00000000..832e430d --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py @@ -0,0 +1,21 @@ +from src.batch.common.batch_context import BatchContext +from src.logging.get_logger import get_logger + +batch_context = BatchContext.get_instance() +logger = get_logger('メルク施設統合マスタ') + + +def exec(): + """メルク施設統合マスタ""" + + # 営業日ではない場合、処理をスキップする + if batch_context.is_not_business_day: + logger.info('営業日ではないため、メルク施設統合マスタ作成処理をスキップします。') + return + + # アルトマーク取込が行われていない場合は処理をスキップする + if not batch_context.is_ultmarc_imported: + logger.info('アルトマーク取込が行われていないため、メルク施設統合マスタ作成処理をスキップします。') + return + + pass diff --git a/ecs/jskult-batch-daily/src/batch/laundering/mst_inst_create.py b/ecs/jskult-batch-daily/src/batch/laundering/mst_inst_create.py deleted file mode 100644 index a2e52df4..00000000 --- a/ecs/jskult-batch-daily/src/batch/laundering/mst_inst_create.py +++ /dev/null @@ -1,3 +0,0 @@ -def exec(): - """メルク施設マスタ作成""" - pass diff --git a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py index c1884b7d..962b3e12 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py @@ -1,11 +1,19 @@ +from src.batch.common.batch_context import BatchContext from src.batch.laundering import emp_chg_inst_laundering from src.logging.get_logger import get_logger +batch_context = BatchContext.get_instance() + logger = get_logger('実績洗替') def exec(): """実績洗替処理""" - logger.info('Start Jisseki Araigae Batch PGM.') + + # 非営業日の場合実績洗替処理は実行しない + if batch_context.is_not_business_day: + logger.info('非営業日のため、実績洗替処理をスキップします。') + return + # 施設担当者洗替 emp_chg_inst_laundering.exec() diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py index 4c331b56..8ddae44a 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py @@ -66,9 +66,10 @@ def exec_import(): def exec_export(): - # 取り込み日に限らず、毎日実行する """V実消化用施設・薬局薬店データ作成処理""" - pass + if not batch_context.is_ultmarc_imported: + logger.info('アルトマーク取込が行われていないため、V実消化用施設・薬局薬店データ作成処理をスキップします。') + return def _import_to_ultmarc_table(dat_file: DatFile): diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 94e8a835..0b865b9b 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -6,7 +6,8 @@ from src.batch.batch_functions import ( update_batch_processing_flag_in_processing) from src.batch.common.batch_context import BatchContext from src.batch.common.calendar_file import CalendarFile -from src.batch.laundering import sales_laundering +from src.batch.laundering import (create_dcf_inst_merge, create_mst_inst, + sales_laundering) from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger @@ -20,7 +21,7 @@ batch_context = BatchContext.get_instance() def exec(): try: - logger.info('日次ジョブ：開始') + logger.info('日次バッチ：開始') try: # 日次バッチ処置中フラグ、dump処理状態区分、処理日を取得 batch_processing_flag, dump_status_kbn, syor_date = get_batch_statuses() @@ -36,7 +37,7 @@ def exec(): # dump取得が正常終了していない場合、後続の処理は行わない if dump_status_kbn not in [constants.DUMP_STATUS_KBN_COMPLETE, constants.DUMP_STATUS_KBN_DUMP_SKIP]: logger.error('dump取得が正常終了していないため、日次バッチ処理を終了します。') - return constants.BATCH_EXIT_CODE_SUCCESS # TODO: エラーで返せば、StepFunctionsでリトライしてくれるので、どう返すかは要検討 + return constants.BATCH_EXIT_CODE_SUCCESS logger.info(f'処理日＝{syor_date}') # バッチ共通設定に処理日を追加 @@ -45,7 +46,7 @@ def exec(): # 稼働日かかどうかを、非営業日ファイルをダウンロードして判定 holiday_list_file_path = ConfigBucket().download_holiday_list() holiday_calendar = CalendarFile(holiday_list_file_path) - batch_context.is_not_business_day = not holiday_calendar.compare_date(syor_date) + batch_context.is_not_business_day = holiday_calendar.compare_date(syor_date) # バッチ処理中に更新 try: @@ -54,7 +55,6 @@ def exec(): logger.exception(f'処理フラグ更新(未処理→処理中) エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS - logger.info('日次処理（アルトマーク）') try: logger.info('アルトマーク取込：起動') ultmarc_process.exec_import() @@ -79,18 +79,16 @@ def exec(): logger.exception(f'V実消化取込処理エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS - # 非稼働日や、月火水に限らず、必ず実施 - logger.info('メルク施設マスタ作成') try: logger.info('メルク施設マスタ作成：起動') + create_mst_inst.exec() logger.info('メルク施設マスタ作成：終了') except BatchOperationException as e: logger.exception(f'メルク施設マスタ作成 エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS - logger.info('日次処理（実績更新）') try: - # ここで、生物由来ロット分解と並行処理 + # TODO: ここで、生物由来ロット分解と並行処理 logger.info('実績更新：起動') sales_laundering.exec() logger.info('生物由来ロット分解：起動') @@ -100,6 +98,16 @@ def exec(): logger.exception(f'実績更新処理エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS + # TODO: ここで並行処理合流 + + try: + logger.info('DCF施設統合マスタ作成：起動') + create_dcf_inst_merge.exec() + logger.info('DCF施設統合マスタ作成：終了') + except BatchOperationException as e: + logger.exception(f'DCF施設統合マスタ作成エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + # バッチ処理完了とし、処理日、バッチ処置中フラグ、dump取得状態区分を更新 try: update_batch_process_complete() @@ -108,8 +116,9 @@ def exec(): return constants.BATCH_EXIT_CODE_SUCCESS # 正常終了を保守ユーザーに通知 - logger.info('[NOTICE]日次ジョブ：終了（正常終了）') + logger.info('[NOTICE]日次バッチ：終了（正常終了）') return constants.BATCH_EXIT_CODE_SUCCESS + except Exception as e: logger.exception(f'日次バッチ処理中に想定外のエラーが発生しました {e}') raise e From 1e6856c51e30d68278fcb1cc353184076320a27a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 17 Apr 2023 11:03:42 +0900 Subject: [PATCH 09/32] =?UTF-8?q?feat:=20=E4=B8=A6=E8=A1=8C=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=AE=E8=A9=A6=E4=BD=9C=E5=93=81=E3=80=82=E4=BE=8B?= =?UTF-8?q?=E5=A4=96=E5=87=A6=E7=90=86=E3=81=A8=E3=81=8B=E3=81=AE=E6=A4=9C?= =?UTF-8?q?=E8=A8=BC=E3=81=8C=E3=81=BE=E3=81=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/bio_sales/__init__.py | 0 .../batch/bio_sales/create_bio_sale_lot.py | 21 +++++++++++ .../src/batch/common/__init__.py | 0 .../src/batch/laundering/create_mst_inst.py | 4 +-- .../src/batch/laundering/sales_laundering.py | 4 ++- .../src/batch/parallel_processes.py | 35 +++++++++++++++++++ ecs/jskult-batch-daily/src/jobctrl_daily.py | 18 ++++------ 7 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 ecs/jskult-batch-daily/src/batch/bio_sales/__init__.py create mode 100644 ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py create mode 100644 ecs/jskult-batch-daily/src/batch/common/__init__.py create mode 100644 ecs/jskult-batch-daily/src/batch/parallel_processes.py diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/__init__.py b/ecs/jskult-batch-daily/src/batch/bio_sales/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py new file mode 100644 index 00000000..187afd2d --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py @@ -0,0 +1,21 @@ +from src.batch.common.batch_context import BatchContext +from src.logging.get_logger import get_logger + +batch_context = BatchContext.get_instance() +logger = get_logger('生物由来卸販売ロット分解') + + +def exec(): + """生物由来卸販売ロット分解""" + + logger.info('生物由来卸販売ロット分解：起動') + # 営業日ではない場合、処理をスキップする + if batch_context.is_not_business_day: + logger.info('営業日ではないため、生物由来卸販売ロット分解処理をスキップします。') + return + pass + + logger.info('作成処理') + + logger.info('生物由来卸販売ロット分解：終了') + return diff --git a/ecs/jskult-batch-daily/src/batch/common/__init__.py b/ecs/jskult-batch-daily/src/batch/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py b/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py index 832e430d..45b8ce71 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py @@ -2,11 +2,11 @@ from src.batch.common.batch_context import BatchContext from src.logging.get_logger import get_logger batch_context = BatchContext.get_instance() -logger = get_logger('メルク施設統合マスタ') +logger = get_logger('メルク施設統合マスタ作成') def exec(): - """メルク施設統合マスタ""" + """メルク施設統合マスタ作成""" # 営業日ではない場合、処理をスキップする if batch_context.is_not_business_day: diff --git a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py index 962b3e12..cc414616 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py @@ -9,7 +9,7 @@ logger = get_logger('実績洗替') def exec(): """実績洗替処理""" - + logger.info('実績更新：起動') # 非営業日の場合実績洗替処理は実行しない if batch_context.is_not_business_day: logger.info('非営業日のため、実績洗替処理をスキップします。') @@ -17,3 +17,5 @@ def exec(): # 施設担当者洗替 emp_chg_inst_laundering.exec() + + logger.info('実績更新：終了') \ No newline at end of file diff --git a/ecs/jskult-batch-daily/src/batch/parallel_processes.py b/ecs/jskult-batch-daily/src/batch/parallel_processes.py new file mode 100644 index 00000000..2f42ebbb --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/parallel_processes.py @@ -0,0 +1,35 @@ +"""並行処理""" + +import concurrent.futures + +from src.batch.bio_sales import create_bio_sale_lot +from src.batch.laundering import sales_laundering +from src.error.exceptions import BatchOperationException +from src.logging.get_logger import get_logger + +logger = get_logger('日次処理コントロール(並行処理)') + + +def exec(): + # 並行処理を開始 + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + + # 実績更新 + future_sales_laundering = executor.submit(sales_laundering.exec) + # 生物由来ロット分解 + future_create_bio_sales_lot = executor.submit(create_bio_sale_lot.exec) + + # 両方の処理が完了するまで待つ + concurrent.futures.wait([future_sales_laundering, future_create_bio_sales_lot]) + + # エラーがあれば呼び出し元でキャッチする + sales_laundering_exc = future_sales_laundering.exception() + create_bio_sales_lot_exc = future_create_bio_sales_lot.exception() + + # いずれかにエラーが発生していれば、、１つのエラーとして返す。 + if sales_laundering_exc is not None or create_bio_sales_lot_exc is not None: + sales_laundering_exc_message = str(sales_laundering_exc) if sales_laundering_exc is not None else '' + create_bio_sales_lot_exc_message = str(create_bio_sales_lot_exc) if create_bio_sales_lot_exc is not None else '' + raise BatchOperationException(f'並行処理中にエラーが発生しました。実績更新="{sales_laundering_exc_message}", 生物由来ロット分解={create_bio_sales_lot_exc_message}') + + return diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 0b865b9b..e0be0809 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -1,13 +1,13 @@ """実消化&アルトマーク 日次バッチ処理""" from src.aws.s3 import ConfigBucket +from src.batch import parallel_processes from src.batch.batch_functions import ( get_batch_statuses, update_batch_process_complete, update_batch_processing_flag_in_processing) from src.batch.common.batch_context import BatchContext from src.batch.common.calendar_file import CalendarFile -from src.batch.laundering import (create_dcf_inst_merge, create_mst_inst, - sales_laundering) +from src.batch.laundering import create_dcf_inst_merge, create_mst_inst from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger @@ -88,18 +88,14 @@ def exec(): return constants.BATCH_EXIT_CODE_SUCCESS try: - # TODO: ここで、生物由来ロット分解と並行処理 - logger.info('実績更新：起動') - sales_laundering.exec() - logger.info('生物由来ロット分解：起動') - logger.info('実績更新：終了') - logger.info('生物由来ロット分解：終了') + # 実績生物由来ロット分解と並行処理 + logger.info('並行処理(実績更新-生物由来ロット分解)：起動') + parallel_processes.exec() + logger.info('並行処理(実績更新-生物由来ロット分解)：終了') except BatchOperationException as e: - logger.exception(f'実績更新処理エラー（異常終了）{e}') + logger.exception(f'並行処理(実績更新-生物由来ロット分解)エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS - # TODO: ここで並行処理合流 - try: logger.info('DCF施設統合マスタ作成：起動') create_dcf_inst_merge.exec() From f3badd63efb2b7c0dba3c22214c6fbee6301f559 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 17 Apr 2023 13:34:48 +0900 Subject: [PATCH 10/32] =?UTF-8?q?feat:=20=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/batch/parallel_processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-batch-daily/src/batch/parallel_processes.py b/ecs/jskult-batch-daily/src/batch/parallel_processes.py index 2f42ebbb..39239909 100644 --- a/ecs/jskult-batch-daily/src/batch/parallel_processes.py +++ b/ecs/jskult-batch-daily/src/batch/parallel_processes.py @@ -26,7 +26,7 @@ def exec(): sales_laundering_exc = future_sales_laundering.exception() create_bio_sales_lot_exc = future_create_bio_sales_lot.exception() - # いずれかにエラーが発生していれば、、１つのエラーとして返す。 + # いずれかにエラーが発生していれば、１つのエラーとして返す。 if sales_laundering_exc is not None or create_bio_sales_lot_exc is not None: sales_laundering_exc_message = str(sales_laundering_exc) if sales_laundering_exc is not None else '' create_bio_sales_lot_exc_message = str(create_bio_sales_lot_exc) if create_bio_sales_lot_exc is not None else '' From 32b5dad3b490114daea2b9b29ebf3c74c2f6591b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 17 Apr 2023 13:39:49 +0900 Subject: [PATCH 11/32] =?UTF-8?q?feat:=20=E4=B8=A6=E5=88=97=E5=87=A6?= =?UTF-8?q?=E7=90=86=E5=86=85=E3=81=A7=E3=81=AF=E3=83=AD=E3=82=B0=E4=B8=8D?= =?UTF-8?q?=E8=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/batch/parallel_processes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/parallel_processes.py b/ecs/jskult-batch-daily/src/batch/parallel_processes.py index 39239909..1b7db6c7 100644 --- a/ecs/jskult-batch-daily/src/batch/parallel_processes.py +++ b/ecs/jskult-batch-daily/src/batch/parallel_processes.py @@ -5,9 +5,6 @@ import concurrent.futures from src.batch.bio_sales import create_bio_sale_lot from src.batch.laundering import sales_laundering from src.error.exceptions import BatchOperationException -from src.logging.get_logger import get_logger - -logger = get_logger('日次処理コントロール(並行処理)') def exec(): From 2083a0250d4e4f8bf2c40de51f1ebc8350fa2f3b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 17 Apr 2023 14:22:21 +0900 Subject: [PATCH 12/32] =?UTF-8?q?style:=20=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/bio_sales/create_bio_sale_lot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py index 187afd2d..7d166157 100644 --- a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py @@ -13,9 +13,8 @@ def exec(): if batch_context.is_not_business_day: logger.info('営業日ではないため、生物由来卸販売ロット分解処理をスキップします。') return - pass - logger.info('作成処理') + # TODO: ここに処理を追記していく logger.info('生物由来卸販売ロット分解：終了') return From 86d055a1a3e25ef040ad441aa8763a36211cdc17 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 10:15:36 +0900 Subject: [PATCH 13/32] =?UTF-8?q?feat:=20=E3=83=91=E3=83=83=E3=82=B1?= =?UTF-8?q?=E3=83=BC=E3=82=B8=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/{laundering => institution}/create_dcf_inst_merge.py | 0 .../src/batch/{laundering => institution}/create_mst_inst.py | 0 ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py | 2 +- ecs/jskult-batch-daily/src/jobctrl_daily.py | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename ecs/jskult-batch-daily/src/batch/{laundering => institution}/create_dcf_inst_merge.py (100%) rename ecs/jskult-batch-daily/src/batch/{laundering => institution}/create_mst_inst.py (100%) diff --git a/ecs/jskult-batch-daily/src/batch/laundering/create_dcf_inst_merge.py b/ecs/jskult-batch-daily/src/batch/institution/create_dcf_inst_merge.py similarity index 100% rename from ecs/jskult-batch-daily/src/batch/laundering/create_dcf_inst_merge.py rename to ecs/jskult-batch-daily/src/batch/institution/create_dcf_inst_merge.py diff --git a/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py b/ecs/jskult-batch-daily/src/batch/institution/create_mst_inst.py similarity index 100% rename from ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py rename to ecs/jskult-batch-daily/src/batch/institution/create_mst_inst.py diff --git a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py index cc414616..72f638d4 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py @@ -18,4 +18,4 @@ def exec(): # 施設担当者洗替 emp_chg_inst_laundering.exec() - logger.info('実績更新：終了') \ No newline at end of file + logger.info('実績更新：終了') diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index e0be0809..1c0b8608 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -7,7 +7,7 @@ from src.batch.batch_functions import ( update_batch_processing_flag_in_processing) from src.batch.common.batch_context import BatchContext from src.batch.common.calendar_file import CalendarFile -from src.batch.laundering import create_dcf_inst_merge, create_mst_inst +from src.batch.institution import create_dcf_inst_merge, create_mst_inst from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger From 26db202b5e9644ef38ad5be643832ac6418f2ad0 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 11:10:41 +0900 Subject: [PATCH 14/32] =?UTF-8?q?feat:=20=E6=B4=97=E3=81=84=E6=9B=BF?= =?UTF-8?q?=E3=81=88=E5=87=A6=E7=90=86=E3=81=AE=E4=B8=80=E9=83=A8=E3=81=A7?= =?UTF-8?q?=E3=81=AF=E3=81=82=E3=82=8B=E3=81=9F=E3=82=81=E3=80=81=E3=82=84?= =?UTF-8?q?=E3=81=A3=E3=81=B1=E3=82=8A=E3=82=82=E3=81=A8=E3=81=AB=E6=88=BB?= =?UTF-8?q?=E3=81=97=E3=81=9F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/{institution => laundering}/create_dcf_inst_merge.py | 0 .../src/batch/{institution => laundering}/create_mst_inst.py | 0 ecs/jskult-batch-daily/src/jobctrl_daily.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename ecs/jskult-batch-daily/src/batch/{institution => laundering}/create_dcf_inst_merge.py (100%) rename ecs/jskult-batch-daily/src/batch/{institution => laundering}/create_mst_inst.py (100%) diff --git a/ecs/jskult-batch-daily/src/batch/institution/create_dcf_inst_merge.py b/ecs/jskult-batch-daily/src/batch/laundering/create_dcf_inst_merge.py similarity index 100% rename from ecs/jskult-batch-daily/src/batch/institution/create_dcf_inst_merge.py rename to ecs/jskult-batch-daily/src/batch/laundering/create_dcf_inst_merge.py diff --git a/ecs/jskult-batch-daily/src/batch/institution/create_mst_inst.py b/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py similarity index 100% rename from ecs/jskult-batch-daily/src/batch/institution/create_mst_inst.py rename to ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 1c0b8608..e0be0809 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -7,7 +7,7 @@ from src.batch.batch_functions import ( update_batch_processing_flag_in_processing) from src.batch.common.batch_context import BatchContext from src.batch.common.calendar_file import CalendarFile -from src.batch.institution import create_dcf_inst_merge, create_mst_inst +from src.batch.laundering import create_dcf_inst_merge, create_mst_inst from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger From 31b5d6a6915aea18fdc81cd5e33088ff7127aeb2 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 11:51:36 +0900 Subject: [PATCH 15/32] =?UTF-8?q?feat:=20dump=E6=9C=AA=E5=8F=96=E5=BE=97?= =?UTF-8?q?=E3=81=A8=E3=81=84=E3=81=86=E3=82=B9=E3=83=86=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E3=82=B9=E3=82=92=E7=84=A1=E3=81=8F=E3=81=97=E3=81=9F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/jobctrl_daily.py | 2 +- ecs/jskult-batch-daily/src/system_var/constants.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index e0be0809..b6926e5c 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -35,7 +35,7 @@ def exec(): return constants.BATCH_EXIT_CODE_SUCCESS # dump取得が正常終了していない場合、後続の処理は行わない - if dump_status_kbn not in [constants.DUMP_STATUS_KBN_COMPLETE, constants.DUMP_STATUS_KBN_DUMP_SKIP]: + if dump_status_kbn != constants.DUMP_STATUS_KBN_COMPLETE: logger.error('dump取得が正常終了していないため、日次バッチ処理を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-batch-daily/src/system_var/constants.py b/ecs/jskult-batch-daily/src/system_var/constants.py index 26e1c1a8..8a0ccbb3 100644 --- a/ecs/jskult-batch-daily/src/system_var/constants.py +++ b/ecs/jskult-batch-daily/src/system_var/constants.py @@ -9,8 +9,6 @@ BATCH_ACTF_BATCH_IN_PROCESSING = '1' DUMP_STATUS_KBN_UNPROCESSED = '0' # dump取得状態区分：dump取得正常終了 DUMP_STATUS_KBN_COMPLETE = '2' -# dump取得状態区分：dump取得スキップ -DUMP_STATUS_KBN_DUMP_SKIP = '3' # カレンダーファイルのコメントシンボル CALENDAR_COMMENT_SYMBOL = '#' From 24c0af39c25045f79136f2d057e96f5a28326b3a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 12:02:53 +0900 Subject: [PATCH 16/32] =?UTF-8?q?style:=20=E3=83=AD=E3=82=B0=E3=83=A1?= =?UTF-8?q?=E3=83=83=E3=82=BB=E3=83=BC=E3=82=B8=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/laundering/sales_laundering.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py index 72f638d4..7b77d8ac 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py @@ -10,9 +10,9 @@ logger = get_logger('実績洗替') def exec(): """実績洗替処理""" logger.info('実績更新：起動') - # 非営業日の場合実績洗替処理は実行しない + # 営業日ではない場合、実績洗替処理は実行しない if batch_context.is_not_business_day: - logger.info('非営業日のため、実績洗替処理をスキップします。') + logger.info('営業日ではないため、実績洗替処理をスキップします。') return # 施設担当者洗替 From e0175b3228bb44f537ade7c25fc408203e90f196 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 12:04:30 +0900 Subject: [PATCH 17/32] =?UTF-8?q?feat:=20=E3=83=AD=E3=82=B0=E3=83=A1?= =?UTF-8?q?=E3=83=83=E3=82=BB=E3=83=BC=E3=82=B8=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/jobctrl_daily.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index b6926e5c..ca9b0022 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -105,11 +105,13 @@ def exec(): return constants.BATCH_EXIT_CODE_SUCCESS # バッチ処理完了とし、処理日、バッチ処置中フラグ、dump取得状態区分を更新 + logger.info('業務日付更新・バッチステータスリフレッシュ：起動') try: update_batch_process_complete() except BatchOperationException as e: - logger.exception(f'日次バッチ完了処理 エラー（異常終了）{e}') + logger.exception(f'業務日付更新・バッチステータスリフレッシュ エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS + logger.info('業務日付更新・バッチステータスリフレッシュ：終了') # 正常終了を保守ユーザーに通知 logger.info('[NOTICE]日次バッチ：終了（正常終了）') From f0b91c6a748ca21449ce8fa2e198d206506dc01e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 12:22:45 +0900 Subject: [PATCH 18/32] =?UTF-8?q?fix:=20=E3=83=A1=E3=83=AB=E3=82=AF?= =?UTF-8?q?=E6=96=BD=E8=A8=AD=E3=83=9E=E3=82=B9=E3=82=BF=E4=BD=9C=E6=88=90?= =?UTF-8?q?=E5=87=A6=E7=90=86=E8=B5=B7=E5=8B=95=E5=88=A4=E5=AE=9A=E3=82=92?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/laundering/create_mst_inst.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py b/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py index 45b8ce71..6b499a71 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/create_mst_inst.py @@ -2,20 +2,15 @@ from src.batch.common.batch_context import BatchContext from src.logging.get_logger import get_logger batch_context = BatchContext.get_instance() -logger = get_logger('メルク施設統合マスタ作成') +logger = get_logger('メルク施設マスタ作成') def exec(): - """メルク施設統合マスタ作成""" + """メルク施設マスタ作成""" - # 営業日ではない場合、処理をスキップする - if batch_context.is_not_business_day: - logger.info('営業日ではないため、メルク施設統合マスタ作成処理をスキップします。') - return - - # アルトマーク取込が行われていない場合は処理をスキップする - if not batch_context.is_ultmarc_imported: - logger.info('アルトマーク取込が行われていないため、メルク施設統合マスタ作成処理をスキップします。') + # 営業日ではないかつ、アルトマーク取込が行われていない場合は処理をスキップする + if batch_context.is_not_business_day is True and batch_context.is_ultmarc_imported is not True: + logger.info('営業日ではない、かつ、アルトマーク取込が行われていないため、メルク施設マスタ作成処理をスキップします。') return pass From 4431e5bbaa906aba5bf68f465570648699b691c4 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 16:51:22 +0900 Subject: [PATCH 19/32] =?UTF-8?q?fix:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=8F=8D=E6=98=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit feat: レビュー指摘対応 db.disconnectのタイミング修正 https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/161#comment-1843452 feat: レビュー指摘反映に伴い、検査例外を出す部分は一箇所に限定 https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/161#comment-1843452 feat: レビュー指摘対応 https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/161#comment-1843505 https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/161#comment-1843511 feat: レビュー指摘反映 https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/161#comment-1843529 feat: ログ出力レベルの調整。内部関数に_をつけた --- .../src/batch/batch_functions.py | 9 +- .../batch/bio_sales/create_bio_sale_lot.py | 20 ---- .../laundering/emp_chg_inst_laundering.py | 93 ++++++++++--------- .../src/batch/ultmarc/ultmarc_process.py | 78 ++++++++-------- ecs/jskult-batch-daily/src/jobctrl_daily.py | 18 +++- 5 files changed, 110 insertions(+), 108 deletions(-) delete mode 100644 ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py diff --git a/ecs/jskult-batch-daily/src/batch/batch_functions.py b/ecs/jskult-batch-daily/src/batch/batch_functions.py index 6c113664..b8e223e2 100644 --- a/ecs/jskult-batch-daily/src/batch/batch_functions.py +++ b/ecs/jskult-batch-daily/src/batch/batch_functions.py @@ -27,7 +27,8 @@ def get_batch_statuses() -> tuple[str, str, str]: hdke_tbl_result = db.execute_select(sql) except DBException as e: raise BatchOperationException(e) - db.disconnect() + finally: + db.disconnect() if len(hdke_tbl_result) == 0: raise BatchOperationException('日付テーブルが取得できませんでした') @@ -56,7 +57,8 @@ def update_batch_processing_flag_in_processing() -> None: db.execute(sql, {'in_processing': constants.BATCH_ACTF_BATCH_IN_PROCESSING}) except DBException as e: raise BatchOperationException(e) - db.disconnect() + finally: + db.disconnect() return @@ -83,7 +85,8 @@ def update_batch_process_complete() -> None: }) except DBException as e: raise BatchOperationException(e) - db.disconnect() + finally: + db.disconnect() return diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py deleted file mode 100644 index 7d166157..00000000 --- a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py +++ /dev/null @@ -1,20 +0,0 @@ -from src.batch.common.batch_context import BatchContext -from src.logging.get_logger import get_logger - -batch_context = BatchContext.get_instance() -logger = get_logger('生物由来卸販売ロット分解') - - -def exec(): - """生物由来卸販売ロット分解""" - - logger.info('生物由来卸販売ロット分解：起動') - # 営業日ではない場合、処理をスキップする - if batch_context.is_not_business_day: - logger.info('営業日ではないため、生物由来卸販売ロット分解処理をスキップします。') - return - - # TODO: ここに処理を追記していく - - logger.info('生物由来卸販売ロット分解：終了') - return diff --git a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py index 6e611bc9..16ff0c79 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py @@ -11,38 +11,42 @@ batch_context = BatchContext.get_instance() def exec(): db = Database.get_instance() - db.connect() - logger.info('##########################') - logger.info('START Changing Employee in charge of institution PGM.') - # 業務日付を取得 - syor_date = batch_context.syor_date - # `emp_chg_inst_lau`をTruncate - truncate_emp_chg_inst_lau(db) - # emp_chg_inst から、`emp_chg_inst_lau`へInsert - insert_into_emp_chg_inst_lau_from_emp_chg_inst(db) - # vop_hco_merge_vから、emp_chg_inst_lauをUpdate - update_emp_chg_inst_lau_from_vop_hco_merge_v(db, syor_date) - # dcf_inst_mergeから、emp_chg_inst_lauをUpdate - update_dcf_inst_merge_from_emp_chg_inst_lau(db, syor_date) - db.disconnect() - logger.info('##########################') - logger.info('End All Processing PGM.') + try: + db.connect() + logger.debug('##########################') + logger.debug('START Changing Employee in charge of institution PGM.') + # 業務日付を取得 + syor_date = batch_context.syor_date + # `emp_chg_inst_lau`をTruncate + _truncate_emp_chg_inst_lau(db) + # emp_chg_inst から、`emp_chg_inst_lau`へInsert + _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db) + # vop_hco_merge_vから、emp_chg_inst_lauをUpdate + _update_emp_chg_inst_lau_from_vop_hco_merge_v(db, syor_date) + # dcf_inst_mergeから、emp_chg_inst_lauをUpdate + _update_dcf_inst_merge_from_emp_chg_inst_lau(db, syor_date) + logger.debug('##########################') + logger.debug('End All Processing PGM.') + except Exception as e: + raise BatchOperationException(e) + finally: + db.disconnect() -def truncate_emp_chg_inst_lau(db: Database): - logger.info("##########################") +def _truncate_emp_chg_inst_lau(db: Database): + logger.debug("##########################") try: db.execute("TRUNCATE TABLE src05.emp_chg_inst_lau") except Exception as e: - logger.info("Error! Truncate Table `emp_chg_inst_lau` is Failed!!!") - raise BatchOperationException(e) + logger.debug("Error! Truncate Table `emp_chg_inst_lau` is Failed!!!") + raise e - logger.info("Table `emp_chg_inst_lau` was truncated!") + logger.debug("Table `emp_chg_inst_lau` was truncated!") return -def insert_into_emp_chg_inst_lau_from_emp_chg_inst(db: Database): - logger.info("##########################") +def _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db: Database): + logger.debug("##########################") try: elapsed_time = ElapsedTime() sql = """ @@ -70,16 +74,16 @@ def insert_into_emp_chg_inst_lau_from_emp_chg_inst(db: Database): logging_sql(logger, sql) logger.info(f'Query OK, {res.rowcount} rows affected ({elapsed_time.of})') except Exception as e: - logger.info("Error! Insert into `emp_chg_inst_lau` from `emp_chg_inst` was failed!!!") - raise BatchOperationException(e) - logger.info("Success! Insert into `emp_chg_inst_lau` from `emp_chg_inst` was inserted!") + logger.debug("Error! Insert into `emp_chg_inst_lau` from `emp_chg_inst` was failed!!!") + raise e + logger.debug("Success! Insert into `emp_chg_inst_lau` from `emp_chg_inst` was inserted!") return -def update_emp_chg_inst_lau_from_vop_hco_merge_v(db: Database, syor_date: str): +def _update_emp_chg_inst_lau_from_vop_hco_merge_v(db: Database, syor_date: str): # vop_hco_merge_vはデータが作られないため、この洗い替え処理は基本空振りする - logger.info("##########################") + logger.debug("##########################") try: select_result = db.execute_select( """ @@ -93,8 +97,8 @@ def update_emp_chg_inst_lau_from_vop_hco_merge_v(db: Database, syor_date: str): {'syor_date': syor_date} ) except Exception as e: - logger.info("Error! `vop_hco_merge_v` Table count error!") - raise BatchOperationException(e) + logger.debug("Error! `vop_hco_merge_v` Table count error!") + raise e count = [row for row in select_result][0]['row_count'] if count == 0: logger.info('vop_hco_merge_v Table Data is not exists!') @@ -137,16 +141,16 @@ def update_emp_chg_inst_lau_from_vop_hco_merge_v(db: Database, syor_date: str): logging_sql(logger, update_sql) logger.info(f'Query OK, {update_result.rowcount} rows affected ({elapsed_time.of})') except Exception as e: - logger.info(f"emp_chg_inst_lau v_inst_cd could not set from {v_inst_cd_merge} to {v_inst_cd_merge}!") - raise BatchOperationException(e) - logger.info(f"Success! emp_chg_inst_lau v_inst_cd was set from {v_inst_cd} to {v_inst_cd_merge}!") + logger.debug(f"emp_chg_inst_lau v_inst_cd could not set from {v_inst_cd_merge} to {v_inst_cd_merge}!") + raise e + logger.debug(f"Success! emp_chg_inst_lau v_inst_cd was set from {v_inst_cd} to {v_inst_cd_merge}!") return -def update_dcf_inst_merge_from_emp_chg_inst_lau(db: Database, syor_date: str): +def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: Database, syor_date: str): # dcf_inst_mergeから、emp_chg_inst_lauをUpdate # Get count from DCF_INST_MERGE - logger.info("##########################") + logger.debug("##########################") try: select_result = db.execute_select( """ @@ -163,18 +167,19 @@ def update_dcf_inst_merge_from_emp_chg_inst_lau(db: Database, syor_date: str): {'syor_date': syor_date} ) except Exception as e: - logger.info("Error! Getting Count of dcf_inst_merge was failed!") - raise BatchOperationException(e) + logger.debug("Error! Getting Count of dcf_inst_merge was failed!") + raise e + count = [row for row in select_result][0]['row_count'] if count == 0: logger.info('dcf_inst_merge Table Data is not exists!') return - logger.info('dcf_inst_merge Table Data is exists!') + logger.debug('dcf_inst_merge Table Data is exists!') # dcf_inst_mergeから、emp_chg_inst_lauをUpdate - logger.info("##########################") - logger.info("#### UPDATE DATA #########") - logger.info("##########################") + logger.debug("##########################") + logger.debug("#### UPDATE DATA #########") + logger.debug("##########################") try: elapsed_time = ElapsedTime() update_sql = """ @@ -205,9 +210,9 @@ def update_dcf_inst_merge_from_emp_chg_inst_lau(db: Database, syor_date: str): logging_sql(logger, update_sql) logger.info(f'Query OK, {res.rowcount} rows affected ({elapsed_time.of})') except Exception as e: - logger.info("emp_chg_inst_lau.v_inst_cd could not set!") - raise BatchOperationException(e) + logger.debug("emp_chg_inst_lau.v_inst_cd could not set!") + raise e - logger.info("emp_chg_inst_lau.v_inst_cd was set!") + logger.debug("emp_chg_inst_lau.v_inst_cd was set!") return diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py index 8ddae44a..0fde5cf7 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py @@ -61,7 +61,6 @@ def exec_import(): batch_context.is_ultmarc_imported = True logger.info('アルトマーク取込処理: 終了') except Exception as e: - logger.exception(e) raise BatchOperationException(e) @@ -74,41 +73,46 @@ def exec_export(): def _import_to_ultmarc_table(dat_file: DatFile): db = Database.get_instance() - # DB接続 - db.connect() - # ファイル単位でトランザクションを行う - db.begin() - logger.info('Transaction BEGIN') - mapper_factory = UltmarcTableMapperFactory() - # datファイルを1行ずつ処理し、各テーブルへ登録 - for line in dat_file: - try: - # 書き込み先のテーブルを特定 - mapper_class = mapper_factory.create( - line.layout_class, - line.records, - db - ) - mapper_class.make_query() - mapper_class.execute_queries() - dat_file.count_up_success() - except Exception as e: - logger.warning(e) - record = line.records - log_message = ','.join([f'"{r}"' for r in record]) - logger.warning(f'ERROR_LINE: {log_message}') - dat_file.count_up_error() - # すべての行を登録終えたらコミットする - db.commit() - db.disconnect() - # 処理結果をログに出力する - logger.info('Transaction COMMIT') - logger.info(f'ultmarc import process RESULT') - logger.info(f'SUCCESS_COUNT={dat_file.success_count}') - logger.info(f'ERROR_COUNT={dat_file.error_count}') - logger.info(f'ALL_COUNT={dat_file.total_count}') + try: + # DB接続 + db.connect() + # ファイル単位でトランザクションを行う + db.begin() + logger.info('Transaction BEGIN') + mapper_factory = UltmarcTableMapperFactory() + # datファイルを1行ずつ処理し、各テーブルへ登録 + for line in dat_file: + try: + # 書き込み先のテーブルを特定 + mapper_class = mapper_factory.create( + line.layout_class, + line.records, + db + ) + mapper_class.make_query() + mapper_class.execute_queries() + dat_file.count_up_success() + except Exception as e: + logger.warning(e) + record = line.records + log_message = ','.join([f'"{r}"' for r in record]) + logger.warning(f'ERROR_LINE: {log_message}') + dat_file.count_up_error() + # すべての行を登録終えたらコミットする + db.commit() + db.disconnect() + # 処理結果をログに出力する + logger.info('Transaction COMMIT') + logger.info(f'ultmarc import process RESULT') + logger.info(f'SUCCESS_COUNT={dat_file.success_count}') + logger.info(f'ERROR_COUNT={dat_file.error_count}') + logger.info(f'ALL_COUNT={dat_file.total_count}') - # 1件でもエラーがあれば、通知用にログに出力する - if dat_file.error_count > 0: - logger.warning('取り込みに失敗した行があります。詳細は`ERROR_LINE:`の行を確認してください。') + # 1件でもエラーがあれば、通知用にログに出力する + if dat_file.error_count > 0: + logger.warning('取り込みに失敗した行があります。詳細は`ERROR_LINE:`の行を確認してください。') + except Exception as e: + raise e + finally: + db.disconnect() return diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index ca9b0022..3788d368 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -43,10 +43,17 @@ def exec(): # バッチ共通設定に処理日を追加 batch_context.syor_date = syor_date - # 稼働日かかどうかを、非営業日ファイルをダウンロードして判定 - holiday_list_file_path = ConfigBucket().download_holiday_list() - holiday_calendar = CalendarFile(holiday_list_file_path) - batch_context.is_not_business_day = holiday_calendar.compare_date(syor_date) + # 稼働日かかどうかを、V実消化非稼働日ファイルをダウンロードして判定 + try: + holiday_list_file_path = ConfigBucket().download_holiday_list() + holiday_calendar = CalendarFile(holiday_list_file_path) + batch_context.is_not_business_day = holiday_calendar.compare_date(syor_date) + except Exception as e: + logger.exception(f'V実消化非稼働日ファイルの読み込みに失敗しました。{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # 調査目的でV実消化稼働日かどうかをログ出力 + logger.debug(f'本日は{"V実消化非稼働日です。" if batch_context.is_not_business_day else "V実消化稼働日です。"}') # バッチ処理中に更新 try: @@ -63,6 +70,9 @@ def exec(): logger.exception(f'アルトマーク取込処理エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS + # 調査目的でアルトマーク取込が行われたかどうかをログ出力 + logger.debug(f'{"アルトマーク取込が行われました。" if batch_context.is_ultmarc_imported else "アルトマーク取込が行われませんでした。"}') + try: logger.info('V実消化用施設・薬局薬店データ作成処理：起動') ultmarc_process.exec_export() From acdedf13afb189a7ea397ca3618b94c94766d1b1 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 18:19:10 +0900 Subject: [PATCH 20/32] =?UTF-8?q?feat:=20=E8=AA=A4=E3=81=A3=E3=81=A6?= =?UTF-8?q?=E5=89=8A=E9=99=A4=E3=81=97=E3=81=9F=E3=83=A2=E3=82=B8=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E3=83=AB=E3=82=92=E5=BE=A9=E6=B4=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/bio_sales/create_bio_sale_lot.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py new file mode 100644 index 00000000..7d166157 --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py @@ -0,0 +1,20 @@ +from src.batch.common.batch_context import BatchContext +from src.logging.get_logger import get_logger + +batch_context = BatchContext.get_instance() +logger = get_logger('生物由来卸販売ロット分解') + + +def exec(): + """生物由来卸販売ロット分解""" + + logger.info('生物由来卸販売ロット分解：起動') + # 営業日ではない場合、処理をスキップする + if batch_context.is_not_business_day: + logger.info('営業日ではないため、生物由来卸販売ロット分解処理をスキップします。') + return + + # TODO: ここに処理を追記していく + + logger.info('生物由来卸販売ロット分解：終了') + return From 5f508ad1238794a89d2845b672c9b8810daab8a8 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 18:26:43 +0900 Subject: [PATCH 21/32] =?UTF-8?q?fix:=20disconnect=E3=80=81commit=E3=82=92?= =?UTF-8?q?=E5=91=BC=E3=81=B6=E4=BD=8D=E7=BD=AE=E3=81=8C=E3=81=8A=E3=81=8B?= =?UTF-8?q?=E3=81=97=E3=81=84=E3=81=AE=E3=81=A7=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/ultmarc/ultmarc_process.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py index 0fde5cf7..83433bda 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py @@ -98,11 +98,7 @@ def _import_to_ultmarc_table(dat_file: DatFile): log_message = ','.join([f'"{r}"' for r in record]) logger.warning(f'ERROR_LINE: {log_message}') dat_file.count_up_error() - # すべての行を登録終えたらコミットする - db.commit() - db.disconnect() # 処理結果をログに出力する - logger.info('Transaction COMMIT') logger.info(f'ultmarc import process RESULT') logger.info(f'SUCCESS_COUNT={dat_file.success_count}') logger.info(f'ERROR_COUNT={dat_file.error_count}') @@ -111,8 +107,8 @@ def _import_to_ultmarc_table(dat_file: DatFile): # 1件でもエラーがあれば、通知用にログに出力する if dat_file.error_count > 0: logger.warning('取り込みに失敗した行があります。詳細は`ERROR_LINE:`の行を確認してください。') - except Exception as e: - raise e finally: + # 終了時に必ずコミットする + db.commit() db.disconnect() return From 0491aa77be9b1c0a7121ccd680c50baed2d3b882 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 19:00:26 +0900 Subject: [PATCH 22/32] =?UTF-8?q?feat:=20=E3=82=A2=E3=83=AB=E3=83=88?= =?UTF-8?q?=E3=83=9E=E3=83=BC=E3=82=AF=E5=87=A6=E7=90=86=E3=83=AD=E3=82=B0?= =?UTF-8?q?=E8=A6=8B=E7=9B=B4=E3=81=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py index 83433bda..4ed7ae03 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py @@ -34,8 +34,7 @@ def exec_import(): # ファイルが複数ある場合はエラーとする if len(dat_file_list) > 1: - logger.error(f'複数の取込ファイルがあるため、異常終了 ファイル一覧:{dat_file_list}') - raise BatchOperationException() + raise BatchOperationException(f'複数の取込ファイルがあるため、異常終了 ファイル一覧:{dat_file_list}') # ファイルの件数は必ず1件になる dat_file_info = dat_file_list[0] @@ -78,7 +77,6 @@ def _import_to_ultmarc_table(dat_file: DatFile): db.connect() # ファイル単位でトランザクションを行う db.begin() - logger.info('Transaction BEGIN') mapper_factory = UltmarcTableMapperFactory() # datファイルを1行ずつ処理し、各テーブルへ登録 for line in dat_file: @@ -96,7 +94,7 @@ def _import_to_ultmarc_table(dat_file: DatFile): logger.warning(e) record = line.records log_message = ','.join([f'"{r}"' for r in record]) - logger.warning(f'ERROR_LINE: {log_message}') + logger.info(f'ERROR_LINE: {log_message}') dat_file.count_up_error() # 処理結果をログに出力する logger.info(f'ultmarc import process RESULT') From e27757e3ee7ef20a218a354549bef89737660bae Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 19:02:17 +0900 Subject: [PATCH 23/32] =?UTF-8?q?feat:=20=E4=B8=A6=E8=A1=8C=E5=87=A6?= =?UTF-8?q?=E7=90=86=E8=A6=8B=E7=9B=B4=E3=81=97=E3=80=82ECS=20=E3=82=BF?= =?UTF-8?q?=E3=82=B9=E3=82=AF=E3=81=AEvCPU=E6=95=B0=E6=9C=80=E9=81=A9?= =?UTF-8?q?=E5=8C=96=E3=81=AE=E3=81=9F=E3=82=81=E3=80=81async-await?= =?UTF-8?q?=E3=81=A7=E4=B8=A6=E5=88=97=E5=87=A6=E7=90=86=E3=82=92=E5=AE=9F?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/entrypoint.py | 15 ++++++-- ...io_sale_lot.py => create_bio_sales_lot.py} | 2 +- .../laundering/emp_chg_inst_laundering.py | 2 +- .../src/batch/laundering/sales_laundering.py | 8 ++--- .../src/batch/parallel_processes.py | 32 ----------------- .../src/batch/parallel_tasks.py | 34 +++++++++++++++++++ ecs/jskult-batch-daily/src/jobctrl_daily.py | 8 +++-- 7 files changed, 58 insertions(+), 43 deletions(-) rename ecs/jskult-batch-daily/src/batch/bio_sales/{create_bio_sale_lot.py => create_bio_sales_lot.py} (97%) delete mode 100644 ecs/jskult-batch-daily/src/batch/parallel_processes.py create mode 100644 ecs/jskult-batch-daily/src/batch/parallel_tasks.py diff --git a/ecs/jskult-batch-daily/entrypoint.py b/ecs/jskult-batch-daily/entrypoint.py index 472efd9f..5b06ca48 100644 --- a/ecs/jskult-batch-daily/entrypoint.py +++ b/ecs/jskult-batch-daily/entrypoint.py @@ -1,10 +1,21 @@ """実消化&アルトマーク 日次バッチのエントリーポイント""" +import asyncio + from src import jobctrl_daily -if __name__ == '__main__': + +# 一部並行実行のため、非同期関数化 +async def run(): try: - exit(jobctrl_daily.exec()) + task = asyncio.create_task(jobctrl_daily.exec()) + result_code = await task + exit(result_code) except Exception: # エラーが起きても、正常系のコードで返す。 # エラーが起きた事実はbatch_process内でログを出す。 exit(0) + + +if __name__ == '__main__': + # 非同期関数を実行 + asyncio.run(run()) diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py similarity index 97% rename from ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py rename to ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py index 7d166157..02080d76 100644 --- a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py @@ -5,7 +5,7 @@ batch_context = BatchContext.get_instance() logger = get_logger('生物由来卸販売ロット分解') -def exec(): +async def exec(): """生物由来卸販売ロット分解""" logger.info('生物由来卸販売ロット分解：起動') diff --git a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py index 16ff0c79..57ef56be 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py @@ -9,7 +9,7 @@ logger = get_logger('48-施設担当者マスタ洗替') batch_context = BatchContext.get_instance() -def exec(): +async def exec(): db = Database.get_instance() try: db.connect() diff --git a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py index 7b77d8ac..3265440f 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py @@ -7,15 +7,15 @@ batch_context = BatchContext.get_instance() logger = get_logger('実績洗替') -def exec(): +async def exec(): """実績洗替処理""" - logger.info('実績更新：起動') + logger.debug('実績更新：起動') # 営業日ではない場合、実績洗替処理は実行しない if batch_context.is_not_business_day: logger.info('営業日ではないため、実績洗替処理をスキップします。') return # 施設担当者洗替 - emp_chg_inst_laundering.exec() + await emp_chg_inst_laundering.exec() - logger.info('実績更新：終了') + logger.debug('実績更新：終了') diff --git a/ecs/jskult-batch-daily/src/batch/parallel_processes.py b/ecs/jskult-batch-daily/src/batch/parallel_processes.py deleted file mode 100644 index 1b7db6c7..00000000 --- a/ecs/jskult-batch-daily/src/batch/parallel_processes.py +++ /dev/null @@ -1,32 +0,0 @@ -"""並行処理""" - -import concurrent.futures - -from src.batch.bio_sales import create_bio_sale_lot -from src.batch.laundering import sales_laundering -from src.error.exceptions import BatchOperationException - - -def exec(): - # 並行処理を開始 - with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: - - # 実績更新 - future_sales_laundering = executor.submit(sales_laundering.exec) - # 生物由来ロット分解 - future_create_bio_sales_lot = executor.submit(create_bio_sale_lot.exec) - - # 両方の処理が完了するまで待つ - concurrent.futures.wait([future_sales_laundering, future_create_bio_sales_lot]) - - # エラーがあれば呼び出し元でキャッチする - sales_laundering_exc = future_sales_laundering.exception() - create_bio_sales_lot_exc = future_create_bio_sales_lot.exception() - - # いずれかにエラーが発生していれば、１つのエラーとして返す。 - if sales_laundering_exc is not None or create_bio_sales_lot_exc is not None: - sales_laundering_exc_message = str(sales_laundering_exc) if sales_laundering_exc is not None else '' - create_bio_sales_lot_exc_message = str(create_bio_sales_lot_exc) if create_bio_sales_lot_exc is not None else '' - raise BatchOperationException(f'並行処理中にエラーが発生しました。実績更新="{sales_laundering_exc_message}", 生物由来ロット分解={create_bio_sales_lot_exc_message}') - - return diff --git a/ecs/jskult-batch-daily/src/batch/parallel_tasks.py b/ecs/jskult-batch-daily/src/batch/parallel_tasks.py new file mode 100644 index 00000000..afb58820 --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/parallel_tasks.py @@ -0,0 +1,34 @@ +"""並行処理""" + +import asyncio + +from src.batch.bio_sales import create_bio_sales_lot +from src.batch.laundering import sales_laundering +from src.error.exceptions import BatchOperationException + + +async def exec(): + tasks = { + '実績更新': sales_laundering.exec(), + '生物由来ロット分解': create_bio_sales_lot.exec() + } + + # 並行処理を開始 + # 両方の処理が完了するまで待つ + done, _ = await asyncio.wait(tasks.values(), return_when=asyncio.ALL_COMPLETED) + task_results = {} + # 実行結果を確認する + for task_key, task in zip(tasks.keys(), done): + try: + task.result() + task_results[task_key] = True + except Exception as e: + task_results[task_key] = str(e) + + # エラーが発生しているかを確認し、エラーがあれば例外とする + if_error_tasks = {k: v for k, v in task_results.items() if v is not True} + if len(if_error_tasks.keys()) != 0: + messages = ', '.join([f'{k}={v}' for k, v in if_error_tasks.items()]) + raise BatchOperationException(f'並行処理中にエラーが発生しました。{messages}') + + return diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 3788d368..5c178220 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -1,7 +1,7 @@ """実消化&アルトマーク 日次バッチ処理""" from src.aws.s3 import ConfigBucket -from src.batch import parallel_processes +from src.batch import parallel_tasks from src.batch.batch_functions import ( get_batch_statuses, update_batch_process_complete, update_batch_processing_flag_in_processing) @@ -19,7 +19,7 @@ logger = get_logger('日次処理コントロール') batch_context = BatchContext.get_instance() -def exec(): +async def exec(): try: logger.info('日次バッチ：開始') try: @@ -100,7 +100,7 @@ def exec(): try: # 実績生物由来ロット分解と並行処理 logger.info('並行処理(実績更新-生物由来ロット分解)：起動') - parallel_processes.exec() + await parallel_tasks.exec() logger.info('並行処理(実績更新-生物由来ロット分解)：終了') except BatchOperationException as e: logger.exception(f'並行処理(実績更新-生物由来ロット分解)エラー（異常終了）{e}') @@ -130,3 +130,5 @@ def exec(): except Exception as e: logger.exception(f'日次バッチ処理中に想定外のエラーが発生しました {e}') raise e + finally: + logger.info('日次バッチ：終了') From bdcb02867efce92e30027475af4362c79adccebc Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 19:14:36 +0900 Subject: [PATCH 24/32] =?UTF-8?q?feat:=20=E3=83=AD=E3=82=B0=E6=96=87?= =?UTF-8?q?=E8=A8=80=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/bio_sales/create_bio_sales_lot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py index 02080d76..be9f2164 100644 --- a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py @@ -7,8 +7,7 @@ logger = get_logger('生物由来卸販売ロット分解') async def exec(): """生物由来卸販売ロット分解""" - - logger.info('生物由来卸販売ロット分解：起動') + logger.debug('生物由来卸販売ロット分解：起動') # 営業日ではない場合、処理をスキップする if batch_context.is_not_business_day: logger.info('営業日ではないため、生物由来卸販売ロット分解処理をスキップします。') @@ -16,5 +15,5 @@ async def exec(): # TODO: ここに処理を追記していく - logger.info('生物由来卸販売ロット分解：終了') + logger.debug('生物由来卸販売ロット分解：終了') return From e40b89206e21e0c77044cc950fc8c74679f51bce Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 19:41:46 +0900 Subject: [PATCH 25/32] =?UTF-8?q?feat:=20=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E5=90=8D=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/.env.example | 2 +- ecs/jskult-batch-daily/src/aws/s3.py | 4 ++-- ecs/jskult-batch-daily/src/system_var/environment.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ecs/jskult-batch-daily/.env.example b/ecs/jskult-batch-daily/.env.example index a7c88968..6df005c8 100644 --- a/ecs/jskult-batch-daily/.env.example +++ b/ecs/jskult-batch-daily/.env.example @@ -10,4 +10,4 @@ ULTMARC_BACKUP_BUCKET=**************** ULTMARC_BACKUP_FOLDER=ultmarc JSKULT_CONFIG_BUCKET=********************** JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar -JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME=jskult_holiday_list.txt +JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME=jskult_holiday_list.txt diff --git a/ecs/jskult-batch-daily/src/aws/s3.py b/ecs/jskult-batch-daily/src/aws/s3.py index aa235224..9927036c 100644 --- a/ecs/jskult-batch-daily/src/aws/s3.py +++ b/ecs/jskult-batch-daily/src/aws/s3.py @@ -82,8 +82,8 @@ class ConfigBucket(S3Bucket): def download_holiday_list(self): # 一時ファイルとして保存する temporary_dir = tempfile.mkdtemp() - temporary_file_path = path.join(temporary_dir, environment.JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME) - holiday_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME}' + temporary_file_path = path.join(temporary_dir, environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME) + holiday_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME}' with open(temporary_file_path, mode='wb') as f: self._s3_client.download_file(self._bucket_name, holiday_list_key, f) f.seek(0) diff --git a/ecs/jskult-batch-daily/src/system_var/environment.py b/ecs/jskult-batch-daily/src/system_var/environment.py index 650826ee..3212de18 100644 --- a/ecs/jskult-batch-daily/src/system_var/environment.py +++ b/ecs/jskult-batch-daily/src/system_var/environment.py @@ -14,7 +14,7 @@ ULTMARC_BACKUP_BUCKET = os.environ['ULTMARC_BACKUP_BUCKET'] ULTMARC_BACKUP_FOLDER = os.environ['ULTMARC_BACKUP_FOLDER'] JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] JSKULT_CONFIG_CALENDAR_FOLDER = os.environ['JSKULT_CONFIG_CALENDAR_FOLDER'] -JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME = os.environ['JSKULT_CONFIG_HOLIDAY_LIST_FILE_NAME'] +JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME = os.environ['JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME'] # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') From 243373e647e24d2574df8ab5c41ed747a671d144 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Apr 2023 20:58:44 +0900 Subject: [PATCH 26/32] =?UTF-8?q?fix:=20=E3=83=90=E3=82=AF=E3=82=A2?= =?UTF-8?q?=E3=83=83=E3=83=97=E3=83=90=E3=82=B1=E3=83=83=E3=83=88=E3=82=92?= =?UTF-8?q?=E7=A4=BA=E3=81=99=E7=92=B0=E5=A2=83=E5=A4=89=E6=95=B0=E5=90=8D?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/.env.example | 2 +- ecs/jskult-batch-daily/src/aws/s3.py | 2 +- ecs/jskult-batch-daily/src/system_var/environment.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ecs/jskult-batch-daily/.env.example b/ecs/jskult-batch-daily/.env.example index 6df005c8..95aef7fe 100644 --- a/ecs/jskult-batch-daily/.env.example +++ b/ecs/jskult-batch-daily/.env.example @@ -6,7 +6,7 @@ DB_SCHEMA=src05 LOG_LEVEL=INFO ULTMARC_DATA_BUCKET=**************** ULTMARC_DATA_FOLDER=recv -ULTMARC_BACKUP_BUCKET=**************** +JSKULT_BACKUP_BUCKET=**************** ULTMARC_BACKUP_FOLDER=ultmarc JSKULT_CONFIG_BUCKET=********************** JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar diff --git a/ecs/jskult-batch-daily/src/aws/s3.py b/ecs/jskult-batch-daily/src/aws/s3.py index 9927036c..2ac3efe6 100644 --- a/ecs/jskult-batch-daily/src/aws/s3.py +++ b/ecs/jskult-batch-daily/src/aws/s3.py @@ -91,7 +91,7 @@ class ConfigBucket(S3Bucket): class JskUltBackupBucket(S3Bucket): - _bucket_name = environment.ULTMARC_BACKUP_BUCKET + _bucket_name = environment.JSKULT_BACKUP_BUCKET class UltmarcBackupBucket(JskUltBackupBucket): diff --git a/ecs/jskult-batch-daily/src/system_var/environment.py b/ecs/jskult-batch-daily/src/system_var/environment.py index 3212de18..b1730224 100644 --- a/ecs/jskult-batch-daily/src/system_var/environment.py +++ b/ecs/jskult-batch-daily/src/system_var/environment.py @@ -10,7 +10,7 @@ DB_SCHEMA = os.environ['DB_SCHEMA'] # AWS ULTMARC_DATA_BUCKET = os.environ['ULTMARC_DATA_BUCKET'] ULTMARC_DATA_FOLDER = os.environ['ULTMARC_DATA_FOLDER'] -ULTMARC_BACKUP_BUCKET = os.environ['ULTMARC_BACKUP_BUCKET'] +JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] ULTMARC_BACKUP_FOLDER = os.environ['ULTMARC_BACKUP_FOLDER'] JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] JSKULT_CONFIG_CALENDAR_FOLDER = os.environ['JSKULT_CONFIG_CALENDAR_FOLDER'] From d8b6dd9978f72eb227e4b4e2fa15ae871691bd71 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 19 Apr 2023 13:18:12 +0900 Subject: [PATCH 27/32] =?UTF-8?q?feat:=20async-await=E3=81=A7=E9=9D=9E?= =?UTF-8?q?=E5=90=8C=E6=9C=9F=E5=87=A6=E7=90=86=E3=82=92=E5=88=B6=E5=BE=A1?= =?UTF-8?q?=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E3=81=97?= =?UTF-8?q?=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/bio_sales/create_bio_sales_lot.py | 5 ++ .../laundering/emp_chg_inst_laundering.py | 43 +++++++++-------- .../src/db/async_database.py | 48 +++++++++++++++++++ 3 files changed, 75 insertions(+), 21 deletions(-) create mode 100644 ecs/jskult-batch-daily/src/db/async_database.py diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py index be9f2164..53b6a2c4 100644 --- a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py @@ -13,6 +13,11 @@ async def exec(): logger.info('営業日ではないため、生物由来卸販売ロット分解処理をスキップします。') return + # 並列処理の確認用コード + # import asyncio + # for i in range(50): + # logger.debug('生物由来卸販売ロット分解：処理中') + # await asyncio.sleep(0.5) # TODO: ここに処理を追記していく logger.debug('生物由来卸販売ロット分解：終了') diff --git a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py index 57ef56be..9cd5ba07 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py @@ -1,51 +1,52 @@ from src.batch.batch_functions import logging_sql from src.batch.common.batch_context import BatchContext -from src.db.database import Database +from src.db.async_database import AsyncDatabase from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger from src.time.elapsed_time import ElapsedTime -logger = get_logger('48-施設担当者マスタ洗替') +logger = get_logger('施設担当者マスタ洗替') batch_context = BatchContext.get_instance() async def exec(): - db = Database.get_instance() + db = AsyncDatabase.get_instance() try: - db.connect() + await db.connect() logger.debug('##########################') logger.debug('START Changing Employee in charge of institution PGM.') # 業務日付を取得 syor_date = batch_context.syor_date # `emp_chg_inst_lau`をTruncate - _truncate_emp_chg_inst_lau(db) + await _truncate_emp_chg_inst_lau(db) # emp_chg_inst から、`emp_chg_inst_lau`へInsert - _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db) + await _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db) # vop_hco_merge_vから、emp_chg_inst_lauをUpdate - _update_emp_chg_inst_lau_from_vop_hco_merge_v(db, syor_date) + await _update_emp_chg_inst_lau_from_vop_hco_merge_v(db, syor_date) # dcf_inst_mergeから、emp_chg_inst_lauをUpdate - _update_dcf_inst_merge_from_emp_chg_inst_lau(db, syor_date) + await _update_dcf_inst_merge_from_emp_chg_inst_lau(db, syor_date) logger.debug('##########################') logger.debug('End All Processing PGM.') except Exception as e: raise BatchOperationException(e) finally: - db.disconnect() + await db.disconnect() -def _truncate_emp_chg_inst_lau(db: Database): +async def _truncate_emp_chg_inst_lau(db: AsyncDatabase): logger.debug("##########################") try: - db.execute("TRUNCATE TABLE src05.emp_chg_inst_lau") + await db.execute("TRUNCATE TABLE src05.emp_chg_inst_lau") except Exception as e: logger.debug("Error! Truncate Table `emp_chg_inst_lau` is Failed!!!") + logger.exception(e) raise e logger.debug("Table `emp_chg_inst_lau` was truncated!") return -def _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db: Database): +async def _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db: AsyncDatabase): logger.debug("##########################") try: elapsed_time = ElapsedTime() @@ -70,22 +71,21 @@ def _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db: Database): WHERE enabled_flg = 'Y' """ - res = db.execute(sql) + res = await db.execute(sql) logging_sql(logger, sql) logger.info(f'Query OK, {res.rowcount} rows affected ({elapsed_time.of})') except Exception as e: logger.debug("Error! Insert into `emp_chg_inst_lau` from `emp_chg_inst` was failed!!!") raise e logger.debug("Success! Insert into `emp_chg_inst_lau` from `emp_chg_inst` was inserted!") - return -def _update_emp_chg_inst_lau_from_vop_hco_merge_v(db: Database, syor_date: str): +async def _update_emp_chg_inst_lau_from_vop_hco_merge_v(db: AsyncDatabase, syor_date: str): # vop_hco_merge_vはデータが作られないため、この洗い替え処理は基本空振りする logger.debug("##########################") try: - select_result = db.execute_select( + select_result = await db.execute_select( """ SELECT COUNT(v_inst_cd) AS row_count @@ -106,7 +106,7 @@ def _update_emp_chg_inst_lau_from_vop_hco_merge_v(db: Database, syor_date: str): logger.info('vop_hco_merge_v Table Data is exists!') # vop_hco_merge_v から、emp_chg_inst_lauをUpdateします - result = db.execute_select( + result = await db.execute_select( """ SELECT v_inst_cd, @@ -144,15 +144,16 @@ def _update_emp_chg_inst_lau_from_vop_hco_merge_v(db: Database, syor_date: str): logger.debug(f"emp_chg_inst_lau v_inst_cd could not set from {v_inst_cd_merge} to {v_inst_cd_merge}!") raise e logger.debug(f"Success! emp_chg_inst_lau v_inst_cd was set from {v_inst_cd} to {v_inst_cd_merge}!") + return -def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: Database, syor_date: str): +async def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: AsyncDatabase, syor_date: str): # dcf_inst_mergeから、emp_chg_inst_lauをUpdate # Get count from DCF_INST_MERGE logger.debug("##########################") try: - select_result = db.execute_select( + select_result = await db.execute_select( """ SELECT COUNT(dcf_inst_cd) AS row_count @@ -166,6 +167,7 @@ def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: Database, syor_date: str): """, {'syor_date': syor_date} ) + except Exception as e: logger.debug("Error! Getting Count of dcf_inst_merge was failed!") raise e @@ -203,7 +205,7 @@ def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: Database, syor_date: str): WHERE el.inst_cd = dm.dcf_inst_cd """ - res = db.execute( + res = await db.execute( update_sql, {'syor_date': syor_date} ) @@ -214,5 +216,4 @@ def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: Database, syor_date: str): raise e logger.debug("emp_chg_inst_lau.v_inst_cd was set!") - return diff --git a/ecs/jskult-batch-daily/src/db/async_database.py b/ecs/jskult-batch-daily/src/db/async_database.py new file mode 100644 index 00000000..fb026fa4 --- /dev/null +++ b/ecs/jskult-batch-daily/src/db/async_database.py @@ -0,0 +1,48 @@ +import asyncio + +from sqlalchemy import CursorResult +from tenacity import retry, stop_after_attempt, wait_exponential + +from src.db.database import Database +from src.logging.get_logger import get_logger +from src.system_var import environment + +logger = get_logger(__name__) + + +class AsyncDatabase(Database): + """データベース非同期操作クラス""" + + def __init__(self, username: str, password: str, host: str, port: int, schema: str) -> None: + super().__init__(username, password, host, port, schema) + + @classmethod + def get_instance(cls): + return cls( + username=environment.DB_USERNAME, + password=environment.DB_PASSWORD, + host=environment.DB_HOST, + port=environment.DB_PORT, + schema=environment.DB_SCHEMA + ) + + @retry( + wait=wait_exponential( + multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT, + min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, + max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS + ), + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + async def connect(self): + await asyncio.get_event_loop().run_in_executor(None, super().connect) + + async def execute_select(self, select_query: str, parameters=None) -> list[dict]: + res = await asyncio.get_event_loop().run_in_executor(None, super().execute_select, select_query, parameters) + return res + + async def execute(self, query: str, parameters=None) -> CursorResult: + res = await asyncio.get_event_loop().run_in_executor(None, super().execute, query, parameters) + return res + + async def disconnect(self): + await asyncio.get_event_loop().run_in_executor(None, super().disconnect) From ac374afb858938777a75879fa1021db1b9380e82 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 19 Apr 2023 13:39:33 +0900 Subject: [PATCH 28/32] =?UTF-8?q?Revert=20"feat:=20async-await=E3=81=A7?= =?UTF-8?q?=E9=9D=9E=E5=90=8C=E6=9C=9F=E5=87=A6=E7=90=86=E3=82=92=E5=88=B6?= =?UTF-8?q?=E5=BE=A1=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E3=81=97=E3=81=9F"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit d8b6dd9978f72eb227e4b4e2fa15ae871691bd71. --- .../batch/bio_sales/create_bio_sales_lot.py | 5 -- .../laundering/emp_chg_inst_laundering.py | 43 ++++++++--------- .../src/db/async_database.py | 48 ------------------- 3 files changed, 21 insertions(+), 75 deletions(-) delete mode 100644 ecs/jskult-batch-daily/src/db/async_database.py diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py index 53b6a2c4..be9f2164 100644 --- a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py @@ -13,11 +13,6 @@ async def exec(): logger.info('営業日ではないため、生物由来卸販売ロット分解処理をスキップします。') return - # 並列処理の確認用コード - # import asyncio - # for i in range(50): - # logger.debug('生物由来卸販売ロット分解：処理中') - # await asyncio.sleep(0.5) # TODO: ここに処理を追記していく logger.debug('生物由来卸販売ロット分解：終了') diff --git a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py index 9cd5ba07..57ef56be 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py @@ -1,52 +1,51 @@ from src.batch.batch_functions import logging_sql from src.batch.common.batch_context import BatchContext -from src.db.async_database import AsyncDatabase +from src.db.database import Database from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger from src.time.elapsed_time import ElapsedTime -logger = get_logger('施設担当者マスタ洗替') +logger = get_logger('48-施設担当者マスタ洗替') batch_context = BatchContext.get_instance() async def exec(): - db = AsyncDatabase.get_instance() + db = Database.get_instance() try: - await db.connect() + db.connect() logger.debug('##########################') logger.debug('START Changing Employee in charge of institution PGM.') # 業務日付を取得 syor_date = batch_context.syor_date # `emp_chg_inst_lau`をTruncate - await _truncate_emp_chg_inst_lau(db) + _truncate_emp_chg_inst_lau(db) # emp_chg_inst から、`emp_chg_inst_lau`へInsert - await _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db) + _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db) # vop_hco_merge_vから、emp_chg_inst_lauをUpdate - await _update_emp_chg_inst_lau_from_vop_hco_merge_v(db, syor_date) + _update_emp_chg_inst_lau_from_vop_hco_merge_v(db, syor_date) # dcf_inst_mergeから、emp_chg_inst_lauをUpdate - await _update_dcf_inst_merge_from_emp_chg_inst_lau(db, syor_date) + _update_dcf_inst_merge_from_emp_chg_inst_lau(db, syor_date) logger.debug('##########################') logger.debug('End All Processing PGM.') except Exception as e: raise BatchOperationException(e) finally: - await db.disconnect() + db.disconnect() -async def _truncate_emp_chg_inst_lau(db: AsyncDatabase): +def _truncate_emp_chg_inst_lau(db: Database): logger.debug("##########################") try: - await db.execute("TRUNCATE TABLE src05.emp_chg_inst_lau") + db.execute("TRUNCATE TABLE src05.emp_chg_inst_lau") except Exception as e: logger.debug("Error! Truncate Table `emp_chg_inst_lau` is Failed!!!") - logger.exception(e) raise e logger.debug("Table `emp_chg_inst_lau` was truncated!") return -async def _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db: AsyncDatabase): +def _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db: Database): logger.debug("##########################") try: elapsed_time = ElapsedTime() @@ -71,21 +70,22 @@ async def _insert_into_emp_chg_inst_lau_from_emp_chg_inst(db: AsyncDatabase): WHERE enabled_flg = 'Y' """ - res = await db.execute(sql) + res = db.execute(sql) logging_sql(logger, sql) logger.info(f'Query OK, {res.rowcount} rows affected ({elapsed_time.of})') except Exception as e: logger.debug("Error! Insert into `emp_chg_inst_lau` from `emp_chg_inst` was failed!!!") raise e logger.debug("Success! Insert into `emp_chg_inst_lau` from `emp_chg_inst` was inserted!") + return -async def _update_emp_chg_inst_lau_from_vop_hco_merge_v(db: AsyncDatabase, syor_date: str): +def _update_emp_chg_inst_lau_from_vop_hco_merge_v(db: Database, syor_date: str): # vop_hco_merge_vはデータが作られないため、この洗い替え処理は基本空振りする logger.debug("##########################") try: - select_result = await db.execute_select( + select_result = db.execute_select( """ SELECT COUNT(v_inst_cd) AS row_count @@ -106,7 +106,7 @@ async def _update_emp_chg_inst_lau_from_vop_hco_merge_v(db: AsyncDatabase, syor_ logger.info('vop_hco_merge_v Table Data is exists!') # vop_hco_merge_v から、emp_chg_inst_lauをUpdateします - result = await db.execute_select( + result = db.execute_select( """ SELECT v_inst_cd, @@ -144,16 +144,15 @@ async def _update_emp_chg_inst_lau_from_vop_hco_merge_v(db: AsyncDatabase, syor_ logger.debug(f"emp_chg_inst_lau v_inst_cd could not set from {v_inst_cd_merge} to {v_inst_cd_merge}!") raise e logger.debug(f"Success! emp_chg_inst_lau v_inst_cd was set from {v_inst_cd} to {v_inst_cd_merge}!") - return -async def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: AsyncDatabase, syor_date: str): +def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: Database, syor_date: str): # dcf_inst_mergeから、emp_chg_inst_lauをUpdate # Get count from DCF_INST_MERGE logger.debug("##########################") try: - select_result = await db.execute_select( + select_result = db.execute_select( """ SELECT COUNT(dcf_inst_cd) AS row_count @@ -167,7 +166,6 @@ async def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: AsyncDatabase, syor_d """, {'syor_date': syor_date} ) - except Exception as e: logger.debug("Error! Getting Count of dcf_inst_merge was failed!") raise e @@ -205,7 +203,7 @@ async def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: AsyncDatabase, syor_d WHERE el.inst_cd = dm.dcf_inst_cd """ - res = await db.execute( + res = db.execute( update_sql, {'syor_date': syor_date} ) @@ -216,4 +214,5 @@ async def _update_dcf_inst_merge_from_emp_chg_inst_lau(db: AsyncDatabase, syor_d raise e logger.debug("emp_chg_inst_lau.v_inst_cd was set!") + return diff --git a/ecs/jskult-batch-daily/src/db/async_database.py b/ecs/jskult-batch-daily/src/db/async_database.py deleted file mode 100644 index fb026fa4..00000000 --- a/ecs/jskult-batch-daily/src/db/async_database.py +++ /dev/null @@ -1,48 +0,0 @@ -import asyncio - -from sqlalchemy import CursorResult -from tenacity import retry, stop_after_attempt, wait_exponential - -from src.db.database import Database -from src.logging.get_logger import get_logger -from src.system_var import environment - -logger = get_logger(__name__) - - -class AsyncDatabase(Database): - """データベース非同期操作クラス""" - - def __init__(self, username: str, password: str, host: str, port: int, schema: str) -> None: - super().__init__(username, password, host, port, schema) - - @classmethod - def get_instance(cls): - return cls( - username=environment.DB_USERNAME, - password=environment.DB_PASSWORD, - host=environment.DB_HOST, - port=environment.DB_PORT, - schema=environment.DB_SCHEMA - ) - - @retry( - wait=wait_exponential( - multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT, - min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, - max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS - ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) - async def connect(self): - await asyncio.get_event_loop().run_in_executor(None, super().connect) - - async def execute_select(self, select_query: str, parameters=None) -> list[dict]: - res = await asyncio.get_event_loop().run_in_executor(None, super().execute_select, select_query, parameters) - return res - - async def execute(self, query: str, parameters=None) -> CursorResult: - res = await asyncio.get_event_loop().run_in_executor(None, super().execute, query, parameters) - return res - - async def disconnect(self): - await asyncio.get_event_loop().run_in_executor(None, super().disconnect) From 3299d88ac7231125cab057f4dac85e3197078a6a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 19 Apr 2023 13:53:17 +0900 Subject: [PATCH 29/32] =?UTF-8?q?Revert=20"feat:=20=E4=B8=A6=E8=A1=8C?= =?UTF-8?q?=E5=87=A6=E7=90=86=E8=A6=8B=E7=9B=B4=E3=81=97=E3=80=82ECS=20?= =?UTF-8?q?=E3=82=BF=E3=82=B9=E3=82=AF=E3=81=AEvCPU=E6=95=B0=E6=9C=80?= =?UTF-8?q?=E9=81=A9=E5=8C=96=E3=81=AE=E3=81=9F=E3=82=81=E3=80=81async-awa?= =?UTF-8?q?it=E3=81=A7=E4=B8=A6=E5=88=97=E5=87=A6=E7=90=86=E3=82=92?= =?UTF-8?q?=E5=AE=9F=E8=A3=85"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit e27757e3ee7ef20a218a354549bef89737660bae. --- ecs/jskult-batch-daily/entrypoint.py | 15 ++------ ...io_sales_lot.py => create_bio_sale_lot.py} | 2 +- .../laundering/emp_chg_inst_laundering.py | 2 +- .../src/batch/laundering/sales_laundering.py | 8 ++--- .../src/batch/parallel_processes.py | 32 +++++++++++++++++ .../src/batch/parallel_tasks.py | 34 ------------------- ecs/jskult-batch-daily/src/jobctrl_daily.py | 8 ++--- 7 files changed, 43 insertions(+), 58 deletions(-) rename ecs/jskult-batch-daily/src/batch/bio_sales/{create_bio_sales_lot.py => create_bio_sale_lot.py} (97%) create mode 100644 ecs/jskult-batch-daily/src/batch/parallel_processes.py delete mode 100644 ecs/jskult-batch-daily/src/batch/parallel_tasks.py diff --git a/ecs/jskult-batch-daily/entrypoint.py b/ecs/jskult-batch-daily/entrypoint.py index 5b06ca48..472efd9f 100644 --- a/ecs/jskult-batch-daily/entrypoint.py +++ b/ecs/jskult-batch-daily/entrypoint.py @@ -1,21 +1,10 @@ """実消化&アルトマーク 日次バッチのエントリーポイント""" -import asyncio - from src import jobctrl_daily - -# 一部並行実行のため、非同期関数化 -async def run(): +if __name__ == '__main__': try: - task = asyncio.create_task(jobctrl_daily.exec()) - result_code = await task - exit(result_code) + exit(jobctrl_daily.exec()) except Exception: # エラーが起きても、正常系のコードで返す。 # エラーが起きた事実はbatch_process内でログを出す。 exit(0) - - -if __name__ == '__main__': - # 非同期関数を実行 - asyncio.run(run()) diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py similarity index 97% rename from ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py rename to ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py index be9f2164..49bd19f1 100644 --- a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py @@ -5,7 +5,7 @@ batch_context = BatchContext.get_instance() logger = get_logger('生物由来卸販売ロット分解') -async def exec(): +def exec(): """生物由来卸販売ロット分解""" logger.debug('生物由来卸販売ロット分解：起動') # 営業日ではない場合、処理をスキップする diff --git a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py index 57ef56be..16ff0c79 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py @@ -9,7 +9,7 @@ logger = get_logger('48-施設担当者マスタ洗替') batch_context = BatchContext.get_instance() -async def exec(): +def exec(): db = Database.get_instance() try: db.connect() diff --git a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py index 3265440f..7b77d8ac 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/sales_laundering.py @@ -7,15 +7,15 @@ batch_context = BatchContext.get_instance() logger = get_logger('実績洗替') -async def exec(): +def exec(): """実績洗替処理""" - logger.debug('実績更新：起動') + logger.info('実績更新：起動') # 営業日ではない場合、実績洗替処理は実行しない if batch_context.is_not_business_day: logger.info('営業日ではないため、実績洗替処理をスキップします。') return # 施設担当者洗替 - await emp_chg_inst_laundering.exec() + emp_chg_inst_laundering.exec() - logger.debug('実績更新：終了') + logger.info('実績更新：終了') diff --git a/ecs/jskult-batch-daily/src/batch/parallel_processes.py b/ecs/jskult-batch-daily/src/batch/parallel_processes.py new file mode 100644 index 00000000..1b7db6c7 --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/parallel_processes.py @@ -0,0 +1,32 @@ +"""並行処理""" + +import concurrent.futures + +from src.batch.bio_sales import create_bio_sale_lot +from src.batch.laundering import sales_laundering +from src.error.exceptions import BatchOperationException + + +def exec(): + # 並行処理を開始 + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + + # 実績更新 + future_sales_laundering = executor.submit(sales_laundering.exec) + # 生物由来ロット分解 + future_create_bio_sales_lot = executor.submit(create_bio_sale_lot.exec) + + # 両方の処理が完了するまで待つ + concurrent.futures.wait([future_sales_laundering, future_create_bio_sales_lot]) + + # エラーがあれば呼び出し元でキャッチする + sales_laundering_exc = future_sales_laundering.exception() + create_bio_sales_lot_exc = future_create_bio_sales_lot.exception() + + # いずれかにエラーが発生していれば、１つのエラーとして返す。 + if sales_laundering_exc is not None or create_bio_sales_lot_exc is not None: + sales_laundering_exc_message = str(sales_laundering_exc) if sales_laundering_exc is not None else '' + create_bio_sales_lot_exc_message = str(create_bio_sales_lot_exc) if create_bio_sales_lot_exc is not None else '' + raise BatchOperationException(f'並行処理中にエラーが発生しました。実績更新="{sales_laundering_exc_message}", 生物由来ロット分解={create_bio_sales_lot_exc_message}') + + return diff --git a/ecs/jskult-batch-daily/src/batch/parallel_tasks.py b/ecs/jskult-batch-daily/src/batch/parallel_tasks.py deleted file mode 100644 index afb58820..00000000 --- a/ecs/jskult-batch-daily/src/batch/parallel_tasks.py +++ /dev/null @@ -1,34 +0,0 @@ -"""並行処理""" - -import asyncio - -from src.batch.bio_sales import create_bio_sales_lot -from src.batch.laundering import sales_laundering -from src.error.exceptions import BatchOperationException - - -async def exec(): - tasks = { - '実績更新': sales_laundering.exec(), - '生物由来ロット分解': create_bio_sales_lot.exec() - } - - # 並行処理を開始 - # 両方の処理が完了するまで待つ - done, _ = await asyncio.wait(tasks.values(), return_when=asyncio.ALL_COMPLETED) - task_results = {} - # 実行結果を確認する - for task_key, task in zip(tasks.keys(), done): - try: - task.result() - task_results[task_key] = True - except Exception as e: - task_results[task_key] = str(e) - - # エラーが発生しているかを確認し、エラーがあれば例外とする - if_error_tasks = {k: v for k, v in task_results.items() if v is not True} - if len(if_error_tasks.keys()) != 0: - messages = ', '.join([f'{k}={v}' for k, v in if_error_tasks.items()]) - raise BatchOperationException(f'並行処理中にエラーが発生しました。{messages}') - - return diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 5c178220..3788d368 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -1,7 +1,7 @@ """実消化&アルトマーク 日次バッチ処理""" from src.aws.s3 import ConfigBucket -from src.batch import parallel_tasks +from src.batch import parallel_processes from src.batch.batch_functions import ( get_batch_statuses, update_batch_process_complete, update_batch_processing_flag_in_processing) @@ -19,7 +19,7 @@ logger = get_logger('日次処理コントロール') batch_context = BatchContext.get_instance() -async def exec(): +def exec(): try: logger.info('日次バッチ：開始') try: @@ -100,7 +100,7 @@ async def exec(): try: # 実績生物由来ロット分解と並行処理 logger.info('並行処理(実績更新-生物由来ロット分解)：起動') - await parallel_tasks.exec() + parallel_processes.exec() logger.info('並行処理(実績更新-生物由来ロット分解)：終了') except BatchOperationException as e: logger.exception(f'並行処理(実績更新-生物由来ロット分解)エラー（異常終了）{e}') @@ -130,5 +130,3 @@ async def exec(): except Exception as e: logger.exception(f'日次バッチ処理中に想定外のエラーが発生しました {e}') raise e - finally: - logger.info('日次バッチ：終了') From 5f7dc50040ed949616d566565e68055422224f8f Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 19 Apr 2023 14:07:32 +0900 Subject: [PATCH 30/32] =?UTF-8?q?feat:=20=E4=B8=A6=E8=A1=8C=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=82=92=E5=85=83=E3=81=AB=E6=88=BB=E3=81=97=E3=81=9F?= =?UTF-8?q?(async-await=E2=86=92ThreadPoolExecutor)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{create_bio_sale_lot.py => create_bio_sales_lot.py} | 6 ++++++ ecs/jskult-batch-daily/src/batch/parallel_processes.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) rename ecs/jskult-batch-daily/src/batch/bio_sales/{create_bio_sale_lot.py => create_bio_sales_lot.py} (83%) diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py similarity index 83% rename from ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py rename to ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py index 49bd19f1..306ff772 100644 --- a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sale_lot.py +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py @@ -13,6 +13,12 @@ def exec(): logger.info('営業日ではないため、生物由来卸販売ロット分解処理をスキップします。') return + # 非同期処理のサンプル + import time + for _ in range(50): + logger.info('処理中') + time.sleep(0.5) + # TODO: ここに処理を追記していく logger.debug('生物由来卸販売ロット分解：終了') diff --git a/ecs/jskult-batch-daily/src/batch/parallel_processes.py b/ecs/jskult-batch-daily/src/batch/parallel_processes.py index 1b7db6c7..603b5e52 100644 --- a/ecs/jskult-batch-daily/src/batch/parallel_processes.py +++ b/ecs/jskult-batch-daily/src/batch/parallel_processes.py @@ -2,7 +2,7 @@ import concurrent.futures -from src.batch.bio_sales import create_bio_sale_lot +from src.batch.bio_sales import create_bio_sales_lot from src.batch.laundering import sales_laundering from src.error.exceptions import BatchOperationException @@ -14,7 +14,7 @@ def exec(): # 実績更新 future_sales_laundering = executor.submit(sales_laundering.exec) # 生物由来ロット分解 - future_create_bio_sales_lot = executor.submit(create_bio_sale_lot.exec) + future_create_bio_sales_lot = executor.submit(create_bio_sales_lot.exec) # 両方の処理が完了するまで待つ concurrent.futures.wait([future_sales_laundering, future_create_bio_sales_lot]) From 70637d960e4016b4471ab598ee08d01bf550a897 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 19 Apr 2023 14:16:20 +0900 Subject: [PATCH 31/32] =?UTF-8?q?feat:=20=E3=83=AD=E3=82=B0ID=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/laundering/emp_chg_inst_laundering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py index 16ff0c79..e9d77529 100644 --- a/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py +++ b/ecs/jskult-batch-daily/src/batch/laundering/emp_chg_inst_laundering.py @@ -5,7 +5,7 @@ from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger from src.time.elapsed_time import ElapsedTime -logger = get_logger('48-施設担当者マスタ洗替') +logger = get_logger('施設担当者マスタ洗替') batch_context = BatchContext.get_instance() From 2ff19c9d6def46e56ec3c11daf31c43f11c7a1b5 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 19 Apr 2023 14:26:27 +0900 Subject: [PATCH 32/32] =?UTF-8?q?feat:=20=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E7=94=A8=E3=81=AE=E3=82=B3=E3=83=BC=E3=83=89=E3=81=AF=E4=B8=80?= =?UTF-8?q?=E6=97=A6=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=82=A2=E3=82=A6?= =?UTF-8?q?=E3=83=88=E3=81=97=E3=81=A6=E3=81=8A=E3=81=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/bio_sales/create_bio_sales_lot.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py index 306ff772..509d7300 100644 --- a/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py +++ b/ecs/jskult-batch-daily/src/batch/bio_sales/create_bio_sales_lot.py @@ -13,11 +13,11 @@ def exec(): logger.info('営業日ではないため、生物由来卸販売ロット分解処理をスキップします。') return - # 非同期処理のサンプル - import time - for _ in range(50): - logger.info('処理中') - time.sleep(0.5) + # # 非同期処理のサンプル + # import time + # for _ in range(50): + # logger.info('処理中') + # time.sleep(0.5) # TODO: ここに処理を追記していく