From 9907a3b6d34eb93c158611195fc437751c0878a5 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 30 May 2025 10:18:52 +0900 Subject: [PATCH 01/23] =?UTF-8?q?feat:=20jskult-batch=E5=86=85=E3=81=AE?= =?UTF-8?q?=E5=90=84=E6=A9=9F=E8=83=BD=E4=BA=8B=E3=81=AB=E4=BD=BF=E3=81=86?= =?UTF-8?q?=E7=92=B0=E5=A2=83=E5=A4=89=E6=95=B0=E3=82=92=E5=88=87=E3=82=8A?= =?UTF-8?q?=E5=88=86=E3=81=91=E3=82=89=E3=82=8C=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=82=AF=E3=83=A9=E3=82=B9=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/dcf_inst_merge_io.py | 45 +++++++++++------ .../batch/environment/__init__.py} | 0 .../environment/dcf_inst_merge_environment.py | 48 +++++++++++++++++++ .../environment/jskult_batch_environment.py | 26 ++++++++++ ecs/jskult-batch/src/error/exceptions.py | 4 ++ .../src/system_var/environment.py | 29 ++++++----- 6 files changed, 125 insertions(+), 27 deletions(-) rename ecs/jskult-batch/{test.py => src/batch/environment/__init__.py} (100%) create mode 100644 ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py create mode 100644 ecs/jskult-batch/src/batch/environment/jskult_batch_environment.py diff --git a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py index 82c7cfbe..499cfa98 100644 --- a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py +++ b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py @@ -4,15 +4,18 @@ import os.path as path import tempfile from src.aws.s3 import JskSendBucket, JskTransferListBucket +from src.batch.environment.dcf_inst_merge_environment import \ + DCFInstMergeEnvironment from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint from src.db.database import Database from src.error.exceptions import (BatchOperationException, + EnvironmentVariableNotSetException, MaxRunCountReachedException) from src.logging.get_logger import get_logger from src.manager.jskult_batch_run_manager import JskultBatchRunManager from src.manager.jskult_batch_status_manager import JskultBatchStatusManager from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager -from src.system_var import environment +from src.system_var import constants logger = get_logger('DCF削除新規マスタ作成') @@ -20,12 +23,22 @@ logger = get_logger('DCF削除新規マスタ作成') class DcfInstMergeIO(JskultBatchEntrypoint): def __init__(self): super().__init__() + # 環境変数をimport + self.environment = DCFInstMergeEnvironment() def execute(self): logger.info("DCF削除新規マスタ作成処理を開始します。") + + # 必須の環境変数が設定されていない場合、エラーにする + try: + self.environment.validate() + except EnvironmentVariableNotSetException as e: + logger.exception(e) + return + jskult_hdke_tbl_manager = JskultHdkeTblManager() jskult_batch_run_manager = JskultBatchRunManager( - environment.BATCH_EXECUTION_ID) + self.environment.BATCH_EXECUTION_ID) if not jskult_hdke_tbl_manager.can_run_process(): logger.error( '日次バッチ処理中またはdump取得が正常終了していないため、DCF削除新規マスタ作成を終了します。') @@ -54,20 +67,21 @@ class DcfInstMergeIO(JskultBatchEntrypoint): transfer_list['jsk_transfer_list']) + len(transfer_list['ult_transfer_list']) jskult_batch_status_manager = JskultBatchStatusManager( - environment.PROCESS_NAME, - # TODO チケットNEWDWH2021-1847の実装で作成した定数に置き換え - 'post_process', - environment.MAX_RUN_COUNT, + self.environment.PROCESS_NAME, + constants.PROCESS_TYPE_POST_PROCESS, + self.environment.MAX_RUN_COUNT, receive_file_count ) try: - jskult_batch_status_manager.set_process_status("start") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_START) try: if not jskult_batch_status_manager.can_run_post_process(): # 後続処理の起動条件を満たしていない場合 # 処理ステータスを「処理待」に設定 - jskult_batch_status_manager.set_process_status("waiting") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_WAITING) # バッチ実行管理テーブルに「retry」で登録 jskult_batch_run_manager.batch_retry() @@ -76,7 +90,8 @@ class DcfInstMergeIO(JskultBatchEntrypoint): except MaxRunCountReachedException: logger.info('最大起動回数に到達したため、DCF削除新規マスタ作成処理を実行します。') - jskult_batch_status_manager.set_process_status("doing") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_DOING) # アルトマーク取込が実行されていた場合にDCF施設削除新規マスタの作成処理を実行 if jskult_batch_status_manager.is_done_ultmarc_import(): @@ -90,17 +105,18 @@ class DcfInstMergeIO(JskultBatchEntrypoint): # CSV出力 dcf_inst_merge_all_records = self._select_dcf_inst_merge_all() file_path = self._make_csv_data( - environment.DCF_INST_MERGE_SEND_FILE_NAME, + self.environment.DCF_INST_MERGE_SEND_FILE_NAME, dcf_inst_merge_all_records) # CSVをS3にアップロード self._upload_dcf_inst_merge_csv_file( - file_path, process_date, environment.DCF_INST_MERGE_SEND_FILE_NAME) + file_path, process_date, self.environment.DCF_INST_MERGE_SEND_FILE_NAME) # 処理が全て正常終了した際に、バッチ実行管理テーブルに「success」で登録 logger.info("DCF削除新規マスタ作成処理を正常終了します。") jskult_batch_run_manager.batch_success() - jskult_batch_status_manager.set_process_status("done") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_DONE) return @@ -108,7 +124,8 @@ class DcfInstMergeIO(JskultBatchEntrypoint): # 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録 logger.exception(f'予期せぬエラーが発生したため、DCF削除新規マスタ作成処理を終了します。{e}') jskult_batch_run_manager.batch_failed() - jskult_batch_status_manager.set_process_status("failed") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) def _select_dcf_inst_merge_all(self) -> tuple[bool, list[dict]]: try: @@ -272,7 +289,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint): 'CREATER', 'CREATE_DATE', 'UPDATER', 'UPDATE_DATE'] with open(csv_file_path, mode='w', encoding='UTF-8') as csv_file: # ヘッダ行書き込み（くくり文字をつけない為にwriterowではなく、writeを使用しています） - csv_file.write(f"{','.join(head_str)}\n") + csv_file.write(f"{','.join(head_str)}\r\n") # UTF-8、CRLF、価囲いありで書き込む writer = csv.writer(csv_file, delimiter=',', lineterminator='\r\n', quotechar='"', doublequote=True, quoting=csv.QUOTE_ALL, diff --git a/ecs/jskult-batch/test.py b/ecs/jskult-batch/src/batch/environment/__init__.py similarity index 100% rename from ecs/jskult-batch/test.py rename to ecs/jskult-batch/src/batch/environment/__init__.py diff --git a/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py b/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py new file mode 100644 index 00000000..3da34501 --- /dev/null +++ b/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py @@ -0,0 +1,48 @@ +from src.batch.environment.jskult_batch_environment import \ + JskultBatchEnvironment +from src.system_var import environment + + +class DCFInstMergeEnvironment(JskultBatchEnvironment): + """実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス""" + + def __init__(self): + self.JSK_IO_BUCKET = environment.JSK_IO_BUCKET + self.JSKULT_BACKUP_BUCKET = environment.JSKULT_BACKUP_BUCKET + self.BATCH_MANAGE_DYNAMODB_TABLE_NAME = environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME + self.BATCH_EXECUTION_ID = environment.BATCH_EXECUTION_ID + self.MAX_RUN_COUNT = environment.MAX_RUN_COUNT + self.PROCESS_NAME = environment.PROCESS_NAME + self.JSK_DATA_SEND_FOLDER = environment.JSK_DATA_SEND_FOLDER + self.JSK_BACKUP_FOLDER = environment.JSK_BACKUP_FOLDER + self.TRANSFER_RESULT_FOLDER = environment.TRANSFER_RESULT_FOLDER + self.TRANSFER_RESULT_FILE_NAME = environment.TRANSFER_RESULT_FILE_NAME + self.DCF_INST_MERGE_SEND_FILE_NAME = environment.DCF_INST_MERGE_SEND_FILE_NAME + + def validate(self): + """ + 必須の環境変数が設定されているかどうか検査する。 + DB関連の環境変数は対象外とする。 + + Raises: + EnvironmentVariableNotSetException: 環境変数の設定ミス + """ + super()._assert_variable_not_empty(self.JSK_IO_BUCKET, 'JSK_IO_BUCKET') + super()._assert_variable_not_empty( + self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET') + super()._assert_variable_not_empty(self.BATCH_MANAGE_DYNAMODB_TABLE_NAME, + 'BATCH_MANAGE_DYNAMODB_TABLE_NAME') + super()._assert_variable_not_empty(self.BATCH_EXECUTION_ID, 'BATCH_EXECUTION_ID') + super()._assert_variable_is_int(self.MAX_RUN_COUNT, 'MAX_RUN_COUNT') + # MAX_RUN_COUNTは数値として扱うため、検査後に変換 + self.MAX_RUN_COUNT = int(self.MAX_RUN_COUNT) + super()._assert_variable_not_empty(self.PROCESS_NAME, 'PROCESS_NAME') + super()._assert_variable_not_empty( + self.JSK_DATA_SEND_FOLDER, 'JSK_DATA_SEND_FOLDER') + super()._assert_variable_not_empty(self.JSK_BACKUP_FOLDER, 'JSK_BACKUP_FOLDER') + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') + super()._assert_variable_not_empty(self.DCF_INST_MERGE_SEND_FILE_NAME, + 'DCF_INST_MERGE_SEND_FILE_NAME') diff --git a/ecs/jskult-batch/src/batch/environment/jskult_batch_environment.py b/ecs/jskult-batch/src/batch/environment/jskult_batch_environment.py new file mode 100644 index 00000000..e5be1d41 --- /dev/null +++ b/ecs/jskult-batch/src/batch/environment/jskult_batch_environment.py @@ -0,0 +1,26 @@ +import abc + +from src.error.exceptions import EnvironmentVariableNotSetException + + +class JskultBatchEnvironment(metaclass=abc.ABCMeta): + """実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス""" + @abc.abstractmethod + def validate(self): + """ + 必須の環境変数が設定されているかどうか検査する。 + DB関連の環境変数は対象外とする。 + """ + pass + + def _assert_variable_not_empty(self, variable: str, variable_name: str): + if variable is None: + raise EnvironmentVariableNotSetException( + f'環境変数[{variable_name}]が設定されていません。') + + def _assert_variable_is_int(self, variable: str, variable_name: str): + try: + int(variable) + except ValueError: + raise EnvironmentVariableNotSetException( + f'環境変数[{variable_name}]が整数ではありません。') diff --git a/ecs/jskult-batch/src/error/exceptions.py b/ecs/jskult-batch/src/error/exceptions.py index aa5f9be6..a6db7ed7 100644 --- a/ecs/jskult-batch/src/error/exceptions.py +++ b/ecs/jskult-batch/src/error/exceptions.py @@ -12,3 +12,7 @@ class BatchOperationException(MeDaCaException): class MaxRunCountReachedException(MeDaCaException): pass + + +class EnvironmentVariableNotSetException(MeDaCaException): + pass diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py index d44805d4..492c9661 100644 --- a/ecs/jskult-batch/src/system_var/environment.py +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -7,21 +7,24 @@ DB_USERNAME = os.environ['DB_USERNAME'] DB_PASSWORD = os.environ['DB_PASSWORD'] DB_SCHEMA = os.environ['DB_SCHEMA'] -# AWS -JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] -BATCH_EXECUTION_ID = os.environ['BATCH_EXECUTION_ID'] -MAX_RUN_COUNT = int(os.environ['MAX_RUN_COUNT']) -TRANSFER_RESULT_FOLDER = os.environ['TRANSFER_RESULT_FOLDER'] -TRANSFER_RESULT_FILE_NAME = os.environ['TRANSFER_RESULT_FILE_NAME'] -DCF_INST_MERGE_SEND_FILE_NAME = os.environ['DCF_INST_MERGE_SEND_FILE_NAME'] -PROCESS_NAME = os.environ['PROCESS_NAME'] -JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] -JSK_IO_BUCKET = os.environ['JSK_IO_BUCKET'] -JSK_BACKUP_FOLDER = os.environ['JSK_BACKUP_FOLDER'] -JSK_DATA_SEND_FOLDER = os.environ['JSK_DATA_SEND_FOLDER'] +# 処理名 +PROCESS_NAME = os.environ.get('PROCESS_NAME', None) # AWS -BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get('BATCH_MANAGE_DYNAMODB_TABLE_NAME') +JSKULT_CONFIG_BUCKET = os.environ.get('JSKULT_CONFIG_BUCKET', None) +BATCH_EXECUTION_ID = os.environ.get('BATCH_EXECUTION_ID', None) +MAX_RUN_COUNT = os.environ.get('MAX_RUN_COUNT', None) +TRANSFER_RESULT_FOLDER = os.environ.get('TRANSFER_RESULT_FOLDER', None) +TRANSFER_RESULT_FILE_NAME = os.environ.get('TRANSFER_RESULT_FILE_NAME', None) +DCF_INST_MERGE_SEND_FILE_NAME = os.environ.get( + 'DCF_INST_MERGE_SEND_FILE_NAME', None) +JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) +JSK_IO_BUCKET = os.environ.get('JSK_IO_BUCKET', None) +JSK_BACKUP_FOLDER = os.environ.get('JSK_BACKUP_FOLDER', None) +JSK_DATA_SEND_FOLDER = os.environ.get('JSK_DATA_SEND_FOLDER', None) +BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get( + 'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) + # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') From 489faafe1bc084283ba340f04f3a1421704d1354 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 30 May 2025 12:18:29 +0900 Subject: [PATCH 02/23] =?UTF-8?q?fix:=20=E4=BB=A5=E4=B8=8B=E3=82=92?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=82=20=E3=83=BB=E3=83=86=E3=83=BC?= =?UTF-8?q?=E3=83=96=E3=83=AB=E5=90=8D=E3=80=81=E3=82=AB=E3=83=A9=E3=83=A0?= =?UTF-8?q?=E5=90=8D=E3=81=8C=E5=A4=A7=E6=96=87=E5=AD=97=E3=81=A0=E3=81=A3?= =?UTF-8?q?=E3=81=9F=E3=81=AE=E3=82=92=E5=B0=8F=E6=96=87=E5=AD=97=E3=81=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=82=20=E3=83=BBDB=E6=8E=A5=E7=B6=9A?= =?UTF-8?q?=E3=80=81begin=E3=81=AFSQL=E5=AE=9F=E8=A1=8C=E3=83=A1=E3=82=BD?= =?UTF-8?q?=E3=83=83=E3=83=89=E3=81=AE=E5=A4=96=E5=81=B4=E3=81=A7=E5=AE=9F?= =?UTF-8?q?=E8=A1=8C=E3=81=97=E3=81=9F=E3=81=84=E3=81=AE=E3=81=A7=E3=80=81?= =?UTF-8?q?db=E3=82=A4=E3=83=B3=E3=82=B9=E3=82=BF=E3=83=B3=E3=82=B9?= =?UTF-8?q?=E3=82=92=E5=8F=96=E3=82=8A=E5=9B=9E=E3=81=99=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E4=BF=AE=E6=AD=A3=E3=80=82=20=E3=83=BB=E3=83=AD?= =?UTF-8?q?=E3=82=B0=E3=83=A1=E3=83=83=E3=82=BB=E3=83=BC=E3=82=B8=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E3=80=82=20=E3=83=BBNOTICE=E3=81=A7=E3=83=AD=E3=82=B0?= =?UTF-8?q?=E5=87=BA=E5=8A=9B=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=82=20=E3=83=BBSELECT=E3=83=A1=E3=82=BD?= =?UTF-8?q?=E3=83=83=E3=83=89=E3=81=AE=E5=AE=A3=E8=A8=80=E4=BD=8D=E7=BD=AE?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/dcf_inst_merge_io.py | 140 +++++++++--------- 1 file changed, 69 insertions(+), 71 deletions(-) diff --git a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py index 499cfa98..6937de3d 100644 --- a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py +++ b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py @@ -73,6 +73,11 @@ class DcfInstMergeIO(JskultBatchEntrypoint): receive_file_count ) try: + # DCF削除新規マスタ作成、出力用にDB接続を開始。 + # トランザクションも開始。 + db = Database.get_instance() + db.connect() + db.to_jst() jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_START) @@ -92,18 +97,19 @@ class DcfInstMergeIO(JskultBatchEntrypoint): jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_DOING) - # アルトマーク取込が実行されていた場合にDCF施設削除新規マスタの作成処理を実行 if jskult_batch_status_manager.is_done_ultmarc_import(): + db.begin() # COM_施設からDCF削除新規マスタに登録 (is_add_dcf_inst_merge, - duplication_inst_records) = self._insert_dcf_inst_merge_from_com_inst(self) + duplication_inst_records) = self._insert_dcf_inst_merge_from_com_inst(db) if is_add_dcf_inst_merge: + logger.info('[NOTICE]DCF施設削除新規マスタが追加されました。') self._output_add_dcf_inst_merge_log( duplication_inst_records) - + db.commit() # CSV出力 - dcf_inst_merge_all_records = self._select_dcf_inst_merge_all() + dcf_inst_merge_all_records = self._select_dcf_inst_merge_all(db) file_path = self._make_csv_data( self.environment.DCF_INST_MERGE_SEND_FILE_NAME, dcf_inst_merge_all_records) @@ -121,100 +127,82 @@ class DcfInstMergeIO(JskultBatchEntrypoint): return except Exception as e: + db.rollback() # 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録 logger.exception(f'予期せぬエラーが発生したため、DCF削除新規マスタ作成処理を終了します。{e}') jskult_batch_run_manager.batch_failed() jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_ERROR) - - def _select_dcf_inst_merge_all(self) -> tuple[bool, list[dict]]: - try: - self._db = Database.get_instance() - self._db.connect() - sql = """\ - SELECT - * - FROM - src07.dcf_inst_merge - """ - dcf_inst_merge_all_records = self._db.execute_select(sql) - return dcf_inst_merge_all_records - except Exception as e: - raise BatchOperationException(e) finally: - self._db.disconnect() + db.disconnect() # com_instからdcf_inst_mergeにinsert - def _insert_dcf_inst_merge_from_com_inst(self) -> tuple[bool, list[dict]]: + def _insert_dcf_inst_merge_from_com_inst(self, db: Database) -> tuple[bool, list[dict]]: try: - self._db = Database.get_instance() - self._db.connect() - self._db.begin() - self._db.to_jst() sql = """\ SELECT - ci.DCF_DSF_INST_CD, - ci.FORM_INST_NAME_KANJI, - ci.DELETE_SCHE_REASON_CD, - ci.DUP_OPP_CD, - ci.SYS_UPDATE_DATE + ci.dcf_dsf_inst_cd AS dcf_dsf_inst_cd, + ci.form_inst_name_kanji AS form_inst_name_kanji, + ci.delete_sche_reason_cd AS delete_sche_reason_cd, + ci.dup_opp_cd AS dup_opp_cd, + ci.sys_update_date AS sys_update_date FROM - src05.COM_INST AS ci + src05.com_inst AS ci WHERE - ci.DUP_OPP_CD IS NOT NULL + ci.dup_opp_cd IS NOT NULL AND - ci.DELETE_SCHE_REASON_CD = 'D' + ci.delete_sche_reason_cd = 'D' AND - ci.DELETE_DATA IS NULL + ci.delete_data IS NULL AND - ci.SYS_UPDATE_DATE BETWEEN src07.get_syor_date() AND NOW() + ci.sys_update_date BETWEEN src07.get_syor_date() AND NOW() AND NOT EXISTS ( SELECT - dim.DCF_INST_CD + dim.dcf_inst_cd FROM - src07.DCF_INST_MERGE AS dim + src07.dcf_inst_merge AS dim WHERE - dim.DCF_INST_CD = ci.DCF_DSF_INST_CD + dim.dcf_inst_cd = ci.dcf_dsf_inst_cd ) AND - (ci.DCF_DSF_INST_CD EXISTS( + (ci.dcf_dsf_inst_cd EXISTS( SELECT - mia.INST_CD + mia.inst_cd FROM - src07.MST_INST_ASSN as mia + src07.mst_inst_assn as mia WHERE - mia.INST_CD = ci.DCF_DSF_INST_CD + mia.inst_cd = ci.dcf_dsf_inst_cd ) ) - OR ci.DCF_DSF_INST_CD EXISTS( + OR ci.dcf_dsf_inst_cd EXISTS( SELECT - ap.PRSB_INST_CD + ap.prsb_inst_cd FROM - src07.ATC_PHARM AS ap + src07.atc_pharm AS ap WHERE - ap.PRSB_INST_CD = ci.DCF_DSF_INST_CD + ap.prsb_inst_cd = ci.dcf_dsf_inst_cd ) - OR ci.DCF_DSF_INST_CD EXISTS( + OR ci.dcf_dsf_inst_cd EXISTS( SELECT - trd.INST_CD + vtsr.inst_cd FROM - src07.TRN_RESULT_DATA AS trd + src07.vw_tebra_sales_refreshed AS vtsr WHERE - trd.INST_CD = ci.DCF_DSF_INST_CD + vtsr.inst_cd = ci.dcf_dsf_inst_cd ) ) ; """ - duplication_inst_records = self._db.execute_select(sql) + duplication_inst_records = db.execute_select(sql) # DCF削除新規マスタ取り込み values_clauses = [] params = {} for clauses_no, row in enumerate(duplication_inst_records, start=1): - dcf_inst_cd_arr = f"DCF_INST_CD{clauses_no}" - dup_opp_cd_arr = f"DUP_OPP_CD{clauses_no}" + dcf_inst_cd_arr = f"dcf_inst_cd{clauses_no}" + dup_opp_cd_arr = f"dup_opp_cd{clauses_no}" values_clause = f"""(:{dcf_inst_cd_arr}, :{dup_opp_cd_arr}, DATE_FORMAT((src07.get_syor_date() + INTERVAL 1 MONTH), @@ -228,33 +216,30 @@ class DcfInstMergeIO(JskultBatchEntrypoint): SYSDATE() )""" values_clauses.append(values_clause) - params[dcf_inst_cd_arr] = row['DCF_DSF_INST_CD'] - params[dup_opp_cd_arr] = row['DUP_OPP_CD'] + params[dcf_inst_cd_arr] = row['dcf_dsf_inst_cd'] + params[dup_opp_cd_arr] = row['dup_opp_cd'] insert_sql = f""" INSERT INTO src07.dcf_inst_merge ( - DCF_INST_CD, - DUP_OPP_CD, - START_MONTH, - INVALID_FLG, - REMARKS, - DCF_INST_CD_NEW, - ENABLED_FLG, - CREATER, - CREATE_DATE, - UPDATER, - UPDATE_DATE + dcf_inst_cd, + dup_opp_cd, + start_month, + invalid_flg, + remarks, + dcf_inst_cd_new, + enabled_flg, + creater, + create_date, + updater, + update_date ) VALUES {','.join(values_clauses)} """ - self._db.execute(insert_sql, params) + db.execute(insert_sql, params) return (True, duplication_inst_records) except Exception as e: - self._db.rollback() raise BatchOperationException(e) - finally: - self._db.disconnect() def _output_add_dcf_inst_merge_log(duplication_inst_records: list[dict]): sys_update_date = duplication_inst_records[0]['sys_update_date'] @@ -270,7 +255,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint): add_dct_inst_merge_list = '\n'.join(add_dct_inst_merge_list) # 顧客報告用にログ出力 logger.info( - f"""DCF施設統合マスタが追加されました。 + f"""DCF施設削除新規マスタが追加されました。 ********************************************************** 適用月度　{set_year_month} ********************************************************** @@ -281,6 +266,19 @@ class DcfInstMergeIO(JskultBatchEntrypoint): return + def _select_dcf_inst_merge_all(self, db: Database) -> tuple[bool, list[dict]]: + try: + sql = """\ + SELECT + * + FROM + src07.dcf_inst_merge + """ + dcf_inst_merge_all_records = db.execute_select(sql) + return dcf_inst_merge_all_records + except Exception as e: + raise BatchOperationException(e) + def _make_csv_data(csv_file_name: str, record_inst: list): temporary_dir = tempfile.mkdtemp() csv_file_path = path.join(temporary_dir, csv_file_name) From cd21cbb79a847001ae81ada79fb4a42ffd0f8def Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 30 May 2025 13:43:53 +0900 Subject: [PATCH 03/23] =?UTF-8?q?feat:=20=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=81=AE=E5=BF=85=E9=A0=88=E3=83=81=E3=82=A7=E3=83=83?= =?UTF-8?q?=E3=82=AF=E3=81=AF=E3=82=B3=E3=83=B3=E3=82=B9=E3=83=88=E3=83=A9?= =?UTF-8?q?=E3=82=AF=E3=82=BF=E3=81=A7=E8=A1=8C=E3=81=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch/src/batch/dcf_inst_merge_io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py index 6937de3d..5f4b6dd7 100644 --- a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py +++ b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py @@ -26,9 +26,6 @@ class DcfInstMergeIO(JskultBatchEntrypoint): # 環境変数をimport self.environment = DCFInstMergeEnvironment() - def execute(self): - logger.info("DCF削除新規マスタ作成処理を開始します。") - # 必須の環境変数が設定されていない場合、エラーにする try: self.environment.validate() @@ -36,6 +33,9 @@ class DcfInstMergeIO(JskultBatchEntrypoint): logger.exception(e) return + def execute(self): + logger.info("DCF削除新規マスタ作成処理を開始します。") + jskult_hdke_tbl_manager = JskultHdkeTblManager() jskult_batch_run_manager = JskultBatchRunManager( self.environment.BATCH_EXECUTION_ID) From e912c5a080139b52d8e265f193455892826086de Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 30 May 2025 18:02:38 +0900 Subject: [PATCH 04/23] =?UTF-8?q?ix:=20=E5=8B=95=E3=81=8F=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch/src/aws/s3.py | 6 +- .../src/batch/dcf_inst_merge_io.py | 60 ++++++++++--------- .../dcf_inst_merge_io_task_settings.env | 1 - 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/ecs/jskult-batch/src/aws/s3.py b/ecs/jskult-batch/src/aws/s3.py index 6e5755be..6150c617 100644 --- a/ecs/jskult-batch/src/aws/s3.py +++ b/ecs/jskult-batch/src/aws/s3.py @@ -100,7 +100,7 @@ class JskUltBackupBucket(S3Bucket): class JskBackupBucket(JskUltBackupBucket): - _folder = environment.JSKULT_BACKUP_BUCKET + _folder = environment.JSK_BACKUP_FOLDER class JskTransferListBucket(JskUltBackupBucket): @@ -133,8 +133,8 @@ class JskSendBucket(S3Bucket): def backup_dcf_inst_merge_csv_file(self, dat_file_key: str, datetime_key: str): # バックアップバケットにコピー - jskult_backup_bucket = JskUltBackupBucket() + jskult_backup_bucket = JskBackupBucket() dat_key = f'{self._send_folder}/{dat_file_key}' - backup_key = f'{jskult_backup_bucket._folder}/{self._send_folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}' + backup_key = f'{jskult_backup_bucket._folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}' self._s3_client.copy(self._bucket_name, dat_key, jskult_backup_bucket._bucket_name, backup_key) diff --git a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py index 5f4b6dd7..d2fc61bf 100644 --- a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py +++ b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py @@ -38,7 +38,9 @@ class DcfInstMergeIO(JskultBatchEntrypoint): jskult_hdke_tbl_manager = JskultHdkeTblManager() jskult_batch_run_manager = JskultBatchRunManager( + self.environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME, self.environment.BATCH_EXECUTION_ID) + if not jskult_hdke_tbl_manager.can_run_process(): logger.error( '日次バッチ処理中またはdump取得が正常終了していないため、DCF削除新規マスタ作成を終了します。') @@ -73,12 +75,6 @@ class DcfInstMergeIO(JskultBatchEntrypoint): receive_file_count ) try: - # DCF削除新規マスタ作成、出力用にDB接続を開始。 - # トランザクションも開始。 - db = Database.get_instance() - db.connect() - db.to_jst() - jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_START) try: @@ -90,13 +86,19 @@ class DcfInstMergeIO(JskultBatchEntrypoint): # バッチ実行管理テーブルに「retry」で登録 jskult_batch_run_manager.batch_retry() - + logger.info("起動条件を満たしていないため、DCF削除新規マスタ作成処理を終了します。") return except MaxRunCountReachedException: logger.info('最大起動回数に到達したため、DCF削除新規マスタ作成処理を実行します。') jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_DOING) + + # DCF削除新規マスタ作成、出力用にDB接続を開始。 + # トランザクションも開始。 + db = Database.get_instance() + db.connect() + db.to_jst() # アルトマーク取込が実行されていた場合にDCF施設削除新規マスタの作成処理を実行 if jskult_batch_status_manager.is_done_ultmarc_import(): db.begin() @@ -116,7 +118,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint): # CSVをS3にアップロード self._upload_dcf_inst_merge_csv_file( - file_path, process_date, self.environment.DCF_INST_MERGE_SEND_FILE_NAME) + self.environment.DCF_INST_MERGE_SEND_FILE_NAME, process_date, file_path) # 処理が全て正常終了した際に、バッチ実行管理テーブルに「success」で登録 logger.info("DCF削除新規マスタ作成処理を正常終了します。") @@ -154,49 +156,51 @@ class DcfInstMergeIO(JskultBatchEntrypoint): AND ci.delete_sche_reason_cd = 'D' AND - ci.delete_data IS NULL + ci.abolish_ymd IS NULL AND ci.sys_update_date BETWEEN src07.get_syor_date() AND NOW() AND NOT EXISTS ( SELECT - dim.dcf_inst_cd + 1 FROM src07.dcf_inst_merge AS dim WHERE dim.dcf_inst_cd = ci.dcf_dsf_inst_cd ) - AND - - (ci.dcf_dsf_inst_cd EXISTS( + AND( + EXISTS( SELECT - mia.inst_cd + 1 FROM src07.mst_inst_assn as mia WHERE mia.inst_cd = ci.dcf_dsf_inst_cd ) - ) - OR ci.dcf_dsf_inst_cd EXISTS( + OR EXISTS( SELECT - ap.prsb_inst_cd + 1 FROM src07.atc_pharm AS ap WHERE ap.prsb_inst_cd = ci.dcf_dsf_inst_cd - ) - OR ci.dcf_dsf_inst_cd EXISTS( - SELECT - vtsr.inst_cd - FROM - src07.vw_tebra_sales_refreshed AS vtsr - WHERE - vtsr.inst_cd = ci.dcf_dsf_inst_cd - ) ) - ; + OR EXISTS( + SELECT + 1 + FROM + view07.vw_tebra_sales_refreshed AS vtsr + WHERE + vtsr.cnvs_inst_cd = ci.dcf_dsf_inst_cd + ) + ); """ duplication_inst_records = db.execute_select(sql) + + if len(duplication_inst_records) == 0: + logger.info('施設統合対象データはありません') + return (False, None) + # DCF削除新規マスタ取り込み values_clauses = [] params = {} @@ -279,7 +283,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint): except Exception as e: raise BatchOperationException(e) - def _make_csv_data(csv_file_name: str, record_inst: list): + def _make_csv_data(self, csv_file_name: str, record_inst: list): temporary_dir = tempfile.mkdtemp() csv_file_path = path.join(temporary_dir, csv_file_name) head_str = ['DCF_INST_CD', 'DUP_OPP_CD', 'START_MONTH', diff --git a/s3/config/jskult/task_settings/dcf_inst_merge_io_task_settings.env b/s3/config/jskult/task_settings/dcf_inst_merge_io_task_settings.env index 8469e66f..09723686 100644 --- a/s3/config/jskult/task_settings/dcf_inst_merge_io_task_settings.env +++ b/s3/config/jskult/task_settings/dcf_inst_merge_io_task_settings.env @@ -6,7 +6,6 @@ JSK_BACKUP_FOLDER=jsk/send TRANSFER_RESULT_FOLDER=transfer_result TRANSFER_RESULT_FILE_NAME=transfer_result.json DCF_INST_MERGE_SEND_FILE_NAME=dcf_inst_merge.csv -JSKULT_CONFIG_BUCKET=mbj-newdwh2021-staging-config DB_CONNECTION_MAX_RETRY_ATTEMPT=1 DB_CONNECTION_RETRY_INTERVAL_INIT=1 DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=1 From 58abea45be03330d0cc54539b0866addab9f1ef7 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 30 May 2025 18:02:58 +0900 Subject: [PATCH 05/23] =?UTF-8?q?fix:=20=E3=82=A2=E3=83=AB=E3=83=88?= =?UTF-8?q?=E3=83=9E=E3=83=BC=E3=82=AF=E5=8F=96=E8=BE=BC=E3=81=AE=E3=83=90?= =?UTF-8?q?=E3=83=83=E3=82=AF=E3=82=A2=E3=83=83=E3=83=97=E5=85=88=E3=81=8C?= =?UTF-8?q?=E3=81=8A=E3=81=8B=E3=81=97=E3=81=8B=E3=81=A3=E3=81=9F=E3=81=AE?= =?UTF-8?q?=E3=81=A7=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-ultmarc-io/src/aws/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-batch-ultmarc-io/src/aws/s3.py b/ecs/jskult-batch-ultmarc-io/src/aws/s3.py index 9c5b8f71..e5dbcb77 100644 --- a/ecs/jskult-batch-ultmarc-io/src/aws/s3.py +++ b/ecs/jskult-batch-ultmarc-io/src/aws/s3.py @@ -109,5 +109,5 @@ class VjskSendBucket(S3Bucket): # バックアップバケットにコピー vjsk_backup_bucket = VjskBackupBucket() dat_key = f'{self._send_folder}/{dat_file_key}' - backup_key = f'{vjsk_backup_bucket._folder}/{self._send_folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}' + backup_key = f'{vjsk_backup_bucket._folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}' self._s3_client.copy(self._bucket_name, dat_key, vjsk_backup_bucket._bucket_name, backup_key) From 7e8141a9d95cfe422c91d2420a8ef8be05d1345b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 3 Jun 2025 09:07:07 +0900 Subject: [PATCH 06/23] =?UTF-8?q?fix:=20=E3=83=90=E3=82=B0=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch/Dockerfile | 2 +- .../src/batch/dcf_inst_merge_io.py | 42 ++++++++++++------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/ecs/jskult-batch/Dockerfile b/ecs/jskult-batch/Dockerfile index fc0fde90..674c6c53 100644 --- a/ecs/jskult-batch/Dockerfile +++ b/ecs/jskult-batch/Dockerfile @@ -15,6 +15,6 @@ RUN \ pip uninstall -y pipenv virtualenv-clone virtualenv COPY src ./src -COPY entrypoint.py entrypoint.py +COPY entrypoint.py entrypoint.py CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py index d2fc61bf..eaafccab 100644 --- a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py +++ b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py @@ -17,7 +17,7 @@ from src.manager.jskult_batch_status_manager import JskultBatchStatusManager from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager from src.system_var import constants -logger = get_logger('DCF削除新規マスタ作成') +logger = get_logger('DCF削除新規マスタ作成/データ出力') class DcfInstMergeIO(JskultBatchEntrypoint): @@ -34,7 +34,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint): return def execute(self): - logger.info("DCF削除新規マスタ作成処理を開始します。") + logger.info("DCF削除新規マスタ作成/データ出力処理を開始します。") jskult_hdke_tbl_manager = JskultHdkeTblManager() jskult_batch_run_manager = JskultBatchRunManager( @@ -101,6 +101,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint): db.to_jst() # アルトマーク取込が実行されていた場合にDCF施設削除新規マスタの作成処理を実行 if jskult_batch_status_manager.is_done_ultmarc_import(): + logger.info("アルトマークデータが取り込まれているため、DCF削除新規マスタ作成処理を開始します。") db.begin() # COM_施設からDCF削除新規マスタに登録 (is_add_dcf_inst_merge, @@ -109,8 +110,11 @@ class DcfInstMergeIO(JskultBatchEntrypoint): logger.info('[NOTICE]DCF施設削除新規マスタが追加されました。') self._output_add_dcf_inst_merge_log( duplication_inst_records) + logger.info("DCF削除新規マスタ作成処理が正常終了しました。") db.commit() - # CSV出力 + + # DCF施設削除新規マスタ出力 + logger.info('DCF施設削除新規マスタ出力を開始します。') dcf_inst_merge_all_records = self._select_dcf_inst_merge_all(db) file_path = self._make_csv_data( self.environment.DCF_INST_MERGE_SEND_FILE_NAME, @@ -119,9 +123,10 @@ class DcfInstMergeIO(JskultBatchEntrypoint): # CSVをS3にアップロード self._upload_dcf_inst_merge_csv_file( self.environment.DCF_INST_MERGE_SEND_FILE_NAME, process_date, file_path) + logger.info("DCF施設削除新規マスタ出力が正常終了しました。") # 処理が全て正常終了した際に、バッチ実行管理テーブルに「success」で登録 - logger.info("DCF削除新規マスタ作成処理を正常終了します。") + logger.info("DCF削除新規マスタ作成/データ出力処理を終了します。") jskult_batch_run_manager.batch_success() jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_DONE) @@ -146,13 +151,20 @@ class DcfInstMergeIO(JskultBatchEntrypoint): SELECT ci.dcf_dsf_inst_cd AS dcf_dsf_inst_cd, ci.form_inst_name_kanji AS form_inst_name_kanji, - ci.delete_sche_reason_cd AS delete_sche_reason_cd, ci.dup_opp_cd AS dup_opp_cd, - ci.sys_update_date AS sys_update_date + ( + SELECT + dupci.form_inst_name_kanji + FROM + src05.com_inst AS dupci + WHERE + dupci.dcf_dsf_inst_cd = ci.dup_opp_cd + ) AS dup_inst_name_kanji, + DATE_FORMAT((src07.get_syor_date() + INTERVAL 1 MONTH), '%Y%m') AS start_month FROM src05.com_inst AS ci WHERE - ci.dup_opp_cd IS NOT NULL + (ci.dup_opp_cd IS NOT NULL OR CHAR_LENGTH(ci.dup_opp_cd) > 0) AND ci.delete_sche_reason_cd = 'D' AND @@ -207,21 +219,23 @@ class DcfInstMergeIO(JskultBatchEntrypoint): for clauses_no, row in enumerate(duplication_inst_records, start=1): dcf_inst_cd_arr = f"dcf_inst_cd{clauses_no}" dup_opp_cd_arr = f"dup_opp_cd{clauses_no}" + start_month_arr = f'start_month{clauses_no}' values_clause = f"""(:{dcf_inst_cd_arr}, :{dup_opp_cd_arr}, - DATE_FORMAT((src07.get_syor_date() + INTERVAL 1 MONTH), + :{start_month_arr}, NULL, NULL, NULL, "Y", - batchuser, + CURRENT_USER(), SYSDATE(), - batchuser, + CURRENT_USER(), SYSDATE() )""" values_clauses.append(values_clause) params[dcf_inst_cd_arr] = row['dcf_dsf_inst_cd'] params[dup_opp_cd_arr] = row['dup_opp_cd'] + params[start_month_arr] = row['start_month'] insert_sql = f""" INSERT INTO src07.dcf_inst_merge ( @@ -245,11 +259,11 @@ class DcfInstMergeIO(JskultBatchEntrypoint): except Exception as e: raise BatchOperationException(e) - def _output_add_dcf_inst_merge_log(duplication_inst_records: list[dict]): - sys_update_date = duplication_inst_records[0]['sys_update_date'] + def _output_add_dcf_inst_merge_log(self, duplication_inst_records: list[dict]): + start_month = duplication_inst_records[0]['start_month'] set_year_month = '{set_year}年{set_month}月'.format( - set_year=sys_update_date[0:4], - set_month=sys_update_date[-2:] + set_year=start_month[0:4], + set_month=start_month[-2:] ) add_dct_inst_merge = 'DCF施設コード　{dcf_dsf_inst_cd}　{form_inst_name_kanji}, 　重複時相手先コード　{dup_opp_cd}　{dup_inst_name_kanji}' add_dct_inst_merge_list = [] From 31a8f79fcac25c5894fa6ad1a174b5b85eae4c04 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 3 Jun 2025 18:27:32 +0900 Subject: [PATCH 07/23] =?UTF-8?q?format:=20=E3=83=95=E3=82=A9=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=83=E3=83=88=E9=81=A9=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/trn_result_data_bio_lot.py | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py b/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py index 65e7408a..24a0941f 100644 --- a/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py +++ b/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py @@ -1,25 +1,22 @@ import json + from src.aws.s3 import JskTransferListBucket from src.batch.environment.trn_result_data_bio_lot_environment import \ TrnResultDataBioLotEnvironment from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint -from src.manager.jskult_batch_run_manager import JskultBatchRunManager -from src.manager.jskult_batch_status_manager import JskultBatchStatusManager -from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager from src.db.database import Database from src.error.exceptions import (BatchOperationException, EnvironmentVariableNotSetException, MaxRunCountReachedException) from src.logging.get_logger import get_logger +from src.manager.jskult_batch_run_manager import JskultBatchRunManager +from src.manager.jskult_batch_status_manager import JskultBatchStatusManager +from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager from src.system_var import constants - - - - - logger = get_logger('生物由来卸販売ロット分解') + class TrnResultDataBioLot(JskultBatchEntrypoint): def __init__(self): super().__init__() @@ -36,7 +33,6 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): """生物由来卸販売ロット分解""" logger.info('生物由来卸販売ロット分解処理開始') - jskult_hdke_tbl_manager = JskultHdkeTblManager() jskult_batch_run_manager = JskultBatchRunManager( self.environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME, @@ -77,12 +73,14 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): ) try: - jskult_batch_status_manager.set_process_status(constants.BATCH_ACTF_BATCH_START) + jskult_batch_status_manager.set_process_status( + constants.BATCH_ACTF_BATCH_START) try: if not jskult_batch_status_manager.can_run_post_process(): # 後続処理の起動条件を満たしていない場合 # 処理ステータスを「処理待」に設定 - jskult_batch_status_manager.set_process_status(constants.PROCESS_STATUS_WAITING) + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_WAITING) # バッチ実行管理テーブルに「retry」で登録 jskult_batch_run_manager.batch_retry() @@ -91,7 +89,8 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): except MaxRunCountReachedException: logger.info('最大起動回数に到達したため、生物由来卸販売ロット分解処理を実行します。') - jskult_batch_status_manager.set_process_status(constants.PROCESS_STATUS_DOING) + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_DOING) db = Database.get_instance() try: db.connect() @@ -105,15 +104,17 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): # 製造ロット管理番号マスタから有効期限を生物由来ロット分解データにセットする self._set_expr_dt_from_lot_num_mst(db) # 施設情報を生物由来ロット分解データにセットする - self._set_inst_info_from_com_inst_or_com_pharm_or_mst_inst_merck(db) + self._set_inst_info_from_com_inst_or_com_pharm_or_mst_inst_merck( + db) db.commit() logger.info('生物由来卸販売ロット分解処理終了') # 処理が全て正常終了した際に、バッチ実行管理テーブルに「success」で登録 logger.info("生物由来卸販売ロット分解処理を正常終了します。") jskult_batch_run_manager.batch_success() - jskult_batch_status_manager.set_process_status(constants.PROCESS_STATUS_DONE) + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_DONE) return - + except Exception as e: db.rollback() raise BatchOperationException(e) @@ -123,8 +124,8 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): # 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録 logger.exception(f'予期せぬエラーが発生したため、生物由来卸販売ロット分解処理を終了します。{e}') jskult_batch_run_manager.batch_failed() - jskult_batch_status_manager.set_process_status(constants.PROCESS_STATUS_ERROR) - + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) def _delete_not_confirm_data_in_trn_result_data_bio_lot(self, db: Database): logger.info('生物由来ロット分解データの未確定データ削除開始') @@ -136,13 +137,12 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): AND bio.seq_no = lot.seq_no AND IFNULL(bio.upd_date, bio.ins_date) >= src07.get_syor_date() """ - res = db.execute(sql) + db.execute(sql) except Exception as e: logger.info('生物由来ロット分解データの未確定データ削除に失敗') raise e logger.info('生物由来ロット分解データの未確定データ削除に成功') - def _insert_trn_result_data_bio_lot(self, db: Database): logger.info('生物由来ロット分解データの作成開始') try: @@ -222,7 +222,6 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): raise e logger.info('生物由来ロット分解データの作成に成功') - def _delete_empty_lot_record(self, db: Database): logger.info('生物由来ロット分解データのロット番号が空のレコードを削除開始') try: @@ -232,13 +231,12 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): -- 空白15桁のデータはロット情報が空とみなして削除する lot.cnvs_lot_no = REPEAT(' ', 15) OR lot.cnvs_lot_no IS NULL """ - res = db.execute(sql) + db.execute(sql) except Exception as e: logger.info('生物由来ロット分解データのロット番号が空のレコードを削除に失敗') raise e logger.info('生物由来ロット分解データのロット番号が空のレコードを削除に成功') - def _set_inst_info_from_com_inst_or_com_pharm_or_mst_inst_merck(self, db: Database): logger.info('COM_施設 or COM_薬局 or メルク独自施設マスタから施設情報を生物由来ロット分解データにセット開始') try: @@ -273,13 +271,13 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): END ) """ - res = db.execute(sql) + db.execute(sql) except Exception as e: - logger.info('COM_施設 or COM_薬局 or メルク独自施設マスタから施設情報を生物由来ロット分解データにセット失敗') + logger.info( + 'COM_施設 or COM_薬局 or メルク独自施設マスタから施設情報を生物由来ロット分解データにセット失敗') raise e logger.info('COM_施設 or COM_薬局 or メルク独自施設マスタから施設情報を生物由来ロット分解データにセット成功') - def _set_ck_last_dt_txt_from_customer_lotno_all(self, db: Database): # ロットマスタから有効期限をセット logger.info('ロットマスタから有効期限をセット開始') @@ -293,7 +291,7 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): SET bio.ck_last_dt_txt = cla.ck_last_dt_txt """ - res = db.execute(sql) + db.execute(sql) except Exception as e: logger.info('ロットマスタから有効期限をセット失敗') raise e From af6d995e6d6a6d1045c8ff941e53fae7ed48226b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 3 Jun 2025 18:38:48 +0900 Subject: [PATCH 08/23] =?UTF-8?q?refactor:=20=E3=83=AA=E3=83=86=E3=83=A9?= =?UTF-8?q?=E3=83=AB=E3=82=92=E5=AE=9A=E6=95=B0=E5=8C=96=E3=80=82=E7=92=B0?= =?UTF-8?q?=E5=A2=83=E5=A4=89=E6=95=B0=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=81=AB=E3=83=95=E3=82=A9=E3=83=BC=E3=83=9E=E3=83=83=E3=83=88?= =?UTF-8?q?=E9=81=A9=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../trn_result_data_bio_lot_environment.py | 13 ++++++++----- .../src/batch/trn_result_data_bio_lot.py | 3 +-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ecs/jskult-batch/src/batch/environment/trn_result_data_bio_lot_environment.py b/ecs/jskult-batch/src/batch/environment/trn_result_data_bio_lot_environment.py index ff0ca12e..4d0bbec1 100644 --- a/ecs/jskult-batch/src/batch/environment/trn_result_data_bio_lot_environment.py +++ b/ecs/jskult-batch/src/batch/environment/trn_result_data_bio_lot_environment.py @@ -23,13 +23,16 @@ class TrnResultDataBioLotEnvironment(JskultBatchEnvironment): Raises: EnvironmentVariableNotSetException: 環境変数の設定ミス """ - super()._assert_variable_not_empty(self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET') - super()._assert_variable_not_empty(self.BATCH_MANAGE_DYNAMODB_TABLE_NAME, 'BATCH_MANAGE_DYNAMODB_TABLE_NAME') + super()._assert_variable_not_empty( + self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET') + super()._assert_variable_not_empty( + self.BATCH_MANAGE_DYNAMODB_TABLE_NAME, 'BATCH_MANAGE_DYNAMODB_TABLE_NAME') super()._assert_variable_not_empty(self.BATCH_EXECUTION_ID, 'BATCH_EXECUTION_ID') super()._assert_variable_is_int(self.MAX_RUN_COUNT, 'MAX_RUN_COUNT') # MAX_RUN_COUNTは数値として扱うため、検査後に変換 self.MAX_RUN_COUNT = int(self.MAX_RUN_COUNT) super()._assert_variable_not_empty(self.PROCESS_NAME, 'PROCESS_NAME') - super()._assert_variable_not_empty(self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') - super()._assert_variable_not_empty(self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') - + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') diff --git a/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py b/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py index 24a0941f..13c1b43e 100644 --- a/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py +++ b/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py @@ -66,8 +66,7 @@ class TrnResultDataBioLot(JskultBatchEntrypoint): jskult_batch_status_manager = JskultBatchStatusManager( self.environment.PROCESS_NAME, - # TODO チケットNEWDWH2021-1847の実装で作成した定数に置き換え - 'post_process', + constants.PROCESS_TYPE_POST_PROCESS, self.environment.MAX_RUN_COUNT, receive_file_count ) From 60fa0ebfedbbe327a276a0c3bc6ceb50541516a7 Mon Sep 17 00:00:00 2001 From: yono Date: Wed, 4 Jun 2025 09:27:49 +0900 Subject: [PATCH 09/23] =?UTF-8?q?fix:=E5=8D=B8=E6=A4=9C=E7=B4=A2=E5=8F=96?= =?UTF-8?q?=E5=BE=97=E9=A0=85=E7=9B=AE=E3=82=92=E3=83=AD=E3=83=83=E3=83=88?= =?UTF-8?q?=E5=88=86=E8=A7=A3=E3=81=AB=E5=A4=89=E6=9B=B4=E3=80=82=E5=8D=B8?= =?UTF-8?q?=E6=A4=9C=E7=B4=A2=E6=99=82=E3=81=AE=E6=A4=9C=E7=B4=A2=E3=82=92?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=81=A7=E3=81=AA=E3=81=8F=E5=8D=B8?= =?UTF-8?q?=E5=90=8D=E3=82=92=E7=94=BB=E9=9D=A2=E3=81=8B=E3=82=89=E9=80=81?= =?UTF-8?q?=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/repositories/wholesaler_master_repository.py | 2 +- ecs/jskult-webapp/src/templates/bioSearchList.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/jskult-webapp/src/repositories/wholesaler_master_repository.py b/ecs/jskult-webapp/src/repositories/wholesaler_master_repository.py index a3e31235..7b474a23 100644 --- a/ecs/jskult-webapp/src/repositories/wholesaler_master_repository.py +++ b/ecs/jskult-webapp/src/repositories/wholesaler_master_repository.py @@ -13,7 +13,7 @@ class WholesalerMasterRepository(BaseRepository): b.orig_whlslr_sub_cd as rec_whs_sub_cd, v2.ws_nm_kj as name, b.cls_whlslr_nm as whs_name - FROM src07.trn_result_data_bio b + FROM src07.trn_result_data_bio_lot b LEFT OUTER JOIN ( SELECT diff --git a/ecs/jskult-webapp/src/templates/bioSearchList.html b/ecs/jskult-webapp/src/templates/bioSearchList.html index 65159f1e..f015d212 100644 --- a/ecs/jskult-webapp/src/templates/bioSearchList.html +++ b/ecs/jskult-webapp/src/templates/bioSearchList.html @@ -47,7 +47,7 @@ + - + @@ -185,7 +185,7 @@ - + From f05ec64f41af7863b8f79defe2feafda6aef3ff3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 6 Jun 2025 15:52:27 +0900 Subject: [PATCH 20/23] =?UTF-8?q?refactor:=20=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E6=83=85=E5=A0=B1=E3=82=92dict=E3=81=A7=E5=8F=96?= =?UTF-8?q?=E3=82=8A=E5=9B=9E=E3=81=97=E3=81=A6=E3=81=84=E3=81=9F=E3=81=8C?= =?UTF-8?q?=E3=80=81=E6=84=8F=E5=91=B3=E3=81=AA=E3=81=84=E3=81=9F=E3=82=81?= =?UTF-8?q?str=E3=81=A7=E5=8F=96=E3=82=8A=E5=9B=9E=E3=81=99=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3=E3=80=82=E4=B8=8D=E8=A6=81?= =?UTF-8?q?=E3=81=AA=E3=83=A1=E3=82=BD=E3=83=83=E3=83=89=E3=82=92=E5=89=8A?= =?UTF-8?q?=E9=99=A4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/aws/s3.py | 52 +++++-------------- 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/ecs/jskult-transfer-receive-file/src/aws/s3.py b/ecs/jskult-transfer-receive-file/src/aws/s3.py index 015992b4..a726c1c3 100644 --- a/ecs/jskult-transfer-receive-file/src/aws/s3.py +++ b/ecs/jskult-transfer-receive-file/src/aws/s3.py @@ -1,11 +1,7 @@ -import gzip import json -import os -import os.path as path -import shutil -import tempfile import boto3 + from src.system_var import environment @@ -20,7 +16,7 @@ class S3Client: return [] contents = response['Contents'] # 末尾がスラッシュで終わるものはフォルダとみなしてスキップする - objects = [{'filename': content['Key'], 'size': content['Size']} + objects = [content['Key'] for content in contents if not content['Key'].endswith('/')] return objects @@ -60,46 +56,26 @@ class JskIOBucket(S3Bucket): self._bucket_name, self._recv_folder) return self._s3_file_list - def download_data_file(self, data_filename: str): - temporary_dir = tempfile.mkdtemp() - temporary_file_path = path.join( - temporary_dir, f'{data_filename.replace(f"{self._recv_folder}/", "")}') - with open(temporary_file_path, mode='wb') as f: - self._s3_client.download_file(self._bucket_name, data_filename, f) - f.seek(0) - return temporary_file_path - - def unzip_data_file(self, filename: str): - temp_dir = os.path.dirname(filename) - decompress_filename = os.path.basename(filename).replace('.gz', '') - decompress_file_path = os.path.join(temp_dir, decompress_filename) - with gzip.open(filename, 'rb') as gz: - with open(decompress_file_path, 'wb') as decompressed_file: - shutil.copyfileobj(gz, decompressed_file) - - ret = [decompress_file_path] - return ret - - def transfer_file_to_import(self, target_file: dict): + def transfer_file_to_import(self, target_file: str): data_import_bucket = DataImportBucket() - transfer_from_file_path = target_file.get("filename") + transfer_from_file_path = target_file transfer_to_filename = transfer_from_file_path.replace( f"{self._recv_folder}/", "") data_import_key = f'{data_import_bucket._folder}/{transfer_to_filename}' self._s3_client.copy(self._bucket_name, transfer_from_file_path, data_import_bucket._bucket_name, data_import_key) - def backup_file(self, target_file: dict, datetime_key: str): + def backup_file(self, target_file: str, datetime_key: str): jsk_backup_bucket = JskBackupBucket() - backup_from_file_path = target_file.get("filename") + backup_from_file_path = target_file backup_to_filename = backup_from_file_path.replace( f"{self._recv_folder}/", "") backup_key = f'{jsk_backup_bucket._folder}/{datetime_key}/{backup_to_filename}' self._s3_client.copy(self._bucket_name, backup_from_file_path, jsk_backup_bucket._bucket_name, backup_key) - def delete_file(self, target_file: dict): - delete_path = target_file.get("filename") + def delete_file(self, target_file: str): + delete_path = target_file self._s3_client.delete_file( self._bucket_name, delete_path) @@ -116,16 +92,16 @@ class UltmarcBucket(S3Bucket): def get_file_list(self): return self._s3_client.list_objects(self._bucket_name, self._folder) - def backup_file(self, target_file: dict, datetime_key: str): + def backup_file(self, target_file: str, datetime_key: str): # バックアップバケットにコピー ultmarc_backup_bucket = UltmarcBackupBucket() - target_file_path = target_file.get("filename") + target_file_path = target_file backup_key = f'{ultmarc_backup_bucket._folder}/{datetime_key}/{target_file_path.replace(f"{self._folder}/", "")}' self._s3_client.copy(self._bucket_name, target_file_path, ultmarc_backup_bucket._bucket_name, backup_key) - def delete_file(self, target_file: dict): - delete_path = target_file.get("filename") + def delete_file(self, target_file: str): + delete_path = target_file self._s3_client.delete_file( self._bucket_name, delete_path) @@ -134,8 +110,8 @@ class UltmarcImportBucket(S3Bucket): _bucket_name = environment.ULTMARC_DATA_BUCKET _folder = environment.ULTMARC_IMPORT_FOLDER - def transfer_file_to_import(self, target_file: dict): - from_file_path = target_file.get("filename") + def transfer_file_to_import(self, target_file: str): + from_file_path = target_file to_filename = from_file_path.replace( f"{UltmarcBucket()._folder}/", "") data_import_key = f'{self._folder}/{to_filename}' From 1d4f20429e443fba47797198a8aafc812a0675d8 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 6 Jun 2025 16:30:31 +0900 Subject: [PATCH 21/23] =?UTF-8?q?feat:=20=E8=BB=A2=E9=80=81=E5=89=8D?= =?UTF-8?q?=E3=81=AB=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=83=97=E3=83=AC?= =?UTF-8?q?=E3=83=95=E3=82=A3=E3=83=83=E3=82=AF=E3=82=B9=E3=81=AE=E9=87=8D?= =?UTF-8?q?=E8=A4=87=E3=82=92=E7=A2=BA=E8=AA=8D=E3=81=99=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-transfer-receive-file/src/main.py | 69 ++++++++++++++++++-- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/ecs/jskult-transfer-receive-file/src/main.py b/ecs/jskult-transfer-receive-file/src/main.py index 55a5f847..3348c5af 100644 --- a/ecs/jskult-transfer-receive-file/src/main.py +++ b/ecs/jskult-transfer-receive-file/src/main.py @@ -1,5 +1,9 @@ """実消化&アルトマーク データ転送処理""" +import itertools +import os +import re + from src.aws.s3 import (JskIOBucket, TransferResultOutputBucket, UltmarcBucket, UltmarcImportBucket) from src.error.exceptions import BatchOperationException @@ -32,12 +36,12 @@ def exec(): # 日次バッチ処理中の場合、後続の処理は行わない if batch_processing_flag == constants.BATCH_ACTF_BATCH_START: - logger.error('日次バッチ処理中のため、日次バッチ処理を終了します。') + logger.error('日次バッチ処理中のため、実消化＆アルトマークデータ転送を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS # dump取得が正常終了していない場合、後続の処理は行わない if dump_status_kbn != constants.DUMP_STATUS_KBN_COMPLETE: - logger.error('dump取得が正常終了していないため、日次バッチ処理を終了します。') + logger.error('dump取得が正常終了していないため、実消化＆アルトマークデータ転送を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS logger.info(f'処理日＝{syor_date}') @@ -54,10 +58,45 @@ def exec(): jsk_receive_file_list = None try: jsk_io_bucket = JskIOBucket() - jsk_receive_file_list: str = jsk_io_bucket.get_file_list() + jsk_receive_file_list: list[str] = jsk_io_bucket.get_file_list() except Exception as e: logger.exception(f'実消化データリスト取得に失敗しました。{e}') return constants.BATCH_EXIT_CODE_SUCCESS + + # 実消化データリストの中で、ファイル種類（ファイル名のプレフィックス）が重複するものがあるかどうかをチェックする。 + # 1) プレフィックスごとにマップを作り、該当するファイル名をリストで集める + # 以下のようなマップが作られる + # { + # "TRN_RESULT_DATA": ["TRN_RESULT_DATA_20250606102030.zip", + # "TRN_RESULT_DATA_20250606112030.zip"], + # "TRN_Recive_Inventry": ["TRN_Recive_Inventry_20250606102030.zip"], + # ... + # } + prefix_map: dict[str, list[str]] = {} + for filename in jsk_receive_file_list: + p = extract_prefix(filename) + prefix_map.setdefault(p, []).append(filename) + + # 2) 重複しているプレフィックスを探す + duplicates = {prefix: file_list for prefix, + file_list in prefix_map.items() if len(file_list) > 1} + + # 3) 重複があれば転送一覧から除外する + if duplicates: + # マップをフラットなリストに変換する + duplicate_files = list( + itertools.chain.from_iterable(duplicates.values())) + logger.warning( + f'実消化データの中で一部重複データがあります。重複データは転送から除外します。重複データ一覧: {duplicate_files}') + # 転送しなかったファイルもバックアップに移動させる + for filename in duplicate_files: + jsk_io_bucket.backup_file(filename, syor_date) + jsk_io_bucket.delete_file(filename) + + # S3内のファイル数と重複ファイルの差集合を取ることで、要素を削除 + jsk_receive_file_list = list( + set(jsk_receive_file_list) - set(duplicate_files)) + logger.info(f'I-4 実消化データリスト取得終了。取得データ一覧：{jsk_receive_file_list}') # ④ 取得した実消化データのリストでループ開始 @@ -72,7 +111,7 @@ def exec(): # ⑧ 転送が完了したファイル名を転送データリストに追加する # ファイル名のみ切り出して追加 transfer_file_lists['jsk_transfer_list'].append( - receive_file['filename'].split('/')[1]) + receive_file.split('/')[1]) # ⑨ ループ終了後、実消化データ転送終了ログ（I-6)を出力する logger.info(f'I-6 実消化データ転送処理終了') @@ -102,12 +141,13 @@ def exec(): # ⑮ 転送が完了したファイル名を転送データリストに追加する # ファイル名のみ切り出して追加 transfer_file_lists['ult_transfer_list'].append( - receive_file['filename'].split('/')[1]) + receive_file.split('/')[1]) # ⑯ ループ終了後、アルトマークデータ転送終了ログ（I-10)を出力する - logger.info(f'I-6 実消化データ転送処理終了') + logger.info(f'I-10 アルトマークデータ転送処理終了') # ⑰ 転送データリストをJSONファイル化し、S3バケットにアップロードする + logger.info(f'I-11 データ転送結果アップロード') TransferResultOutputBucket().put_transfer_result(transfer_file_lists, syor_date) # ⑱ 処理終了ログ(I-12)を出力する @@ -118,3 +158,20 @@ def exec(): except Exception as e: logger.exception(f'実消化&アルトマーク データ転送処理中に想定外のエラーが発生しました {e}') raise e + + +def extract_prefix(filename: str) -> str: + """ + ファイル名から『タイムスタンプ部分 + 拡張子』を除いたプレフィックスを返す。 + 例: "TRN_RESULT_DATA_20250606102030.zip" -> "TRN_RESULT_DATA" + """ + # 拡張子を取り除く + file, ext = os.path.splitext(os.path.basename(filename)) + # ファイル名の末尾がタイムスタンプ（数字14桁）である想定なので、最後の '_' 以降を削除 + # TRN_RESULT_DATA_20250606102030 -> ["TRN_RESULT_DATA", "20250606102030"] + parts = file.rsplit('_', 1) + if len(parts) == 2 and re.fullmatch(r"\d{14}", parts[1]): + return parts[0] + else: + # 「最後がタイムスタンプじゃない」場合、そのままのファイル名全体を返す + return filename From 3e9d0c612dca4c73c3838f87a4c6313d61020787 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 6 Jun 2025 17:58:17 +0900 Subject: [PATCH 22/23] =?UTF-8?q?feat:=20=E3=82=A2=E3=83=AB=E3=83=88?= =?UTF-8?q?=E3=83=9E=E3=83=BC=E3=82=AF=E3=83=87=E3=83=BC=E3=82=BF=E3=81=AE?= =?UTF-8?q?=E8=BB=A2=E9=80=81=E5=87=A6=E7=90=86=E3=82=82=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-transfer-receive-file/src/main.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ecs/jskult-transfer-receive-file/src/main.py b/ecs/jskult-transfer-receive-file/src/main.py index 3348c5af..180500ba 100644 --- a/ecs/jskult-transfer-receive-file/src/main.py +++ b/ecs/jskult-transfer-receive-file/src/main.py @@ -87,7 +87,7 @@ def exec(): duplicate_files = list( itertools.chain.from_iterable(duplicates.values())) logger.warning( - f'実消化データの中で一部重複データがあります。重複データは転送から除外します。重複データ一覧: {duplicate_files}') + f'W-1 実消化データの中で一部重複データがあります。重複データは転送から除外します。重複データ一覧: {duplicate_files}') # 転送しなかったファイルもバックアップに移動させる for filename in duplicate_files: jsk_io_bucket.backup_file(filename, syor_date) @@ -125,6 +125,17 @@ def exec(): except Exception as e: logger.exception(f'アルトマークデータリスト取得に失敗しました。{e}') return constants.BATCH_EXIT_CODE_SUCCESS + + # アルトマークデータは1件以上送られてくるのが想定外のため、1件より多かったら連携から除外する + if len(ultmarc_receive_file_list) > 1: + logger.warning( + f'W-2 アルトマークデータが複数配置されているため、転送から除外します。重複データ一覧: {ultmarc_receive_file_list}') + # 転送しなかった場合でもバックアップに移動させる + for filename in ultmarc_receive_file_list: + ultmarc_bucket.backup_file(filename, syor_date) + ultmarc_bucket.delete_file(filename) + # 連携しないようにするため、リストを0件に書き換える。 + ultmarc_receive_file_list = [] logger.info( f'I-8 アルトマークデータリスト取得終了。取得データ一覧：{ultmarc_receive_file_list}') From e4c2a5c45d1d61efc6a06cc3749f79c0b2bcd848 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 6 Jun 2025 17:58:55 +0900 Subject: [PATCH 23/23] =?UTF-8?q?fix:=20=E3=82=A2=E3=83=AB=E3=83=88?= =?UTF-8?q?=E3=83=9E=E3=83=BC=E3=82=AF=E3=83=87=E3=83=BC=E3=82=BF=E5=8F=96?= =?UTF-8?q?=E8=BE=BC=E3=81=8C=E5=AE=8C=E4=BA=86=E3=81=97=E3=81=9F=E3=82=89?= =?UTF-8?q?=E3=80=81=E3=83=90=E3=83=83=E3=83=81=E3=82=B9=E3=83=86=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E3=82=B9=E7=AE=A1=E7=90=86=E3=81=AF=E5=AE=8C=E4=BA=86?= =?UTF-8?q?=E3=81=A8=E3=81=99=E3=82=8B=E3=80=82=20=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E5=87=BA=E5=8A=9B=E3=81=A7=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E3=81=8C=E8=B5=B7=E3=81=8D=E3=81=A6=E3=82=82=E3=83=90=E3=83=83?= =?UTF-8?q?=E3=83=81=E3=82=B9=E3=83=86=E3=83=BC=E3=82=BF=E3=82=B9=E3=81=AF?= =?UTF-8?q?=E6=9B=B8=E3=81=8D=E6=8F=9B=E3=81=88=E3=81=AA=E3=81=84=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-ultmarc-io/src/main.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ecs/jskult-batch-ultmarc-io/src/main.py b/ecs/jskult-batch-ultmarc-io/src/main.py index cd952ddb..fac189ab 100644 --- a/ecs/jskult-batch-ultmarc-io/src/main.py +++ b/ecs/jskult-batch-ultmarc-io/src/main.py @@ -55,18 +55,17 @@ def exec(): batch_status_manager.set_process_status(constants.PROCESS_STATUS_ERROR) return constants.BATCH_EXIT_CODE_SUCCESS + # アルトマーク取込が正常終了していればバッチステータスを処理済に変更 + # DCF/DSFデータ作成でエラーになっても、バッチ処理としては完了したと判断する。 + batch_status_manager.set_process_status(constants.PROCESS_STATUS_DONE) + try: logger.info('実消化用DCF/DSFデータ作成処理：起動') output_dcf_dsf_data.exec() logger.info('実消化用DCF/DSFデータ作成処理：終了') except BatchOperationException as e: logger.exception(f'実消化用施設DCF/DSF作成処理エラー（異常終了）{e}') - # バッチステータスをエラーに変更 - batch_status_manager.set_process_status(constants.PROCESS_STATUS_ERROR) return constants.BATCH_EXIT_CODE_SUCCESS - # バッチステータスを処理済に変更 - batch_status_manager.set_process_status(constants.PROCESS_STATUS_DONE) - logger.info('アルトマーク取込/データ出力：終了') return constants.BATCH_EXIT_CODE_SUCCESS