From cca854be178336cc13422e4562ec3f38db130d7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Tue, 4 Jul 2023 15:24:05 +0900 Subject: [PATCH 1/8] =?UTF-8?q?=E4=BB=AE=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/.env.example | 3 + ecs/jskult-batch-daily/src/aws/s3.py | 26 ++ .../batch/ultmarc/export_vjsk_csv_process.py | 270 ++++++++++++++++++ .../src/batch/ultmarc/ultmarc_process.py | 13 +- ecs/jskult-batch-daily/src/jobctrl_daily.py | 8 +- .../src/system_var/environment.py | 3 + 6 files changed, 318 insertions(+), 5 deletions(-) create mode 100644 ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py diff --git a/ecs/jskult-batch-daily/.env.example b/ecs/jskult-batch-daily/.env.example index 95aef7fe..4a0eefbe 100644 --- a/ecs/jskult-batch-daily/.env.example +++ b/ecs/jskult-batch-daily/.env.example @@ -11,3 +11,6 @@ ULTMARC_BACKUP_FOLDER=ultmarc JSKULT_CONFIG_BUCKET=********************** JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME=jskult_holiday_list.txt +VJSK_BACKUP_FOLDER=vjsk +VJSK_DATA_SEND_FOLDER=send +VJSK_DATA_BUCKET=************* \ No newline at end of file diff --git a/ecs/jskult-batch-daily/src/aws/s3.py b/ecs/jskult-batch-daily/src/aws/s3.py index 68ed0a7c..fab7e222 100644 --- a/ecs/jskult-batch-daily/src/aws/s3.py +++ b/ecs/jskult-batch-daily/src/aws/s3.py @@ -76,6 +76,28 @@ class UltmarcBucket(S3Bucket): self._s3_client.delete_file(self._bucket_name, dat_file_key) +class VjskBucket(S3Bucket): + _bucket_name = environment.VJSK_DATA_BUCKET + _folder = environment.VJSK_DATA_SEND_FOLDER + + def list_dat_file(self): + return self._s3_client.list_objects(self._bucket_name, self._folder) + + def upload_dat_file(self, vjsk_create_csv: str, csv_file_path: str): + # S3バケットにファイルを移動 + csv_file_name = f'{self._folder}/{vjsk_create_csv}' + s3_client = S3Client() + s3_client.upload_file(csv_file_path, self._bucket_name, csv_file_name) + return + + def backup_dat_file(self, dat_file_key: str, datetime_key: str): + # バックアップバケットにコピー + vjsk_backup_bucket = VjskBackupBucket() + dat_key = f'{self._folder}/{dat_file_key}' + backup_key = f'{vjsk_backup_bucket._folder}/{self._folder}/{datetime_key}/{dat_file_key.replace(f"{self._folder}/", "")}' + self._s3_client.copy(self._bucket_name, dat_key, vjsk_backup_bucket._bucket_name, backup_key) + + class ConfigBucket(S3Bucket): _bucket_name = environment.JSKULT_CONFIG_BUCKET @@ -96,3 +118,7 @@ class JskUltBackupBucket(S3Bucket): class UltmarcBackupBucket(JskUltBackupBucket): _folder = environment.ULTMARC_BACKUP_FOLDER + + +class VjskBackupBucket(JskUltBackupBucket): + _folder = environment.VJSK_BACKUP_FOLDER diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py new file mode 100644 index 00000000..27243e4a --- /dev/null +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py @@ -0,0 +1,270 @@ +"""アルトマークデータ処理""" + +from src.aws.s3 import UltmarcBucket, VjskBucket +from src.batch.common.batch_context import BatchContext + +from src.db.database import Database +from src.logging.get_logger import get_logger +import tempfile +import os.path as path +import csv + +logger = get_logger('V実用消化施設データ作成処理') +ultmarc_bucket = UltmarcBucket() +batch_context = BatchContext.get_instance() + +sql_err_msg = "SQL実行エラーです。" +vjsk_csv_file_name = 'ComInst.csv' + + +def exec(): + db = Database.get_instance() + try: + logger.info('バッチ処理を開始しました。') + + try: + # DB接続 + db.connect() + # ファイル単位でトランザクションを行う + db.begin() + except Exception as e: + logger.info('DB接続エラーです。') + raise e + + # CSVファイルの作成用のSQL実行(施設) + record_inst_csv = csv_data_inst_select(db) + # CSVファイルの作成用のSQL実行(薬局) + record_pharm_csv = csv_data_pharm_select(db) + # CSVファイル作成 + csv_file_path = make_csv_data(record_inst_csv, record_pharm_csv) + + vjsk_bucket = VjskBucket() + try: + # s3へデータ移動 + vjsk_bucket.upload_dat_file(vjsk_csv_file_name, csv_file_path) + except Exception as e: + logger.info('S3バケットDWHへCSVデータを作成できませんでした。') + raise e + + try: + # 処理後ファイルをバックアップ + batch_context = BatchContext.get_instance() + vjsk_bucket.backup_dat_file(vjsk_csv_file_name, batch_context.syor_date) + except Exception as e: + logger.info('バックアップバケットへCSVデータをコピーできませんでした。') + raise e + + csv_count = len(record_inst_csv) + len(record_pharm_csv) + logger.info(f'CSV出力件数: {csv_count}。') + logger.info('バッチ処理を正常に終了しました。') + except Exception as e: + raise e + finally: + # 終了時に必ずコミットする + db.commit() + db.disconnect() + return + + +def csv_data_inst_select(db): + # CSVファイルの作成用のSQL実行(施設) + try: + # 施設テーブル検索SQL + sql = """\ + SELECT dcf_dsf_inst_cd, + inst_div_cd, + addr_unknown_reason_cd, + form_inst_name_kana, + inst_name_kana, + form_inst_name_kanji, + inst_name_kanji, + rltd_univ_prnt_cd, + bed_num, + close_flg, + estab_sche_flg, + close_start_ym, + estab_sche_ym, + ward_abolish_flg, + inst_repre_cd, + inst_repre_kana, + inst_repre, + phone_number_non_flg, + unconf_flg, + inst_phone_number, + inst_addr_kana, + inst_addr, + postal_number, + village_cd, + prefc_cd, + city_cd, + addr_display_number, + addr_cnt_kana, + addr_cnt, + manage_cd, + delete_sche_reason_cd, + hp_assrt_cd, + dup_opp_cd, + insp_item_micrb, + insp_item_serum, + insp_item_blood, + insp_item_patho, + insp_item_paras, + insp_item_biochem, + insp_item_ri, + re_exam_cd, + prmit_bed_num_other, + prmit_bed_num_mental, + prmit_bed_num_tuber, + prmit_bed_num_infection, + prmit_bed_num_sum, + prmit_bed_num_gen, + prmit_bed_num_rcup, + prmit_bed_maint_ymd, + inst_pharm_div, + abolish_ymd, + delete_flg, + filler_1, + filler_2, + filler_3, + filler_4, + filler_5, + regist_date, + create_user, + update_date, + update_user, + sys_regist_date, + regist_prgm_id, + sys_update_date, + update_prgm_id + FROM src05.com_inst ORDER BY dcf_dsf_inst_cd + """ + return db.execute_select(sql) + except Exception as e: + logger.debug(f'{sql_err_msg}') + raise e + + +def csv_data_pharm_select(db): + # CSVファイルの作成用のSQL実行(薬局) + try: + # 薬局テーブル検索SQL + sql = """\ + SELECT dcf_dsf_inst_cd, + inst_div_cd, + addr_unknown_reason_cd, + form_inst_name_kana, + inst_name_kana, + form_inst_name_kanji, + inst_name_kanji, + '', + '', + close_flg, + estab_sche_flg, + close_start_ym, + estab_sche_ym, + '', + '', + inst_repre_kana, + inst_repre, + phone_number_non_flg, + unconf_flg, + inst_phone_number, + inst_addr_kana, + inst_addr, + postal_number, + village_cd, + prefc_cd, + city_cd, + addr_display_number, + addr_cnt_kana, + addr_cnt, + manage_cd, + delete_sche_reason_cd, + '', + dup_opp_cd, + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + '', + inst_pharm_div, + abolish_ymd, + delete_flg, + filler_1, + filler_2, + filler_3, + filler_4, + filler_5, + regist_date, + create_user, + update_date, + update_user, + sys_regist_date, + regist_prgm_id, + sys_update_date, + update_prgm_id + FROM src05.com_pharm ORDER BY dcf_dsf_inst_cd + """ + return db.execute_select(sql) + except Exception as e: + logger.debug(f'{sql_err_msg}') + raise e + + +def make_csv_data(record_inst_csv: list, record_pharm_cs: list): + # 一時ファイルとして保存する(CSVファイル) + try: + + temporary_dir = tempfile.mkdtemp() + csv_file_path = path.join(temporary_dir, vjsk_csv_file_name) + + head_str = ['DCF_DSF_INST_CD', 'INST_DIV_CD', 'ADDR_UNKNOWN_REASON_CD', 'FORM_INST_NAME_KANA', 'INST_NAME_KANA', + 'FORM_INST_NAME_KANJI', 'INST_NAME_KANJI', 'RLTD_UNIV_PRNT_CD', 'BED_NUM', 'CLOSE_FLG', 'ESTAB_SCHE_FLG', + 'CLOSE_START_YM', 'ESTAB_SCHE_YM', 'WARD_ABOLISH_FLG', 'INST_REPRE_CD', 'INST_REPRE_KANA', 'INST_REPRE', + 'PHONE_NUMBER_NON_FLG', 'UNCONF_FLG', 'INST_PHONE_NUMBER', 'INST_ADDR_KANA', 'INST_ADDR', 'POSTAL_NUMBER', + 'VILLAGE_CD', 'PREFC_CD', 'CITY_CD', 'ADDR_DISPLAY_NUMBER', 'ADDR_CNT_KANA', 'ADDR_CNT', 'MANAGE_CD', + 'DELETE_SCHE_REASON_CD', 'HP_ASSRT_CD', 'DUP_OPP_CD', 'INSP_ITEM_MICRB', 'INSP_ITEM_SERUM', 'INSP_ITEM_BLOOD', + 'INSP_ITEM_PATHO', 'INSP_ITEM_PARAS', 'INSP_ITEM_BIOCHEM', 'INSP_ITEM_RI', 'RE_EXAM_CD', 'PRMIT_BED_NUM_OTHER', + 'PRMIT_BED_NUM_MENTAL', 'PRMIT_BED_NUM_TUBER', 'PRMIT_BED_NUM_INFECTION', 'PRMIT_BED_NUM_SUM', 'PRMIT_BED_NUM_GEN', + 'PRMIT_BED_NUM_RCUP', 'PRMIT_BED_MAINT_YMD', 'INST_PHARM_DIV', 'ABOLISH_YMD', 'DELETE_FLG', 'FILLER_1', 'FILLER_2', + 'FILLER_3', 'FILLER_4', 'FILLER_5', 'REGIST_DATE', 'CREATE_USER', 'UPDATE_DATE', 'UPDATE_USER', 'SYS_REGIST_DATE', + 'REGIST_PRGM_ID', 'SYS_UPDATE_DATE', 'UPDATE_PRGM_ID'] + + # Shift-JIS、CRLF、価囲いありで書き込む + with open(csv_file_path, mode='w', encoding='cp932') as csv_file: + writer = csv.writer(csv_file, delimiter=',', lineterminator='\n', + quotechar='"', doublequote=True, quoting=csv.QUOTE_ALL, + strict=True + ) + # ヘッダ行書き込み + writer.writerow(head_str) + + # データ部分書き込み(施設) + for record_inst_data in record_inst_csv: + record_inst_value = list(record_inst_data.values()) + csv_data = ['' if n is None else n for n in record_inst_value] + writer.writerow(csv_data) + + # データ部分書き込み(薬局) + for record_pharm_data in record_pharm_cs: + record_pharm_value = list(record_pharm_data.values()) + csv_data = ['' if n is None else n for n in record_pharm_value] + writer.writerow(csv_data) + + except Exception as e: + logger.info('ワークデータの作成に失敗しました。') + logger.info('バッチ処理を異常終了しました。') + raise e + + return csv_file_path diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py index b511a9c8..e0e6ba22 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py @@ -5,6 +5,7 @@ from datetime import datetime from src.aws.s3 import UltmarcBucket from src.batch.common.batch_context import BatchContext from src.batch.ultmarc.datfile import DatFile +from src.batch.ultmarc import export_vjsk_csv_process from src.batch.ultmarc.utmp_tables.ultmarc_table_mapper_factory import \ UltmarcTableMapperFactory from src.db.database import Database @@ -61,11 +62,19 @@ def exec_import(): def exec_export(): - """V実消化用施設・薬局薬店データ作成処理""" + """V実消化用施設データ作成処理""" if not batch_context.is_ultmarc_imported: - logger.info('アルトマーク取込が行われていないため、V実消化用施設・薬局薬店データ作成処理をスキップします。') + logger.info('アルトマーク取込が行われていないため、V実消化用施設データ作成処理をスキップします。') return + try: + logger.info('V実用消化施設データ作成処理: 開始') + export_vjsk_csv_process.exec() + logger.info('V実用消化施設データ作成処理: 終了') + except Exception as e: + raise BatchOperationException(e) + return + def _import_to_ultmarc_table(dat_file: DatFile): db = Database.get_instance() diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index a98c0d16..d77216ff 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -75,11 +75,13 @@ def exec(): logger.debug(f'{"アルトマーク取込が行われました。" if batch_context.is_ultmarc_imported else "アルトマーク取込が行われませんでした。"}') try: - logger.info('V実消化用施設・薬局薬店データ作成処理：起動') + logger.info('V実消化用施設データ作成処理：起動') +# ***********************実行させるためにtrueにしておく（後で消す）*********************** + batch_context.is_ultmarc_imported = True ultmarc_process.exec_export() - logger.info('V実消化用施設・薬局薬店データ作成処理：終了') + logger.info('V実消化用施設データ作成処理：終了') except BatchOperationException as e: - logger.exception(f'V実消化用施設・薬局薬店データ作成処理エラー（異常終了）{e}') + logger.exception(f'V実消化用施設データ作成処理エラー（異常終了）{e}') return constants.BATCH_EXIT_CODE_SUCCESS logger.info('日次処理（V実消化）') diff --git a/ecs/jskult-batch-daily/src/system_var/environment.py b/ecs/jskult-batch-daily/src/system_var/environment.py index b1730224..4144319f 100644 --- a/ecs/jskult-batch-daily/src/system_var/environment.py +++ b/ecs/jskult-batch-daily/src/system_var/environment.py @@ -15,6 +15,9 @@ ULTMARC_BACKUP_FOLDER = os.environ['ULTMARC_BACKUP_FOLDER'] JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] JSKULT_CONFIG_CALENDAR_FOLDER = os.environ['JSKULT_CONFIG_CALENDAR_FOLDER'] JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME = os.environ['JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME'] +VJSK_BACKUP_FOLDER = os.environ['VJSK_BACKUP_FOLDER'] +VJSK_DATA_SEND_FOLDER = os.environ['VJSK_DATA_SEND_FOLDER'] +VJSK_DATA_BUCKET = os.environ['VJSK_DATA_BUCKET'] # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') From 92c14d2fa0f2fb499dcdd91b40e0c4d061a549a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Tue, 4 Jul 2023 15:25:21 +0900 Subject: [PATCH 2/8] =?UTF-8?q?=E4=BB=AE=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/jobctrl_daily.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index d77216ff..45e297e9 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -76,8 +76,6 @@ def exec(): try: logger.info('V実消化用施設データ作成処理：起動') -# ***********************実行させるためにtrueにしておく（後で消す）*********************** - batch_context.is_ultmarc_imported = True ultmarc_process.exec_export() logger.info('V実消化用施設データ作成処理：終了') except BatchOperationException as e: From f5de058284bdaea1d842435f217500ee5a9e9b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 5 Jul 2023 09:45:58 +0900 Subject: [PATCH 3/8] =?UTF-8?q?=E4=BB=AE=E5=AE=8C=E6=88=90=EF=BC=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/ultmarc/export_vjsk_csv_process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py index 27243e4a..a1b2ff23 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py @@ -243,12 +243,12 @@ def make_csv_data(record_inst_csv: list, record_pharm_cs: list): # Shift-JIS、CRLF、価囲いありで書き込む with open(csv_file_path, mode='w', encoding='cp932') as csv_file: + # ヘッダ行書き込み + csv_file.write(f"{','.join(head_str)}\n") writer = csv.writer(csv_file, delimiter=',', lineterminator='\n', quotechar='"', doublequote=True, quoting=csv.QUOTE_ALL, strict=True ) - # ヘッダ行書き込み - writer.writerow(head_str) # データ部分書き込み(施設) for record_inst_data in record_inst_csv: From 6df0c43e78a68ff53184b17c841ef8b52105e34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 5 Jul 2023 11:45:29 +0900 Subject: [PATCH 4/8] =?UTF-8?q?ultmarc=5Fprocess=E3=82=92=E4=BD=BF?= =?UTF-8?q?=E3=82=8F=E3=81=AA=E3=81=84=E3=82=88=E3=81=86=E3=81=AB=E3=81=97?= =?UTF-8?q?=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/ultmarc/export_vjsk_csv_process.py | 5 +++-- .../src/batch/ultmarc/ultmarc_process.py | 17 +---------------- ecs/jskult-batch-daily/src/jobctrl_daily.py | 6 +++++- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py index a1b2ff23..ddd2a0ae 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py @@ -241,10 +241,11 @@ def make_csv_data(record_inst_csv: list, record_pharm_cs: list): 'FILLER_3', 'FILLER_4', 'FILLER_5', 'REGIST_DATE', 'CREATE_USER', 'UPDATE_DATE', 'UPDATE_USER', 'SYS_REGIST_DATE', 'REGIST_PRGM_ID', 'SYS_UPDATE_DATE', 'UPDATE_PRGM_ID'] - # Shift-JIS、CRLF、価囲いありで書き込む with open(csv_file_path, mode='w', encoding='cp932') as csv_file: - # ヘッダ行書き込み + # ヘッダ行書き込み（くくり文字を加えない為にwriteで出力する） csv_file.write(f"{','.join(head_str)}\n") + + # Shift-JIS、CRLF、価囲いありで書き込む writer = csv.writer(csv_file, delimiter=',', lineterminator='\n', quotechar='"', doublequote=True, quoting=csv.QUOTE_ALL, strict=True diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py index e0e6ba22..0e2ffe6a 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/ultmarc_process.py @@ -5,7 +5,7 @@ from datetime import datetime from src.aws.s3 import UltmarcBucket from src.batch.common.batch_context import BatchContext from src.batch.ultmarc.datfile import DatFile -from src.batch.ultmarc import export_vjsk_csv_process + from src.batch.ultmarc.utmp_tables.ultmarc_table_mapper_factory import \ UltmarcTableMapperFactory from src.db.database import Database @@ -61,21 +61,6 @@ def exec_import(): raise BatchOperationException(e) -def exec_export(): - """V実消化用施設データ作成処理""" - if not batch_context.is_ultmarc_imported: - logger.info('アルトマーク取込が行われていないため、V実消化用施設データ作成処理をスキップします。') - return - - try: - logger.info('V実用消化施設データ作成処理: 開始') - export_vjsk_csv_process.exec() - logger.info('V実用消化施設データ作成処理: 終了') - except Exception as e: - raise BatchOperationException(e) - return - - def _import_to_ultmarc_table(dat_file: DatFile): db = Database.get_instance() try: diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 45e297e9..3b314b26 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -13,6 +13,7 @@ from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger from src.system_var import constants +from src.batch.ultmarc import export_vjsk_csv_process logger = get_logger('日次処理コントロール') @@ -75,8 +76,11 @@ def exec(): logger.debug(f'{"アルトマーク取込が行われました。" if batch_context.is_ultmarc_imported else "アルトマーク取込が行われませんでした。"}') try: + if not batch_context.is_ultmarc_imported: + logger.info('アルトマーク取込が行われていないため、V実消化用施設データ作成処理をスキップします。') + return logger.info('V実消化用施設データ作成処理：起動') - ultmarc_process.exec_export() + export_vjsk_csv_process.exec() logger.info('V実消化用施設データ作成処理：終了') except BatchOperationException as e: logger.exception(f'V実消化用施設データ作成処理エラー（異常終了）{e}') From 552f2475e3d96846d5973bf834a2cb6e08798270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 5 Jul 2023 14:10:41 +0900 Subject: [PATCH 5/8] =?UTF-8?q?=E3=83=8D=E3=83=BC=E3=83=9F=E3=83=B3?= =?UTF-8?q?=E3=82=B0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...rt_vjsk_csv_process.py => vjsk_process.py} | 31 +++++++++---------- ecs/jskult-batch-daily/src/jobctrl_daily.py | 4 +-- 2 files changed, 17 insertions(+), 18 deletions(-) rename ecs/jskult-batch-daily/src/batch/ultmarc/{export_vjsk_csv_process.py => vjsk_process.py} (91%) diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/vjsk_process.py similarity index 91% rename from ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py rename to ecs/jskult-batch-daily/src/batch/ultmarc/vjsk_process.py index ddd2a0ae..9c0b83a1 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/export_vjsk_csv_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/vjsk_process.py @@ -1,6 +1,6 @@ """アルトマークデータ処理""" -from src.aws.s3 import UltmarcBucket, VjskBucket +from src.aws.s3 import VjskBucket from src.batch.common.batch_context import BatchContext from src.db.database import Database @@ -10,14 +10,13 @@ import os.path as path import csv logger = get_logger('V実用消化施設データ作成処理') -ultmarc_bucket = UltmarcBucket() -batch_context = BatchContext.get_instance() sql_err_msg = "SQL実行エラーです。" -vjsk_csv_file_name = 'ComInst.csv' def exec(): + vjsk_csv_file_name = 'ComInst.csv' + db = Database.get_instance() try: logger.info('バッチ処理を開始しました。') @@ -32,11 +31,11 @@ def exec(): raise e # CSVファイルの作成用のSQL実行(施設) - record_inst_csv = csv_data_inst_select(db) + record_inst = select_inst_record(db) # CSVファイルの作成用のSQL実行(薬局) - record_pharm_csv = csv_data_pharm_select(db) + record_pharm = select_pharm_record(db) # CSVファイル作成 - csv_file_path = make_csv_data(record_inst_csv, record_pharm_csv) + csv_file_path = make_csv_data(record_inst, record_pharm, vjsk_csv_file_name) vjsk_bucket = VjskBucket() try: @@ -54,7 +53,7 @@ def exec(): logger.info('バックアップバケットへCSVデータをコピーできませんでした。') raise e - csv_count = len(record_inst_csv) + len(record_pharm_csv) + csv_count = len(record_inst) + len(record_pharm) logger.info(f'CSV出力件数: {csv_count}。') logger.info('バッチ処理を正常に終了しました。') except Exception as e: @@ -66,8 +65,8 @@ def exec(): return -def csv_data_inst_select(db): - # CSVファイルの作成用のSQL実行(施設) +def select_inst_record(db): + # CSVファイル作成用のSQL実行(施設) try: # 施設テーブル検索SQL sql = """\ @@ -144,8 +143,8 @@ def csv_data_inst_select(db): raise e -def csv_data_pharm_select(db): - # CSVファイルの作成用のSQL実行(薬局) +def select_pharm_record(db): + # CSVファイル作成用のSQL実行(薬局) try: # 薬局テーブル検索SQL sql = """\ @@ -222,7 +221,7 @@ def csv_data_pharm_select(db): raise e -def make_csv_data(record_inst_csv: list, record_pharm_cs: list): +def make_csv_data(record_inst: list, record_pharm: list, vjsk_csv_file_name: str): # 一時ファイルとして保存する(CSVファイル) try: @@ -242,7 +241,7 @@ def make_csv_data(record_inst_csv: list, record_pharm_cs: list): 'REGIST_PRGM_ID', 'SYS_UPDATE_DATE', 'UPDATE_PRGM_ID'] with open(csv_file_path, mode='w', encoding='cp932') as csv_file: - # ヘッダ行書き込み（くくり文字を加えない為にwriteで出力する） + # ヘッダ行書き込み（くくり文字をつけない為にwriterowではなく、writeを使用しています） csv_file.write(f"{','.join(head_str)}\n") # Shift-JIS、CRLF、価囲いありで書き込む @@ -252,13 +251,13 @@ def make_csv_data(record_inst_csv: list, record_pharm_cs: list): ) # データ部分書き込み(施設) - for record_inst_data in record_inst_csv: + for record_inst_data in record_inst: record_inst_value = list(record_inst_data.values()) csv_data = ['' if n is None else n for n in record_inst_value] writer.writerow(csv_data) # データ部分書き込み(薬局) - for record_pharm_data in record_pharm_cs: + for record_pharm_data in record_pharm: record_pharm_value = list(record_pharm_data.values()) csv_data = ['' if n is None else n for n in record_pharm_value] writer.writerow(csv_data) diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 3b314b26..0e92bd1b 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -13,7 +13,7 @@ from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger from src.system_var import constants -from src.batch.ultmarc import export_vjsk_csv_process +from src.batch.ultmarc import vjsk_process logger = get_logger('日次処理コントロール') @@ -80,7 +80,7 @@ def exec(): logger.info('アルトマーク取込が行われていないため、V実消化用施設データ作成処理をスキップします。') return logger.info('V実消化用施設データ作成処理：起動') - export_vjsk_csv_process.exec() + vjsk_process.exec() logger.info('V実消化用施設データ作成処理：終了') except BatchOperationException as e: logger.exception(f'V実消化用施設データ作成処理エラー（異常終了）{e}') From afd5db75e5e2bac621e0925d35869357cf2eb08c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 5 Jul 2023 16:32:49 +0900 Subject: [PATCH 6/8] =?UTF-8?q?=E6=8C=87=E6=91=98=E4=BA=8B=E9=A0=85?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...cess.py => output_vjsk_inst_pharm_data.py} | 69 ++++++++++--------- ecs/jskult-batch-daily/src/jobctrl_daily.py | 7 +- 2 files changed, 38 insertions(+), 38 deletions(-) rename ecs/jskult-batch-daily/src/batch/ultmarc/{vjsk_process.py => output_vjsk_inst_pharm_data.py} (84%) diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/vjsk_process.py b/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py similarity index 84% rename from ecs/jskult-batch-daily/src/batch/ultmarc/vjsk_process.py rename to ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py index 9c0b83a1..1fb338b7 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/vjsk_process.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py @@ -1,4 +1,4 @@ -"""アルトマークデータ処理""" +"""output_vjsk_inst_pharm_data""" from src.aws.s3 import VjskBucket from src.batch.common.batch_context import BatchContext @@ -9,7 +9,7 @@ import tempfile import os.path as path import csv -logger = get_logger('V実用消化施設データ作成処理') +logger = get_logger('V実消化用施設データ作成処理') sql_err_msg = "SQL実行エラーです。" @@ -17,15 +17,20 @@ sql_err_msg = "SQL実行エラーです。" def exec(): vjsk_csv_file_name = 'ComInst.csv' + # バッチ共通設定を取得 + batch_context = BatchContext.get_instance() + + if not batch_context.is_ultmarc_imported: + logger.info('アルトマーク取込が行われていないため、V実消化用施設データ作成処理をスキップします。') + return + db = Database.get_instance() try: - logger.info('バッチ処理を開始しました。') + logger.info('処理開始') try: # DB接続 db.connect() - # ファイル単位でトランザクションを行う - db.begin() except Exception as e: logger.info('DB接続エラーです。') raise e @@ -42,7 +47,7 @@ def exec(): # s3へデータ移動 vjsk_bucket.upload_dat_file(vjsk_csv_file_name, csv_file_path) except Exception as e: - logger.info('S3バケットDWHへCSVデータを作成できませんでした。') + logger.info('S3バケットにCSVデータを作成できませんでした。') raise e try: @@ -54,13 +59,11 @@ def exec(): raise e csv_count = len(record_inst) + len(record_pharm) - logger.info(f'CSV出力件数: {csv_count}。') - logger.info('バッチ処理を正常に終了しました。') + logger.info(f'CSV出力件数: {csv_count}') + logger.info('正常終了') except Exception as e: raise e finally: - # 終了時に必ずコミットする - db.commit() db.disconnect() return @@ -155,14 +158,14 @@ def select_pharm_record(db): inst_name_kana, form_inst_name_kanji, inst_name_kanji, - '', - '', + '' AS rltd_univ_prnt_cd, + '' AS bed_num, close_flg, estab_sche_flg, close_start_ym, estab_sche_ym, - '', - '', + '' AS ward_abolish_flg, + '' AS inst_repre_cd, inst_repre_kana, inst_repre, phone_number_non_flg, @@ -179,24 +182,24 @@ def select_pharm_record(db): addr_cnt, manage_cd, delete_sche_reason_cd, - '', + '' AS hp_assrt_cd, dup_opp_cd, - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', - '', + '' AS insp_item_micrb, + '' AS insp_item_serum, + '' AS insp_item_blood, + '' AS insp_item_patho, + '' AS insp_item_paras, + '' AS insp_item_biochem, + '' AS insp_item_ri, + '' AS re_exam_cd, + '' AS prmit_bed_num_other, + '' AS prmit_bed_num_mental, + '' AS prmit_bed_num_tuber, + '' AS prmit_bed_num_infection, + '' AS prmit_bed_num_sum, + '' AS prmit_bed_num_gen, + '' AS prmit_bed_num_rcup, + '' AS prmit_bed_maint_ymd, inst_pharm_div, abolish_ymd, delete_flg, @@ -240,7 +243,7 @@ def make_csv_data(record_inst: list, record_pharm: list, vjsk_csv_file_name: str 'FILLER_3', 'FILLER_4', 'FILLER_5', 'REGIST_DATE', 'CREATE_USER', 'UPDATE_DATE', 'UPDATE_USER', 'SYS_REGIST_DATE', 'REGIST_PRGM_ID', 'SYS_UPDATE_DATE', 'UPDATE_PRGM_ID'] - with open(csv_file_path, mode='w', encoding='cp932') as csv_file: + with open(csv_file_path, mode='w', encoding='UTF-8') as csv_file: # ヘッダ行書き込み（くくり文字をつけない為にwriterowではなく、writeを使用しています） csv_file.write(f"{','.join(head_str)}\n") @@ -264,7 +267,7 @@ def make_csv_data(record_inst: list, record_pharm: list, vjsk_csv_file_name: str except Exception as e: logger.info('ワークデータの作成に失敗しました。') - logger.info('バッチ処理を異常終了しました。') + logger.info('CSVデータの作成に失敗しました。') raise e return csv_file_path diff --git a/ecs/jskult-batch-daily/src/jobctrl_daily.py b/ecs/jskult-batch-daily/src/jobctrl_daily.py index 0e92bd1b..e6751a47 100644 --- a/ecs/jskult-batch-daily/src/jobctrl_daily.py +++ b/ecs/jskult-batch-daily/src/jobctrl_daily.py @@ -13,7 +13,7 @@ from src.batch.ultmarc import ultmarc_process from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger from src.system_var import constants -from src.batch.ultmarc import vjsk_process +from src.batch.ultmarc import output_vjsk_inst_pharm_data logger = get_logger('日次処理コントロール') @@ -76,11 +76,8 @@ def exec(): logger.debug(f'{"アルトマーク取込が行われました。" if batch_context.is_ultmarc_imported else "アルトマーク取込が行われませんでした。"}') try: - if not batch_context.is_ultmarc_imported: - logger.info('アルトマーク取込が行われていないため、V実消化用施設データ作成処理をスキップします。') - return logger.info('V実消化用施設データ作成処理：起動') - vjsk_process.exec() + output_vjsk_inst_pharm_data.exec() logger.info('V実消化用施設データ作成処理：終了') except BatchOperationException as e: logger.exception(f'V実消化用施設データ作成処理エラー（異常終了）{e}') From 518c84695e59af7fecfdf50f9244b53834809065 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 5 Jul 2023 18:05:44 +0900 Subject: [PATCH 7/8] =?UTF-8?q?=E5=90=8D=E7=A7=B0=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/aws/s3.py | 44 +++++++++---------- .../ultmarc/output_vjsk_inst_pharm_data.py | 8 ++-- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/ecs/jskult-batch-daily/src/aws/s3.py b/ecs/jskult-batch-daily/src/aws/s3.py index 8cf6c3cf..86ffc169 100644 --- a/ecs/jskult-batch-daily/src/aws/s3.py +++ b/ecs/jskult-batch-daily/src/aws/s3.py @@ -1,4 +1,3 @@ -import io import os import os.path as path import tarfile @@ -80,28 +79,6 @@ class UltmarcBucket(S3Bucket): self._s3_client.delete_file(self._bucket_name, dat_file_key) -class VjskBucket(S3Bucket): - _bucket_name = environment.VJSK_DATA_BUCKET - _folder = environment.VJSK_DATA_SEND_FOLDER - - def list_dat_file(self): - return self._s3_client.list_objects(self._bucket_name, self._folder) - - def upload_dat_file(self, vjsk_create_csv: str, csv_file_path: str): - # S3バケットにファイルを移動 - csv_file_name = f'{self._folder}/{vjsk_create_csv}' - s3_client = S3Client() - s3_client.upload_file(csv_file_path, self._bucket_name, csv_file_name) - return - - def backup_dat_file(self, dat_file_key: str, datetime_key: str): - # バックアップバケットにコピー - vjsk_backup_bucket = VjskBackupBucket() - dat_key = f'{self._folder}/{dat_file_key}' - backup_key = f'{vjsk_backup_bucket._folder}/{self._folder}/{datetime_key}/{dat_file_key.replace(f"{self._folder}/", "")}' - self._s3_client.copy(self._bucket_name, dat_key, vjsk_backup_bucket._bucket_name, backup_key) - - class ConfigBucket(S3Bucket): _bucket_name = environment.JSKULT_CONFIG_BUCKET @@ -139,7 +116,7 @@ class VjskBackupBucket(JskUltBackupBucket): class VjskReceiveBucket(S3Bucket): - _bucket_name = environment.JSKULT_DATA_BUCKET + _bucket_name = environment.VJSK_DATA_BUCKET _recv_folder = environment.JSKULT_DATA_FOLDER_RECV _s3_file_list = None @@ -176,3 +153,22 @@ class VjskReceiveBucket(S3Bucket): self._s3_client.copy(self._bucket_name, backup_from_file_path, jskult_backup_bucket._bucket_name, backup_key) self._s3_client.delete_file(self._bucket_name, backup_from_file_path) + + +class VjskSendBucket(S3Bucket): + _bucket_name = environment.VJSK_DATA_BUCKET + _send_folder = environment.VJSK_DATA_SEND_FOLDER + + def upload_vjsk_csv_file(self, vjsk_create_csv: str, csv_file_path: str): + # S3バケットにファイルを移動 + csv_file_name = f'{self._send_folder}/{vjsk_create_csv}' + s3_client = S3Client() + s3_client.upload_file(csv_file_path, self._bucket_name, csv_file_name) + return + + def backup_vjsk_csv_file(self, dat_file_key: str, datetime_key: str): + # バックアップバケットにコピー + vjsk_backup_bucket = VjskBackupBucket() + dat_key = f'{self._send_folder}/{dat_file_key}' + backup_key = f'{vjsk_backup_bucket._folder}/{self._send_folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}' + self._s3_client.copy(self._bucket_name, dat_key, vjsk_backup_bucket._bucket_name, backup_key) diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py b/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py index 1fb338b7..f72e8d92 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py @@ -1,6 +1,6 @@ """output_vjsk_inst_pharm_data""" -from src.aws.s3 import VjskBucket +from src.aws.s3 import VjskSendBucket from src.batch.common.batch_context import BatchContext from src.db.database import Database @@ -42,10 +42,10 @@ def exec(): # CSVファイル作成 csv_file_path = make_csv_data(record_inst, record_pharm, vjsk_csv_file_name) - vjsk_bucket = VjskBucket() + vjsk_bucket = VjskSendBucket() try: # s3へデータ移動 - vjsk_bucket.upload_dat_file(vjsk_csv_file_name, csv_file_path) + vjsk_bucket.upload_vjsk_csv_file(vjsk_csv_file_name, csv_file_path) except Exception as e: logger.info('S3バケットにCSVデータを作成できませんでした。') raise e @@ -53,7 +53,7 @@ def exec(): try: # 処理後ファイルをバックアップ batch_context = BatchContext.get_instance() - vjsk_bucket.backup_dat_file(vjsk_csv_file_name, batch_context.syor_date) + vjsk_bucket.backup_vjsk_csv_file(vjsk_csv_file_name, batch_context.syor_date) except Exception as e: logger.info('バックアップバケットへCSVデータをコピーできませんでした。') raise e From a8de16a3bd5492ac1a5538fa77079486e3b3f394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Thu, 6 Jul 2023 13:03:58 +0900 Subject: [PATCH 8/8] =?UTF-8?q?=E6=8C=87=E6=91=98=E4=BA=8B=E9=A0=85?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/aws/s3.py | 4 ++-- .../src/batch/ultmarc/output_vjsk_inst_pharm_data.py | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ecs/jskult-batch-daily/src/aws/s3.py b/ecs/jskult-batch-daily/src/aws/s3.py index 86ffc169..764b600c 100644 --- a/ecs/jskult-batch-daily/src/aws/s3.py +++ b/ecs/jskult-batch-daily/src/aws/s3.py @@ -159,14 +159,14 @@ class VjskSendBucket(S3Bucket): _bucket_name = environment.VJSK_DATA_BUCKET _send_folder = environment.VJSK_DATA_SEND_FOLDER - def upload_vjsk_csv_file(self, vjsk_create_csv: str, csv_file_path: str): + def upload_inst_pharm_csv_file(self, vjsk_create_csv: str, csv_file_path: str): # S3バケットにファイルを移動 csv_file_name = f'{self._send_folder}/{vjsk_create_csv}' s3_client = S3Client() s3_client.upload_file(csv_file_path, self._bucket_name, csv_file_name) return - def backup_vjsk_csv_file(self, dat_file_key: str, datetime_key: str): + def backup_inst_pharm_csv_file(self, dat_file_key: str, datetime_key: str): # バックアップバケットにコピー vjsk_backup_bucket = VjskBackupBucket() dat_key = f'{self._send_folder}/{dat_file_key}' diff --git a/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py b/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py index f72e8d92..838dae34 100644 --- a/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py +++ b/ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py @@ -45,15 +45,14 @@ def exec(): vjsk_bucket = VjskSendBucket() try: # s3へデータ移動 - vjsk_bucket.upload_vjsk_csv_file(vjsk_csv_file_name, csv_file_path) + vjsk_bucket.upload_inst_pharm_csv_file(vjsk_csv_file_name, csv_file_path) except Exception as e: logger.info('S3バケットにCSVデータを作成できませんでした。') raise e try: # 処理後ファイルをバックアップ - batch_context = BatchContext.get_instance() - vjsk_bucket.backup_vjsk_csv_file(vjsk_csv_file_name, batch_context.syor_date) + vjsk_bucket.backup_inst_pharm_csv_file(vjsk_csv_file_name, batch_context.syor_date) except Exception as e: logger.info('バックアップバケットへCSVデータをコピーできませんでした。') raise e @@ -142,7 +141,7 @@ def select_inst_record(db): """ return db.execute_select(sql) except Exception as e: - logger.debug(f'{sql_err_msg}') + logger.debug(sql_err_msg) raise e @@ -220,7 +219,7 @@ def select_pharm_record(db): """ return db.execute_select(sql) except Exception as e: - logger.debug(f'{sql_err_msg}') + logger.debug(sql_err_msg) raise e @@ -266,7 +265,6 @@ def make_csv_data(record_inst: list, record_pharm: list, vjsk_csv_file_name: str writer.writerow(csv_data) except Exception as e: - logger.info('ワークデータの作成に失敗しました。') logger.info('CSVデータの作成に失敗しました。') raise e