diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py index 8dac0688..e1d3571d 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py @@ -4,22 +4,28 @@ import pytest from src.aws.s3 import VjskReceiveBucket from src.batch.common.batch_context import BatchContext +from src.batch.vjsk.vjsk_data_load_manager import VjskDataLoadManager # from src.batch.vjsk.vjsk_data_load_manager import VjskDataLoadManager from src.batch.vjsk.vjsk_importer import (_check_received_files, _import_file_to_db) from src.db.database import Database +# from tests.testing_vjsk_utility import create_vjsk_assertion_dictionary +from tests.testing_vjsk_utility import (assert_table_results, + create_vjsk_assertion_list) class TestImportFileToDb: db: Database batch_context: BatchContext - test_file_path: str + test_file_path_import_all: str + test_file_path_load_individual: str @pytest.fixture(autouse=True, scope='function') def pre_test(self, database: Database): """テスト実行前後処理""" # setup - self.test_file_path = path.join(path.dirname(__file__), "testdata") + self.test_file_path_import_all = path.join(path.dirname(__file__), "testdata", "TestImportFileToDb") + self.test_file_path_load_individual = path.join(path.dirname(__file__), "testdata") self.batch_context = BatchContext.get_instance() @@ -69,7 +75,7 @@ class TestImportFileToDb: "lot_num_mst_202304270000.gz" ] for test_file in test_files: - file_name = path.join(self.test_file_path, test_file) + file_name = path.join(self.test_file_path_import_all, test_file) key = f"{receive_folder}/{test_file}" s3_client.upload_file(file_name, bucket_name, key) @@ -93,8 +99,6 @@ class TestImportFileToDb: received_s3_files = _check_received_files() _import_file_to_db(received_s3_files) - # self.db.connect() - # # 検証 (卸在庫データファイル) # table_name_org = mapper.get_org_table(mapper.CONDKEY_STOCK_SLIP_DATA) # table_name_src = mapper.get_src_table(mapper.CONDKEY_STOCK_SLIP_DATA) @@ -116,36 +120,65 @@ class TestImportFileToDb: key = f"{receive_folder}/{test_file}" s3_client.delete_object(Bucket=bucket_name, Key=key) - # def test_load_stock_slip_data_ok(self, mapper): - # table_name_org = mapper.get_org_table(mapper.CONDKEY_SLIP_DATA) - # table_name_src = mapper.get_src_table(mapper.CONDKEY_SLIP_DATA) + def test_load_01_stock_slip_data_ok(self, mapper): + table_name_org = mapper.get_org_table(mapper.CONDKEY_STOCK_SLIP_DATA) + table_name_src = mapper.get_src_table(mapper.CONDKEY_STOCK_SLIP_DATA) - # # setup - # self.batch_context.is_vjsk_stock_import_day = True - # self.db.execute(f"truncate table {table_name_src}") + # setup + self.batch_context.is_vjsk_stock_import_day = True + self.db.execute(f"truncate table {table_name_src}") - # # assertion (insert) - # target_dict = { - # "condkey": mapper.CONDKEY_STOCK_SLIP_DATA, - # "src_file_path": path.join(self.test_file_path, "stock_slip_data_202304280000.tsv") - # } - # VjskDataLoadManager.load(target_dict) + # assertion1 (insert 4row) - # result = self.db.execute(f"select * from {table_name_org}") - # assert result.rowcount == 4 - # result = self.db.execute(f"select * from {table_name_src}") - # assert result.rowcount == 4 + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_STOCK_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, "stock_slip_data_202304280000.tsv") + } + VjskDataLoadManager.load(target_dict) - # # assertion (update) - # target_dict = { - # "condkey": mapper.CONDKEY_STOCK_SLIP_DATA, - # "src_file_path": path.join(self.test_file_path, "stock_slip_data_202304290000.tsv") - # } - # VjskDataLoadManager.load(target_dict) + # 期待値データファイル読み込み + assert_list = create_vjsk_assertion_list(target_dict["src_file_path"]) + # orgテーブル結果を取得 + result_org = self.db.execute_select(f"select * from {table_name_org}") + # 突合から除外する項目 + ignore_columns = ['dwh_upd_dt'] + # orgテーブル結果が期待値通りかを突合 + assert_table_results(result_org, assert_list, ignore_columns) + # srcテーブル結果を取得 + result_src = self.db.execute_select(f"select * from {table_name_src}") + # 突合から除外する項目 + ignore_columns = ['dwh_upd_dt'] + # srcテーブル結果が期待値通りかを突合 + assert_table_results(result_src, assert_list, ignore_columns) - # result_org = self.db.execute(f"select * from {table_name_org}") - # assert result_org.rowcount == 4 - # result_src1 = self.db.execute(f"select * from {table_name_src}") - # assert result_src1.rowcount == 6 + # assertion2 (update 2row +insert 2row) - # # teardown + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_STOCK_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, "stock_slip_data_202304290000.tsv") + } + VjskDataLoadManager.load(target_dict) + + # 期待値データファイル読み込み + assert_list = create_vjsk_assertion_list(target_dict["src_file_path"]) + # orgテーブル結果を取得 + result_org = self.db.execute_select(f"select * from {table_name_org}") + # 突合から除外する項目 + ignore_columns = ['dwh_upd_dt'] + # orgテーブル結果が期待値通りかを突合 + assert_table_results(result_org, assert_list, ignore_columns) + # srcテーブル結果(orgテーブル結果のPK値で一致するもの)を取得 + result_src = self.db.execute_select( + f"select * from {table_name_src} s inner join {table_name_org} o on (s.slip_mgt_num = o.slip_mgt_num)") + # 突合から除外する項目 + ignore_columns = ['dwh_upd_dt'] + # srcテーブル結果が期待値通りかを突合 + assert_table_results(result_src, assert_list, ignore_columns) + + # srcテーブル結果のレコード件数 (insert 4row + update 2row + insert 2row = 6row) + result_src_count = self.db.execute_select(f"select count(*) from {table_name_src} ") + assert result_src_count[0]['count(*)'] == 6 + + # teardown diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304280000.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304280000.tsv new file mode 100644 index 00000000..cc6c3902 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304280000.tsv @@ -0,0 +1,5 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_sto_place" "rec_stock_ymd" "rec_comm_cd" "rec_amt" "rev_stok_no_sign" "rev_jan_cd" "rec_free_item" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "exec_dt" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "rec_sts_kbn" "ins_dt" "ins_usr" +"D463630101 23022849630021900003500000 ｾﾄﾛﾀｲﾄﾞﾁﾕｳｼﾔﾖｳ0.25MG 1V" "363" "01" "01 " "230228" "496300219" "000035" "0" "0000" " ｾﾄﾛﾀｲﾄﾞﾁﾕｳｼﾔﾖｳ0.25MG 1V" "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000059" "59" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"D4625301026 2302284963001270000040000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "253" "01" "026 " "230228" "496300127" "000004" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000060" "60" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"D4625301026 2302284963001340000220000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "253" "01" "026 " "230228" "496300134" "000022" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000061" "61" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"D4625301026 2302284963004170000500000001ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "253" "01" "026 " "230228" "496300417" "000050" "0" "0000" "01ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000062" "62" "202303142041" "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "23-03-14 20:41:57" "SYSTEM" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304290000.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304290000.tsv new file mode 100644 index 00000000..0b180c24 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304290000.tsv @@ -0,0 +1,5 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_sto_place" "rec_stock_ymd" "rec_comm_cd" "rec_amt" "rev_stok_no_sign" "rev_jan_cd" "rec_free_item" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "exec_dt" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "rec_sts_kbn" "ins_dt" "ins_usr" +"DAY2-301026 2302284963001340000220000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "253" "01" "026 " "230228" "496300134" "000022" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000061" "61" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"DAY2-301026 2302284963004170000500000001ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "253" "01" "026 " "230228" "496300417" "000050" "0" "0000" "01ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000062" "62" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"DAY2-301027 2302284963001100000020000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ300 1ﾄｳ40 " "253" "01" "027 " "230228" "496300110" "000002" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ300 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000063" "63" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"DAY2-301027 2302284963001270000110000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "253" "01" "027 " "230228" "496300127" "000011" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000064" "64" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" diff --git a/ecs/jskult-batch-daily/tests/testing_vjsk_utility.py b/ecs/jskult-batch-daily/tests/testing_vjsk_utility.py new file mode 100644 index 00000000..1849e325 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/testing_vjsk_utility.py @@ -0,0 +1,91 @@ + + +import csv +from datetime import datetime + + +def create_vjsk_assertion_list(file_path: str) -> list: + """DB登録期待値リストを作成する + + Args: + file_path (str): DB登録期待値ファイル(tsvファイル)のパス + ※DB登録期待値ファイルの前提 + 受領データファイルと同じ + BOM付きtsv形式 + 一行目はカラム名になっているヘッダ行 + + Returns: + List(dict) DB登録期待値辞書リスト + """ + with open(file_path, encoding='utf_8_sig', newline='') as tsv_file: + header = tsv_file.readline().strip('\n').replace('"', '').split('\t') + reader = csv.DictReader(tsv_file, fieldnames=header, delimiter='\t') + rows = [r for r in reader] + + # DB抽出値と比較できるように、リテラル値をDB抽出値と同じデータフォーマットに変換 + for row in rows: + for k, v in row.items(): + converted_value = v + if v == 'NULL': + converted_value = None + if is_valid_date_format(v, '%Y/%m/%d') is True: # YYYY/MM/DD + converted_value = datetime.strptime(v, '%Y/%m/%d').date() + if is_valid_date_format(v, '%Y-%m-%d') is True: # YYYY-MM-DD + converted_value = datetime.strptime(v, '%Y-%m-%d').date() + if is_valid_date_format(v, '%Y/%m/%d %H:%M:%S') is True: # YYYY/MM/DD HH:MM:SS + converted_value = datetime.strptime(v, '%Y/%m/%d %H:%M:%S') + if is_valid_date_format(v, '%Y-%m-%d %H:%M:%S') is True: # YYYY-MM-DD HH:MM:SS + converted_value = datetime.strptime(v, '%Y-%m-%d %H:%M:%S') + if is_valid_date_format(v, '%y-%m-%d %H:%M:%S') is True: # YY-MM-DD HH:MM:SS + converted_value = datetime.strptime(v, '%y-%m-%d %H:%M:%S') + + row[k] = converted_value + + return rows + + +def is_valid_date_format(date_str: str, date_format): + """日付文字列が、与えられたフォーマットにマッチするかを検査する + + Args: + date_str (str): 日付文字列 + date_format (str, optional): 日付のフォーマット + + Returns: + _type_: 正しい日付文字列の場合、True、それ以外はFalse + """ + try: + datetime.strptime(date_str, date_format) + return True + except ValueError: + return False + + +def assert_table_results(actual_rows: list[dict], expect_rows: list[dict], ignore_col_name: list = None) -> None: + """テーブル同士の取得結果突き合わせ + + Args: + actual_rows (list[dict]): テスト結果の辞書リスト + expect_rows (list[dict]): 期待値の辞書リスト + ignore_col_name (list): 比較を無視するDBのカラム名. Default None. + """ + # 取得件数が一致すること + assert len(actual_rows) == len(expect_rows) + + line_number = 0 + # 1行ずつ調査 + for actual_row, expect_row in zip(actual_rows, expect_rows): + line_number += 1 + # 1カラムずつ調査 + for actual_col_name, expect_col_name in zip(actual_row, expect_row): + # テストメソッド側で個別に確認するものはスキップさせる + if ignore_col_name is not None and actual_col_name in ignore_col_name: + continue + else: + actual_value = actual_row[actual_col_name] + expect_value = expect_row[expect_col_name] + if isinstance(actual_value, (int)): + expect_value = int(expect_value) + if isinstance(actual_value, (float)): + expect_value = float(expect_value) + assert actual_value == expect_value, f'{line_number}行目:{actual_col_name}が、期待値と一致しませんでした'