From 8f90e4a333d7bd13e5a6497a8623b3b512d35ca1 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Apr 2023 00:41:42 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=89=E3=82=92=E7=B0=A1=E7=95=A5=E5=8C=96=E3=80=82?= =?UTF-8?q?CSV=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=8B=E3=82=89?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=83=87=E3=83=BC=E3=82=BF=E3=80=81?= =?UTF-8?q?=E6=9C=9F=E5=BE=85=E5=80=A4=E3=82=92=E8=AA=AD=E3=82=80=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../table_mapper/com_alma/com_alma_insert.csv | 6 ++ .../com_alma/expect_com_alma_insert.csv | 7 ++ .../com_alma/test_com_alma_mapper.py | 66 ++++++-------- .../tests/testing_utility.py | 85 ++++++++++++++++++- 4 files changed, 121 insertions(+), 43 deletions(-) create mode 100644 ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/com_alma_insert.csv create mode 100644 ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/expect_com_alma_insert.csv diff --git a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/com_alma_insert.csv b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/com_alma_insert.csv new file mode 100644 index 00000000..c01c5605 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/com_alma_insert.csv @@ -0,0 +1,6 @@ +"004","001","A","20141113","20141114","北大","1","2014/11/21 22:53","VANLOAD",,,"2014/11/21 22:53" +"004","002","A","20141113","20141114","札幌医","1","2014/11/21 22:53","VANLOAD",,,"2014/11/21 22:53" +"004","003","A","20141113","20141114","弘大","1","2014/11/21 22:53","VANLOAD",,,"2014/11/21 22:53" +"004","004","A","20141113","20141114","岩手医","1","2014/11/21 22:53","VANLOAD",,,"2014/11/21 22:53" +"004","005","A","20141113","20141114","東北大","1","2014/11/21 22:53","VANLOAD",,,"2014/11/21 22:53" +"004","006","A","20141113","20141114","福島医","1","2014/11/21 22:53","VANLOAD",,,"2014/11/21 22:53" diff --git a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/expect_com_alma_insert.csv b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/expect_com_alma_insert.csv new file mode 100644 index 00000000..8fafdb5f --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/expect_com_alma_insert.csv @@ -0,0 +1,7 @@ +"alma_cd","alma","regist_ymd","update_ymd","delete_ymd","regist_date","create_user","update_date","update_user","sys_regist_date","regist_prgm_id","sys_update_date","update_prgm_id" +"001","北大","20171020","NULL","NULL","NULL","NULL","NULL","NULL","2017/10/20 10:27:33","com_alma_mapper","2017/10/20 10:27:33","com_alma_mapper" +"002","札幌医","20171020","NULL","NULL","NULL","NULL","NULL","NULL","2017/10/20 10:27:33","com_alma_mapper","2017/10/20 10:27:33","com_alma_mapper" +"003","弘大","20171020","NULL","NULL","NULL","NULL","NULL","NULL","2017/10/20 10:27:33","com_alma_mapper","2017/10/20 10:27:33","com_alma_mapper" +"004","岩手医","20171020","NULL","NULL","NULL","NULL","NULL","NULL","2017/10/20 10:27:33","com_alma_mapper","2017/10/20 10:27:33","com_alma_mapper" +"005","東北大","20171020","NULL","NULL","NULL","NULL","NULL","NULL","2017/10/20 10:27:33","com_alma_mapper","2017/10/20 10:27:33","com_alma_mapper" +"006","福島医","20171020","NULL","NULL","NULL","NULL","NULL","NULL","2017/10/20 10:27:33","com_alma_mapper","2017/10/20 10:27:33","com_alma_mapper" diff --git a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/test_com_alma_mapper.py b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/test_com_alma_mapper.py index 5404b43a..174aca65 100644 --- a/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/test_com_alma_mapper.py +++ b/ecs/jskult-batch-daily/tests/batch/ultmarc/utmp_tables/table_mapper/com_alma/test_com_alma_mapper.py @@ -1,3 +1,4 @@ +import os.path as path from datetime import datetime import pytest @@ -7,11 +8,14 @@ from src.batch.ultmarc.utmp_tables.table_mapper.concrete import com_alma_mapper from src.db.database import Database from tests.testing_utility import (assert_table_results, create_delete_sql_with_parameter, + create_expect_data_from_csv, create_insert_sql_with_parameter, create_ultmarc_common_column_names, create_ultmarc_common_column_values, create_ultmarc_table_mapper_sut, - create_ultmarc_test_csv, get_module_name) + create_ultmarc_test_csv, + create_ultmarc_test_data_from_csv, + get_module_name) class TestComAlmaMapper: @@ -19,6 +23,7 @@ class TestComAlmaMapper: db: Database batch_config: BatchConfig + test_file_path: str = path.dirname(__file__) table_columns = ["alma_cd", "alma"] + create_ultmarc_common_column_names() @pytest.fixture(autouse=True, scope='function') @@ -37,12 +42,12 @@ class TestComAlmaMapper: self.db.rollback() self.db.disconnect() - def test_insert_record(self, expect_datetime: datetime, expect_date_str: str): + def test_insert_record(self): """ Cases: COM_出身校テーブルにレコードを登録する Arranges: - - CSVデータを用意する + - CSVデータを用意し、読み込む - 追加対象となるレコードを削除する Expects: - 登録内容が期待値と一致すること @@ -51,56 +56,33 @@ class TestComAlmaMapper: # Arrange # 処理日設定 self.batch_config.syor_date = datetime.strftime(datetime.now(), '%Y/%m/%d') - # テストデータ作成 - csv_floating_data_lst = [ - ['001', '北大', 'A'], - ['002', '札幌医', 'A'], - ['003', '弘大', 'A'], - ['004', '岩手医', 'A'], - ['005', '東北大', 'A'], - ['006', '福島医', 'A'] - ] - test_data_list = [ - f'"004","{data[0]}","{data[2]}","20141113","20141114","{data[1]}","1","2014/11/21 22:53","VANLOAD",,,"2014/11/21 22:53"' - for data in csv_floating_data_lst - ] - test_dat_file = create_ultmarc_test_csv(test_data_list) + # テスト用のCSVを読み込む + test_dat_file = create_ultmarc_test_data_from_csv(path.join(self.test_file_path, 'com_alma_insert.csv')) # 一旦全データをDBから削除 delete_sql, delete_parameter = create_delete_sql_with_parameter('src05.com_alma', {'1': '1'}) self.db.execute(delete_sql, delete_parameter) - # sut(system under test)作成 + # Act for i, line in enumerate(test_dat_file, start=1): sut: com_alma_mapper.ComAlmaMapper = create_ultmarc_table_mapper_sut(line, self.db) assert type(sut) is com_alma_mapper.ComAlmaMapper, f'{i}行目:マッパークラスが期通りか' - # Act sut.make_query() sut.execute_queries() - # Assert - # 期待値作成 - # 期待値となるモジュール名 - module_name = get_module_name(com_alma_mapper) - expect_data = csv_floating_data_lst[i - 1] - expect_row = [expect_data[0], expect_data[1]] + create_ultmarc_common_column_values( - regist_ymd=expect_date_str, - sys_regist_date=expect_datetime, - regist_prgm_id=module_name, - sys_update_date=expect_datetime, - update_prgm_id=module_name - ) - expect_rows = [{c: r for c, r in zip(self.table_columns, expect_row)}] - actual_rows = self.db.execute_select(f"SELECT * FROM src05.com_alma WHERE alma_cd = '{expect_data[0]}'") - - # 期待値検査 - assert_table_results(actual_rows, expect_rows, line_number=i, ignore_col_name=['sys_update_date', 'sys_regist_date']) - - # 動的日付項目の個別確認 - for actual_row, expect_row in zip(actual_rows, expect_rows): - for actual_col_name, expect_col_name in zip(actual_row, expect_row): - if actual_col_name in ['sys_regist_date', 'sys_update_date']: - assert actual_row[actual_col_name] <= expect_row[expect_col_name], f'{i}行目:{actual_col_name}が、期待値の日時以前であること' + # Assert + # 期待値ファイルを読み込む + expect_data_list = create_expect_data_from_csv(path.join(self.test_file_path, 'expect_com_alma_insert.csv')) + primary_keys = [f"'{primary_key['alma_cd']}'" for primary_key in expect_data_list] + actual_select_sql = f"SELECT * FROM src05.com_alma WHERE alma_cd IN ({','.join(primary_keys)})" + actual_data_list = self.db.execute_select(actual_select_sql) + # 期待値検査 + assert_table_results(actual_data_list, expect_data_list, line_number=i, ignore_col_name=['regist_ymd', 'sys_update_date', 'sys_regist_date']) + # 動的日付項目の個別確認 + for actual_row, expect_row in zip(actual_data_list, expect_data_list): + for actual_col_name, expect_col_name in zip(actual_row, expect_row): + if actual_col_name in ['regist_ymd', 'sys_regist_date', 'sys_update_date']: + assert actual_row[actual_col_name] >= expect_row[expect_col_name], f'{i}行目:{actual_col_name}が、期待値の以降であること' def test_update_record(self, expect_datetime: datetime, expect_date_str: str): """ diff --git a/ecs/jskult-batch-daily/tests/testing_utility.py b/ecs/jskult-batch-daily/tests/testing_utility.py index 88934496..032cb53a 100644 --- a/ecs/jskult-batch-daily/tests/testing_utility.py +++ b/ecs/jskult-batch-daily/tests/testing_utility.py @@ -1,6 +1,8 @@ """テスト用共通処理関数""" - +import csv import io +import tempfile +from datetime import datetime from types import ModuleType from src.batch.ultmarc.datfile import DatFile, DatFileLine @@ -11,6 +13,58 @@ from src.batch.ultmarc.utmp_tables.ultmarc_table_mapper_factory import \ from src.db.database import Database +def create_ultmarc_test_data_from_csv(file_path: str) -> DatFile: + """ファイルから、アルトマーク取り込み用のテストデータを作成する + + Args: + file_path (str): csvファイルのパス + + Returns: + DatFile: データファイルオブジェクト + """ + + # 一度、Shift-JISファイルで書き出す + with open(file_path, encoding='utf8') as csv_file, tempfile.NamedTemporaryFile('w', encoding='cp932') as tmp_file: + tmp_file.write(csv_file.read()) + tmp_file.seek(0) + tmpfile_path = tmp_file.name + dat_file = DatFile.from_path(tmpfile_path) + + return dat_file + + +def create_expect_data_from_csv(file_path: str) -> list[dict]: + """ファイルから、DBの期待値データを作成する + + Args: + file_path (str): csvファイルのパス + + Returns: + DatFile: データファイルオブジェクト + """ + + with open(file_path, encoding='utf8') as csv_file: + # ヘッダ行を取得 + header = csv_file.readline().replace('"', '').split(',') + reader = csv.DictReader(csv_file, fieldnames=header) + rows = [r for r in reader] + + # データ型変換 + for row in rows: + for k, v in row.items(): + converted_value = v + if v == 'NULL': + converted_value = None + if is_valid_date(v) is True: + converted_value = datetime.strptime(v, '%Y/%m/%d') + if is_valid_datetime(v) is True: + converted_value = datetime.strptime(v, '%Y/%m/%d %H:%M:%S') + + row[k] = converted_value + + return rows + + def create_ultmarc_test_csv(csv_rows: list[str]) -> DatFile: """アルトマーク取込テストのCSVを作成 Args: @@ -156,6 +210,35 @@ def get_module_name(module: ModuleType) -> str: return module.__name__.split('.')[-1] +def is_valid_date(date_str: str, date_format='%Y/%m/%d'): + """日付文字列が、与えられたフォーマットにマッチするかを検査する + + Args: + date_str (str): 日付文字列 + date_format (str, optional): 日付のフォーマット. Defaults to '%Y/%m/%d'. + + Returns: + _type_: 正しい日付文字列の場合、True、それ以外はFalse + """ + try: + datetime.strptime(date_str, date_format) + return True + except ValueError: + return False + + +def is_valid_datetime(date_str: str): + """日付文字列が、YYYY/MM/DD HH:MM:SSとマッチするかを検査する + + Args: + date_str (str): 日付文字列 + + Returns: + _type_: 正しい日付文字列の場合、True、それ以外はFalse + """ + return is_valid_date(date_str, '%Y/%m/%d %H:%M:%S') + + def assert_table_results(actual_rows: list[dict], expect_rows: list[dict], line_number: int, ignore_col_name: list = None) -> None: """テーブル同士の取得結果突き合わせ