自動テスト実装 卸在庫データ
This commit is contained in:
x.azuma.m@nds-tyo.co.jp
parent
9c8e3d1bac
commit
368fef1f5b
@ -4,22 +4,28 @@ import pytest
|
||||
|
||||
from src.aws.s3 import VjskReceiveBucket
|
||||
from src.batch.common.batch_context import BatchContext
|
||||
from src.batch.vjsk.vjsk_data_load_manager import VjskDataLoadManager
|
||||
# from src.batch.vjsk.vjsk_data_load_manager import VjskDataLoadManager
|
||||
from src.batch.vjsk.vjsk_importer import (_check_received_files,
|
||||
_import_file_to_db)
|
||||
from src.db.database import Database
|
||||
# from tests.testing_vjsk_utility import create_vjsk_assertion_dictionary
|
||||
from tests.testing_vjsk_utility import (assert_table_results,
|
||||
create_vjsk_assertion_list)
|
||||
|
||||
|
||||
class TestImportFileToDb:
|
||||
db: Database
|
||||
batch_context: BatchContext
|
||||
test_file_path: str
|
||||
test_file_path_import_all: str
|
||||
test_file_path_load_individual: str
|
||||
|
||||
@pytest.fixture(autouse=True, scope='function')
|
||||
def pre_test(self, database: Database):
|
||||
"""テスト実行前後処理"""
|
||||
# setup
|
||||
self.test_file_path = path.join(path.dirname(__file__), "testdata")
|
||||
self.test_file_path_import_all = path.join(path.dirname(__file__), "testdata", "TestImportFileToDb")
|
||||
self.test_file_path_load_individual = path.join(path.dirname(__file__), "testdata")
|
||||
|
||||
self.batch_context = BatchContext.get_instance()
|
||||
|
||||
@ -69,7 +75,7 @@ class TestImportFileToDb:
|
||||
"lot_num_mst_202304270000.gz"
|
||||
]
|
||||
for test_file in test_files:
|
||||
file_name = path.join(self.test_file_path, test_file)
|
||||
file_name = path.join(self.test_file_path_import_all, test_file)
|
||||
key = f"{receive_folder}/{test_file}"
|
||||
s3_client.upload_file(file_name, bucket_name, key)
|
||||
|
||||
@ -93,8 +99,6 @@ class TestImportFileToDb:
|
||||
received_s3_files = _check_received_files()
|
||||
_import_file_to_db(received_s3_files)
|
||||
|
||||
# self.db.connect()
|
||||
|
||||
# # 検証 (卸在庫データファイル)
|
||||
# table_name_org = mapper.get_org_table(mapper.CONDKEY_STOCK_SLIP_DATA)
|
||||
# table_name_src = mapper.get_src_table(mapper.CONDKEY_STOCK_SLIP_DATA)
|
||||
@ -116,36 +120,65 @@ class TestImportFileToDb:
|
||||
key = f"{receive_folder}/{test_file}"
|
||||
s3_client.delete_object(Bucket=bucket_name, Key=key)
|
||||
|
||||
# def test_load_stock_slip_data_ok(self, mapper):
|
||||
# table_name_org = mapper.get_org_table(mapper.CONDKEY_SLIP_DATA)
|
||||
# table_name_src = mapper.get_src_table(mapper.CONDKEY_SLIP_DATA)
|
||||
def test_load_01_stock_slip_data_ok(self, mapper):
|
||||
table_name_org = mapper.get_org_table(mapper.CONDKEY_STOCK_SLIP_DATA)
|
||||
table_name_src = mapper.get_src_table(mapper.CONDKEY_STOCK_SLIP_DATA)
|
||||
|
||||
# # setup
|
||||
# self.batch_context.is_vjsk_stock_import_day = True
|
||||
# self.db.execute(f"truncate table {table_name_src}")
|
||||
# setup
|
||||
self.batch_context.is_vjsk_stock_import_day = True
|
||||
self.db.execute(f"truncate table {table_name_src}")
|
||||
|
||||
# # assertion (insert)
|
||||
# target_dict = {
|
||||
# "condkey": mapper.CONDKEY_STOCK_SLIP_DATA,
|
||||
# "src_file_path": path.join(self.test_file_path, "stock_slip_data_202304280000.tsv")
|
||||
# }
|
||||
# VjskDataLoadManager.load(target_dict)
|
||||
# assertion1 (insert 4row)
|
||||
|
||||
# result = self.db.execute(f"select * from {table_name_org}")
|
||||
# assert result.rowcount == 4
|
||||
# result = self.db.execute(f"select * from {table_name_src}")
|
||||
# assert result.rowcount == 4
|
||||
# 処理実行
|
||||
target_dict = {
|
||||
"condkey": mapper.CONDKEY_STOCK_SLIP_DATA,
|
||||
"src_file_path": path.join(self.test_file_path_load_individual, "stock_slip_data_202304280000.tsv")
|
||||
}
|
||||
VjskDataLoadManager.load(target_dict)
|
||||
|
||||
# # assertion (update)
|
||||
# target_dict = {
|
||||
# "condkey": mapper.CONDKEY_STOCK_SLIP_DATA,
|
||||
# "src_file_path": path.join(self.test_file_path, "stock_slip_data_202304290000.tsv")
|
||||
# }
|
||||
# VjskDataLoadManager.load(target_dict)
|
||||
# 期待値データファイル読み込み
|
||||
assert_list = create_vjsk_assertion_list(target_dict["src_file_path"])
|
||||
# orgテーブル結果を取得
|
||||
result_org = self.db.execute_select(f"select * from {table_name_org}")
|
||||
# 突合から除外する項目
|
||||
ignore_columns = ['dwh_upd_dt']
|
||||
# orgテーブル結果が期待値通りかを突合
|
||||
assert_table_results(result_org, assert_list, ignore_columns)
|
||||
# srcテーブル結果を取得
|
||||
result_src = self.db.execute_select(f"select * from {table_name_src}")
|
||||
# 突合から除外する項目
|
||||
ignore_columns = ['dwh_upd_dt']
|
||||
# srcテーブル結果が期待値通りかを突合
|
||||
assert_table_results(result_src, assert_list, ignore_columns)
|
||||
|
||||
# result_org = self.db.execute(f"select * from {table_name_org}")
|
||||
# assert result_org.rowcount == 4
|
||||
# result_src1 = self.db.execute(f"select * from {table_name_src}")
|
||||
# assert result_src1.rowcount == 6
|
||||
# assertion2 (update 2row +insert 2row)
|
||||
|
||||
# # teardown
|
||||
# 処理実行
|
||||
target_dict = {
|
||||
"condkey": mapper.CONDKEY_STOCK_SLIP_DATA,
|
||||
"src_file_path": path.join(self.test_file_path_load_individual, "stock_slip_data_202304290000.tsv")
|
||||
}
|
||||
VjskDataLoadManager.load(target_dict)
|
||||
|
||||
# 期待値データファイル読み込み
|
||||
assert_list = create_vjsk_assertion_list(target_dict["src_file_path"])
|
||||
# orgテーブル結果を取得
|
||||
result_org = self.db.execute_select(f"select * from {table_name_org}")
|
||||
# 突合から除外する項目
|
||||
ignore_columns = ['dwh_upd_dt']
|
||||
# orgテーブル結果が期待値通りかを突合
|
||||
assert_table_results(result_org, assert_list, ignore_columns)
|
||||
# srcテーブル結果(orgテーブル結果のPK値で一致するもの)を取得
|
||||
result_src = self.db.execute_select(
|
||||
f"select * from {table_name_src} s inner join {table_name_org} o on (s.slip_mgt_num = o.slip_mgt_num)")
|
||||
# 突合から除外する項目
|
||||
ignore_columns = ['dwh_upd_dt']
|
||||
# srcテーブル結果が期待値通りかを突合
|
||||
assert_table_results(result_src, assert_list, ignore_columns)
|
||||
|
||||
# srcテーブル結果のレコード件数 (insert 4row + update 2row + insert 2row = 6row)
|
||||
result_src_count = self.db.execute_select(f"select count(*) from {table_name_src} ")
|
||||
assert result_src_count[0]['count(*)'] == 6
|
||||
|
||||
# teardown
|
||||
|
||||
5
ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304280000.tsv
vendored
Normal file
5
ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304280000.tsv
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_sto_place" "rec_stock_ymd" "rec_comm_cd" "rec_amt" "rev_stok_no_sign" "rev_jan_cd" "rec_free_item" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "exec_dt" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "rec_sts_kbn" "ins_dt" "ins_usr"
|
||||
"D463630101 23022849630021900003500000 ｾﾄﾛﾀｲﾄﾞﾁﾕｳｼﾔﾖｳ0.25MG 1V" "363" "01" "01 " "230228" "496300219" "000035" "0" "0000" " ｾﾄﾛﾀｲﾄﾞﾁﾕｳｼﾔﾖｳ0.25MG 1V" "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000059" "59" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM"
|
||||
"D4625301026 2302284963001270000040000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "253" "01" "026 " "230228" "496300127" "000004" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000060" "60" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM"
|
||||
"D4625301026 2302284963001340000220000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "253" "01" "026 " "230228" "496300134" "000022" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000061" "61" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM"
|
||||
"D4625301026 2302284963004170000500000001ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "253" "01" "026 " "230228" "496300417" "000050" "0" "0000" "01ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000062" "62" "202303142041" "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "23-03-14 20:41:57" "SYSTEM"
|
||||
|
5
ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304290000.tsv
vendored
Normal file
5
ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/stock_slip_data_202304290000.tsv
vendored
Normal file
@ -0,0 +1,5 @@
|
||||
"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_sto_place" "rec_stock_ymd" "rec_comm_cd" "rec_amt" "rev_stok_no_sign" "rev_jan_cd" "rec_free_item" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "exec_dt" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "rec_sts_kbn" "ins_dt" "ins_usr"
|
||||
"DAY2-301026 2302284963001340000220000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "253" "01" "026 " "230228" "496300134" "000022" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000061" "61" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM"
|
||||
"DAY2-301026 2302284963004170000500000001ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "253" "01" "026 " "230228" "496300417" "000050" "0" "0000" "01ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000062" "62" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM"
|
||||
"DAY2-301027 2302284963001100000020000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ300 1ﾄｳ40 " "253" "01" "027 " "230228" "496300110" "000002" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ300 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000063" "63" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM"
|
||||
"DAY2-301027 2302284963001270000110000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "253" "01" "027 " "230228" "496300127" "000011" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000064" "64" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM"
|
||||
|
91
ecs/jskult-batch-daily/tests/testing_vjsk_utility.py
Normal file
91
ecs/jskult-batch-daily/tests/testing_vjsk_utility.py
Normal file
@ -0,0 +1,91 @@
|
||||
|
||||
|
||||
import csv
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def create_vjsk_assertion_list(file_path: str) -> list:
|
||||
"""DB登録期待値リストを作成する
|
||||
|
||||
Args:
|
||||
file_path (str): DB登録期待値ファイル(tsvファイル)のパス
|
||||
※DB登録期待値ファイルの前提
|
||||
受領データファイルと同じ
|
||||
BOM付きtsv形式
|
||||
一行目はカラム名になっているヘッダ行
|
||||
|
||||
Returns:
|
||||
List(dict) DB登録期待値辞書リスト
|
||||
"""
|
||||
with open(file_path, encoding='utf_8_sig', newline='') as tsv_file:
|
||||
header = tsv_file.readline().strip('\n').replace('"', '').split('\t')
|
||||
reader = csv.DictReader(tsv_file, fieldnames=header, delimiter='\t')
|
||||
rows = [r for r in reader]
|
||||
|
||||
# DB抽出値と比較できるように、リテラル値をDB抽出値と同じデータフォーマットに変換
|
||||
for row in rows:
|
||||
for k, v in row.items():
|
||||
converted_value = v
|
||||
if v == 'NULL':
|
||||
converted_value = None
|
||||
if is_valid_date_format(v, '%Y/%m/%d') is True: # YYYY/MM/DD
|
||||
converted_value = datetime.strptime(v, '%Y/%m/%d').date()
|
||||
if is_valid_date_format(v, '%Y-%m-%d') is True: # YYYY-MM-DD
|
||||
converted_value = datetime.strptime(v, '%Y-%m-%d').date()
|
||||
if is_valid_date_format(v, '%Y/%m/%d %H:%M:%S') is True: # YYYY/MM/DD HH:MM:SS
|
||||
converted_value = datetime.strptime(v, '%Y/%m/%d %H:%M:%S')
|
||||
if is_valid_date_format(v, '%Y-%m-%d %H:%M:%S') is True: # YYYY-MM-DD HH:MM:SS
|
||||
converted_value = datetime.strptime(v, '%Y-%m-%d %H:%M:%S')
|
||||
if is_valid_date_format(v, '%y-%m-%d %H:%M:%S') is True: # YY-MM-DD HH:MM:SS
|
||||
converted_value = datetime.strptime(v, '%y-%m-%d %H:%M:%S')
|
||||
|
||||
row[k] = converted_value
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def is_valid_date_format(date_str: str, date_format):
|
||||
"""日付文字列が、与えられたフォーマットにマッチするかを検査する
|
||||
|
||||
Args:
|
||||
date_str (str): 日付文字列
|
||||
date_format (str, optional): 日付のフォーマット
|
||||
|
||||
Returns:
|
||||
_type_: 正しい日付文字列の場合、True、それ以外はFalse
|
||||
"""
|
||||
try:
|
||||
datetime.strptime(date_str, date_format)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def assert_table_results(actual_rows: list[dict], expect_rows: list[dict], ignore_col_name: list = None) -> None:
|
||||
"""テーブル同士の取得結果突き合わせ
|
||||
|
||||
Args:
|
||||
actual_rows (list[dict]): テスト結果の辞書リスト
|
||||
expect_rows (list[dict]): 期待値の辞書リスト
|
||||
ignore_col_name (list): 比較を無視するDBのカラム名. Default None.
|
||||
"""
|
||||
# 取得件数が一致すること
|
||||
assert len(actual_rows) == len(expect_rows)
|
||||
|
||||
line_number = 0
|
||||
# 1行ずつ調査
|
||||
for actual_row, expect_row in zip(actual_rows, expect_rows):
|
||||
line_number += 1
|
||||
# 1カラムずつ調査
|
||||
for actual_col_name, expect_col_name in zip(actual_row, expect_row):
|
||||
# テストメソッド側で個別に確認するものはスキップさせる
|
||||
if ignore_col_name is not None and actual_col_name in ignore_col_name:
|
||||
continue
|
||||
else:
|
||||
actual_value = actual_row[actual_col_name]
|
||||
expect_value = expect_row[expect_col_name]
|
||||
if isinstance(actual_value, (int)):
|
||||
expect_value = int(expect_value)
|
||||
if isinstance(actual_value, (float)):
|
||||
expect_value = float(expect_value)
|
||||
assert actual_value == expect_value, f'{line_number}行目:{actual_col_name}が、期待値と一致しませんでした'
|
||||
Loading…
x
Reference in New Issue
Block a user