自動テスト実装 異常系

ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/test_vjsk_file_check.py

@@ -10,7 +10,7 @@ def test_check_received_files_ok1(init_check_received_files_ok1):
     観点
         正常系 : 卸在庫データ取込対象日
     期待値
-        例外が発生する
+        例外が発生しない
     """
     batch_context = BatchContext.get_instance()
     batch_context.is_vjsk_stock_import_day = True
@@ -29,7 +29,7 @@ def test_check_received_files_ok2(init_check_received_files_ok2):
     観点
         正常系 : 卸在庫データ取込対象日以外
     期待値
-        例外が発生する
+        例外が発生しない
     """
     batch_context = BatchContext.get_instance()
     batch_context.is_vjsk_stock_import_day = False

ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py

@@ -10,6 +10,7 @@ from src.batch.vjsk.vjsk_data_load_manager import VjskDataLoadManager
 from src.batch.vjsk.vjsk_importer import (_check_received_files,
                                           _import_file_to_db)
 from src.db.database import Database
+from src.error.exceptions import BatchOperationException
 # from tests.testing_vjsk_utility import create_vjsk_assertion_dictionary
 from tests.testing_vjsk_utility import (assert_table_results,
                                         create_vjsk_assertion_list)
@@ -20,6 +21,7 @@ class TestImportFileToDb:
     batch_context: BatchContext
     test_file_path_import_all: str
     test_file_path_load_individual: str
+    test_file_path_unzip_error: str
 
     @pytest.fixture(autouse=True, scope='function')
     def pre_test(self, database: Database):
@@ -27,6 +29,7 @@ class TestImportFileToDb:
         # setup
         self.test_file_path_import_all = path.join(path.dirname(__file__), "testdata", "TestImportFileToDb")
         self.test_file_path_load_individual = path.join(path.dirname(__file__), "testdata")
+        self.test_file_path_unzip_error = path.join(path.dirname(__file__), "testdata", "UnzipError")
 
         self.batch_context = BatchContext.get_instance()
 
@@ -80,41 +83,72 @@ class TestImportFileToDb:
             key = f"{receive_folder}/{test_file}"
             s3_client.upload_file(file_name, bucket_name, key)
 
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_STOCK_SLIP_DATA)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_SLIP_DATA)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_ORG_CNV_MST)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_VOP_HCO_MERGE)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_WHS_MST)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_HLD_MST)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_FCL_MST)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_MKR_ORG_HORIZON)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_TRAN_KBN_MST)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_PHM_PRD_MST)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_PHM_PRICE_MST)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_WHS_CUSTOMER_MST)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_MDB_CONV_MST)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_BIO_SLIP_DATA)}")
-        # self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_LOT_NUM_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_STOCK_SLIP_DATA)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_SLIP_DATA)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_ORG_CNV_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_VOP_HCO_MERGE)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_WHS_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_HLD_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_FCL_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_MKR_ORG_HORIZON)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_TRAN_KBN_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_PHM_PRD_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_PHM_PRICE_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_WHS_CUSTOMER_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_MDB_CONV_MST)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_BIO_SLIP_DATA)}")
+        self.db.execute(f"truncate table {mapper.get_src_table(mapper.CONDKEY_LOT_NUM_MST)}")
 
         # assertion
         received_s3_files = _check_received_files()
         _import_file_to_db(received_s3_files)
 
-        # # 検証 (卸在庫データファイル)
-        # table_name_org = mapper.get_org_table(mapper.CONDKEY_STOCK_SLIP_DATA)
-        # table_name_src = mapper.get_src_table(mapper.CONDKEY_STOCK_SLIP_DATA)
-        # result = self.db.execute(f"select * from {table_name_org}")
-        # assert result.rowcount == 10
-        # result = self.db.execute(f"select * from {table_name_src}")
-        # assert result.rowcount == 10
-
-        # # 検証 (卸販売データ)
-        # table_name_org = mapper.get_org_table(mapper.CONDKEY_SLIP_DATA)
-        # table_name_src = mapper.get_src_table(mapper.CONDKEY_SLIP_DATA)
-        # result = self.db.execute(f"select * from {table_name_org}")
-        # assert result.rowcount == 10
-        # result = self.db.execute(f"select * from {table_name_src}")
-        # assert result.rowcount == 10
+        # 検証
+        condkey = mapper.CONDKEY_STOCK_SLIP_DATA
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_SLIP_DATA
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_ORG_CNV_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_VOP_HCO_MERGE
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_WHS_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_HLD_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_FCL_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_MKR_ORG_HORIZON
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_TRAN_KBN_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_PHM_PRD_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_PHM_PRICE_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_WHS_CUSTOMER_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_MDB_CONV_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_BIO_SLIP_DATA
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
+        condkey = mapper.CONDKEY_LOT_NUM_MST
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_org_table(condkey)}"))
+        assert 10 == len(self.db.execute_select(f"select * from {mapper.get_src_table(condkey)}"))
 
         # teardown
         for test_file in test_files:
@@ -1074,3 +1108,187 @@ class TestImportFileToDb:
         assert result_src_count[0]['count(*)'] == 6
 
         # teardown
+
+    def test_unzip_to_error(self, s3_client, bucket_name, receive_folder, mapper):
+        """
+        観点
+            異常系 : gzファイルが解凍できない
+        期待値
+            例外が発生する
+        """
+        # setup - 卸在庫データ取込対象日
+        self.batch_context.is_vjsk_stock_import_day = True
+
+        # setup - S3受領バケットの内容をすべて削除する
+        vjsk_recv_bucket = VjskReceiveBucket()
+        s3_files = vjsk_recv_bucket.get_s3_file_list()
+        for file_obj in s3_files:
+            s3_client.delete_object(Bucket=bucket_name, Key=file_obj.get("filename"))
+
+        # setup - テスト用受領ファイルをS3受領バケットにupload
+        # ※.gzだが、7zipで圧縮してあるので、解凍に失敗するのが期待値
+        test_files = [
+            "stock_slip_data_202304270000.gz",
+            "slip_data_202304270000.gz",
+            "org_cnv_mst_202304270000.gz",
+            "vop_hco_merge_202304270000.gz",
+            "whs_mst_202304270000.gz",
+            "hld_mst_202304270000.gz",
+            "fcl_mst_202304270000.gz",
+            "mkr_org_horizon_202304270000.gz",
+            "tran_kbn_mst_202304270000.gz",
+            "phm_prd_mst_202304270000.gz",
+            "phm_price_mst_202304270000.gz",
+            "whs_customer_mst_202304270000.gz",
+            "mdb_conv_mst_202304270000.gz",
+            "bio_slip_data_202304270000.gz",
+            "lot_num_mst_202304270000.gz"
+        ]
+        for test_file in test_files:
+            file_name = path.join(self.test_file_path_unzip_error, test_file)
+            key = f"{receive_folder}/{test_file}"
+            s3_client.upload_file(file_name, bucket_name, key)
+
+        # assertion
+        received_s3_files = _check_received_files()
+        with pytest.raises(Exception) as e:
+            _import_file_to_db(received_s3_files)
+
+        # 検証
+        assert str(e.value) == "file could not be opened successfully"
+
+        # teardown
+        for test_file in test_files:
+            key = f"{receive_folder}/{test_file}"
+            s3_client.delete_object(Bucket=bucket_name, Key=key)
+
+    def test_load_data_error(self, mapper):
+        """
+        観点
+            異常系 : 日付型矛盾のデータ ※製品価格マスタファイルで確認
+        期待値
+            例外が発生する
+        """
+
+        # setup
+        self.batch_context.is_vjsk_stock_import_day = True
+
+        # 処理実行
+        target_dict = {
+            "condkey": mapper.CONDKEY_PHM_PRICE_MST,
+            "src_file_path": path.join(self.test_file_path_load_individual, "phm_price_mst_dataerror.tsv")
+        }
+
+        with pytest.raises(BatchOperationException) as e:
+            VjskDataLoadManager.load(target_dict)
+
+        # 検証
+        assert str(e.value).startswith("SQL Error:") > 0
+
+        # teardown
+
+    def test_load_format_error(self, mapper):
+        """
+        観点
+            異常系 : tsvファイルが途中で欠落している
+        期待値
+            例外が発生する
+        """
+
+        # setup
+        self.batch_context.is_vjsk_stock_import_day = True
+
+        # 処理実行
+        target_dict = {
+            "condkey": mapper.CONDKEY_PHM_PRICE_MST,
+            "src_file_path": path.join(self.test_file_path_load_individual, "phm_price_mst_formaterror.tsv")
+        }
+
+        with pytest.raises(BatchOperationException) as e:
+            VjskDataLoadManager.load(target_dict)
+
+        # 検証
+        assert str(e.value).startswith("SQL Error:") > 0
+
+        # teardown
+
+    def test_s3backup_ok(self, s3_client, bucket_name, receive_folder, mapper):
+        """
+        観点
+            正常系 : S3受領ファイルのバックアップフォルダ移動が完了する
+        期待値
+            例外が発生する
+        """
+        # setup - 卸在庫データ取込対象日
+        self.batch_context.is_vjsk_stock_import_day = True
+
+        # setup - S3受領バケットの内容をすべて削除する
+        vjsk_recv_bucket = VjskReceiveBucket()
+        s3_files = vjsk_recv_bucket.get_s3_file_list()
+        for file_obj in s3_files:
+            s3_client.delete_object(Bucket=bucket_name, Key=file_obj.get("filename"))
+
+        # setup - テスト用受領ファイルをS3受領バケットにupload
+        # ※.gzだが、7zipで圧縮してあるので、解凍に失敗するのが期待値
+        test_files = [
+            "stock_slip_data_202304270000.gz",
+            "slip_data_202304270000.gz",
+            "org_cnv_mst_202304270000.gz",
+            "vop_hco_merge_202304270000.gz",
+            "whs_mst_202304270000.gz",
+            "hld_mst_202304270000.gz",
+            "fcl_mst_202304270000.gz",
+            "mkr_org_horizon_202304270000.gz",
+            "tran_kbn_mst_202304270000.gz",
+            "phm_prd_mst_202304270000.gz",
+            "phm_price_mst_202304270000.gz",
+            "whs_customer_mst_202304270000.gz",
+            "mdb_conv_mst_202304270000.gz",
+            "bio_slip_data_202304270000.gz",
+            "lot_num_mst_202304270000.gz"
+        ]
+        for test_file in test_files:
+            file_name = path.join(self.test_file_path_import_all, test_file)
+            key = f"{receive_folder}/{test_file}"
+            s3_client.upload_file(file_name, bucket_name, key)
+
+        # assertion
+        received_s3_files = _check_received_files()
+        vjsk_recv_bucket.backup_dat_file(received_s3_files, "test")
+
+        # 検証
+
+        # teardown
+        for test_file in test_files:
+            key = f"{receive_folder}/{test_file}"
+            s3_client.delete_object(Bucket=bucket_name, Key=key)
+
+    def test_s3backup_to_error(self, s3_client, bucket_name, receive_folder, mapper):
+        """
+        観点
+            異常系 : S3受領ファイルのバックアップフォルダ移動ができない
+        期待値
+            例外が発生する
+        """
+        # setup - 卸在庫データ取込対象日
+        self.batch_context.is_vjsk_stock_import_day = True
+
+        # setup - S3受領バケットの内容をすべて削除する
+        vjsk_recv_bucket = VjskReceiveBucket()
+        s3_files = vjsk_recv_bucket.get_s3_file_list()
+        for file_obj in s3_files:
+            s3_client.delete_object(Bucket=bucket_name, Key=file_obj.get("filename"))
+
+        # setup
+
+        # assertion
+        with pytest.raises(Exception) as e:
+            # 有りもしないファイルをバックアップフォルダにコピーさせてコケさせる
+            received_s3_files = []
+            received_s3_files.append({"filename": "dummy.dummy"})
+            vjsk_recv_bucket.backup_dat_file(received_s3_files, "test")
+
+        # 検証
+        assert str(e.value) == "An error occurred (404) when calling the HeadObject operation: Not Found"
+
+        # teardown

ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_dataerror.tsv

@@ -0,0 +1,3 @@
+"phm_prd_cd"	"phm_price_kind"	"sub_no"	"price"	"start_dt"	"end_dt"	"dsp_odr"	"rec_sts_kbn"	"ins_dt"	"upd_dt"
+"123456701"	"01"	"1"	"12345.6"	"yyyy0401"	"20190930"	"10"	"0"	"18-03-07 09:48:00"	"19-09-19 11:23:47"
+"123456701"	"02"	"1"	"12587.8"	"20191001"	"99991231"	"10"	"0"	"19-09-19 11:24:05"	"19-09-19 11:24:05"

ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_formaterror.tsv

@@ -0,0 +1,2 @@
+"phm_prd_cd"	"phm_price_kind"	"sub_no"	"price"	"start_dt"	"end_dt"	"dsp_odr"	"rec_sts_kbn"	"ins_dt"	"upd_dt"
+"123456701"	"01"	"1"	"12345.6"	"202304