diff --git a/ecs/jskult-batch-daily/Pipfile b/ecs/jskult-batch-daily/Pipfile index 1e6adf91..a40e6c17 100644 --- a/ecs/jskult-batch-daily/Pipfile +++ b/ecs/jskult-batch-daily/Pipfile @@ -6,6 +6,8 @@ name = "pypi" [scripts] "test:ultmarc" = "pytest tests/batch/ultmarc/" "test:ultmarc:cov" = "pytest --cov=src/batch/ultmarc/ --cov-branch --cov-report=term-missing tests/batch/ultmarc/" +"test:vjsk" = "pytest tests/batch/vjsk/" +"test:vjsk:cov" = "pytest --cov=src/batch/vjsk/ --cov-branch --cov-report=term-missing tests/batch/vjsk/" [packages] boto3 = "*" diff --git a/ecs/jskult-batch-daily/README.md b/ecs/jskult-batch-daily/README.md index d0f37149..144cf9b8 100644 --- a/ecs/jskult-batch-daily/README.md +++ b/ecs/jskult-batch-daily/README.md @@ -44,7 +44,7 @@ - 「entrypoint.py」が、バッチ処理のエントリーポイント。 - 実際の処理は、「src/jobctrl_daily.py」で行っている。 -## 単体テスト +## 単体テスト(アルトマーク取込処理) アルトマーク取込処理は、単体テストコードを使用してテスト自動化を行う @@ -68,7 +68,7 @@ | コマンド | 概要 | | ---------------- | -------------------------------------------------------------------------------------------- | | test:ultmarc | tests/batch/ultmarc フォルダ配下のユニットテストを実行する | -| test:ultmarc:cov | tests/batch/ultmarc フォルダ配下のユニットテストを実行し、テストカバレッジを取得する(C1, C2) | +| test:ultmarc:cov | tests/batch/ultmarc フォルダ配下のユニットテストを実行し、テストカバレッジを取得する(C0, C1) | ### テスト共通関数の仕様 @@ -156,6 +156,43 @@ - テスト結果データと期待値データを突き合わせ、期待値どおりとなっているかを確認する - ignore_col_nameに指定したカラムは、呼び出し元のテストコード内で個別に突き合わせする + +## 単体テスト(実消化データ取込処理) + +実消化データは、単体テストコードを使用してテスト自動化を行う + +### テスト準備 + +※単体テスト(アルトマーク取込処理)と同じ + +### テスト用のサブコマンド一覧 + +- `pipenv run`のあとに、サブコマンドとしてユーザー定義スクリプトを実行できる + - `Pipfile`内の「scripts」セクションに宣言されている + +| コマンド | 概要 | +| ---------------- | -------------------------------------------------------------------------------------------- | +| test:vjsk | tests/batch/vjsk フォルダ配下のユニットテストを実行する | +| test:vjsk:cov | tests/batch/vjsk フォルダ配下のユニットテストを実行し、テストカバレッジを取得する(C0, C1) | + +### テスト共通関数の仕様 + +- tests/testing_vjsk_utility.py内の共通関数の仕様について記載する + +#### create_vjsk_assertion_list + +- 概要 + - DB登録期待値リストを作成する +- Args: + - file_path (str): DB登録期待値ファイル(tsvファイル)のパス + - memo: ※DB登録期待値ファイルの前提 + - memo: 受領データファイルと同じ + - memo: BOM付きtsv形式 + - memo: 一行目はカラム名になっているヘッダ行 + - Returns: + - List(dict) DB登録期待値辞書リスト + + ## フォルダ構成 ```text @@ -203,16 +240,53 @@ └── tests -- ユニットテストのルートディレクト ├── batch │ └── ultmarc -- アルトマーク関連のユニットテストを格納する - │ └── utmp_tables - │ └── table_mapper -- 以下、マッパークラス単位でフォルダを切る - │ └── com_alma - │ ├── test_com_alma_mapper.py -- テストコード本体 - │ ├── com_alma_insert.csv -- S3に配置される想定のテストCSVデータ。ケースごとに用意する。 - │ ... - │ ├── db_com_alma_before_update.csv -- テスト時に事前にDBに登録しておくデータ。CSVで用意する。 - │ ... - │ ├── expect_com_alma_insert.csv -- テストの期待値データ。CSVで用意する。 - │ ... + │ │ └── utmp_tables + │ │ └── table_mapper -- 以下、マッパークラス単位でフォルダを切る + │ │ └── com_alma + │ │ ├── test_com_alma_mapper.py -- テストコード本体 + │ │ ├── com_alma_insert.csv -- S3に配置される想定のテストCSVデータ。ケースごとに用意する。 + │ │ ... + │ │ ├── db_com_alma_before_update.csv -- テスト時に事前にDBに登録しておくデータ。CSVで用意する。 + │ │ ... + │ │ ├── expect_com_alma_insert.csv -- テストの期待値データ。CSVで用意する。 + │ │ ... + │ └─vjsk -- 実消化データ取込処理関連のユニットテストを格納する + │ │ + │ ├─vjsk_file_check -- 受領ファイルチェック処理関連のユニットテストを格納する + │ │ ├─conftest.py -- テスト内で共通利用できるフィクスチャの宣言 + │ │ └─test_vjsk_file_check.py -- テストクラス本体 + │ │ + │ └─vjsk_load -- 受領データ登録処理関連のユニットテストを格納する + │ │ conftest.py -- テスト内で共通利用できるフィクスチャの宣言 + │ │ test_vjsk_load.py -- テストクラス本体 + │ │ + │ └─testdata -- テストモジュールが使用するテストデータを格納する + │ │ bio_slip_data_202304280000.tsv -- 正常ケースの単体確認用 + │ │ ... -- *20230428* は新規4件の登録確認用 + │ │ whs_mst_202304290000.tsv -- *20230429* は更新2件+追加新規2件の登録確認用 + │ │ + │ ├─NoData -- 正常ケースの単体確認用 + │ │ bio_slip_data_nodatarecord.tsv -- ヘッダ行のみでデータが0件の動作確認用 + │ │ ... + │ │ whs_mst_nodatarecord.tsv + │ │ + │ ├─TestFormatErrorFile -- 異常ケースの単体確認用 + │ │ bio_slip_data_formaterror.tsv -- 末尾行のタブ数が想定と異なる(ファイル欠落がある)ときの動作確認用 + │ │ ... + │ │ whs_mst_formaterror.tsv + │ │ + │ ├─TestImportFileToDb -- 正常ケースの単体確認用 + │ │ bio_slip_data_202304270000.gz -- 対向元システムから送られてきた状態(gz圧縮)の受領データファイルの動作確認用 + │ │ ... + │ │ whs_mst_202304270000.gz + │ │ + │ └─UnzipError -- 異常ケースの単体確認用 + │ bio_slip_data_202304270000.gz -- gz圧縮ファイルが解凍できないときの動作確認用 + │ ... + │ whs_mst_202304270000.gz + │ ├── conftest.py -- テスト内で共通利用できるフィクスチャを宣言する(執筆時点ではDBのみ) - └── testing_utility.py -- テストの共通関数 + ├── testing_utility.py -- テストの共通関数 + └── testing_vjsk_utility.py -- テストの共通関数(実消化データ取込処理関連) ``` + diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py index cef4e1ec..6006c030 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py @@ -1,4 +1,6 @@ # from src.batch.vjsk.vjsk_recv_file_manager import VjskDatFile +import os + from src.batch.vjsk.vjsk_recv_file_mapper import VjskReceiveFileMapper from src.db.database import Database from src.error.exceptions import BatchOperationException @@ -9,10 +11,21 @@ mapper = VjskReceiveFileMapper() class VjskDataLoadManager: + """ + Ｖ実消化データ取込機能クラス + """ + def __init__(self): pass def _import_to_db(src_file_name: str, condkey: str): + """ + 概要 + 指定されたtsvファイル src_file_name を策席スキーマに登録する + 引数 + src_file_name: ローカルストレージにある取込対象tsvファイルパス + condkey: 受領データの種類を一意に示す値(VjskReceiveFileMapperクラスで管理されているCONDKEY値) + """ logger.debug(f"_import_to_db start (src_file_name : {src_file_name}, condkey : {condkey})") db = Database.get_instance() @@ -22,6 +35,7 @@ class VjskDataLoadManager: upsert_sql = mapper.get_upsert_sql(condkey) try: + # データベース接続 db.connect() db.execute("SET SESSION sql_mode = 'TRADITIONAL';") @@ -50,51 +64,96 @@ class VjskDataLoadManager: # MEMO: https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.BaseCursorResult.rowcount logger.info(f'{table_name_org}を{table_name_src}にUPSERT') + # データベースコミット db.commit() + except Exception as e: db.rollback() raise BatchOperationException(e) + finally: + # データベース切断 db.disconnect() logger.debug("_import_to_db done") return def _get_tsv_last_row_tab_count(src_file_name: str) -> int: + """ + 概要 + 指定されたtsvファイル src_file_name の末尾行に含まれるタブ文字数を取得する + 引数 + src_file_name: ローカルストレージにある取込対象tsvファイルパス + """ # memo: tsvファイルが数百MBに及ぶことを想定して、末尾から1行分を参照する # memo: 前提1 行区切りは LF('\n') + # memo: 前提2 正常時のファイル終端にある文字は、末尾行の LF('\n') + # memo: 前提3 ファイルエンコードはBOM付UTF-8(先頭3byteが b'\xEF' + b'\xBB' + b'\xBF' ) buf_count = 0 + # ファイルサイズ取得 + file_size = os.path.getsize(src_file_name) + + # ファイルサイズが0byteなら処理終了 + if file_size == 0: + return buf_count + # バイナリモードでファイルオープン with open(src_file_name, 'rb') as file: - # ファイルの末尾から2バイト手前に移動 - file.seek(-2, 2) - # 改行文字を見つけるまで逆方向に読み進める - while file.read(1) != b'\n': - # 1バイト戻って再度読み込み - file.seek(-2, 1) - # 末尾行を抽出 - last_line = file.readline().decode().rstrip('\n') - # 末尾行に含まれるタブ文字の数を抽出 + # ファイルポインタを末尾に移動 + file.seek(0, os.SEEK_END) + + # ファイルポインタが先頭+1になるまで逆方向にシークする + while file.tell() > 1: + # 2byte戻って + file.seek(-2, os.SEEK_CUR) + # 1byte読む(同時に+1シークする) + char = file.read(1) + # 行区切りを検出したらループ終了 + # memo: UTF-8 バイトシーケンスとして、b'\n' が全角文字の一部にはならない + if char == b'\n': + break + # ファイル先頭のBOM3byte目の BF を検出したらループ終了 + # memo: UTF-8 バイトシーケンスとして、b'\xbf' が全角文字の一部の可能性がある(例:全角片仮名の「タ」) + # memo: charに代入したときのfile.read(1)によって、ファイルポインタは2→3になっている前提のロジック + if char == b'\xbf' and file.tell() == 3: + break + last_line = file.readline().decode('utf-8-sig').rstrip('\n') buf_count = last_line.count('\t') return buf_count @classmethod def load(self, target: dict): + """ + 概要 + 取込対象受領ファイル target をデータベースに登録する + 引数 + target: { + condkey: 受領データの種類を一意に示す値(VjskReceiveFileMapperクラスで管理されているCONDKEY値) + src_file_path: ローカルストレージにある取込対象tsvファイルパス + } + """ logger.debug(f'load start target:{target}') # S3からローカルストレージにdownloadした登録対象のtsvファイルパスを取得 local_file_name = target["src_file_path"] # tsvファイル末尾行のTABの数が総定数と一致しない場合は例外をスロー - # TODO: ↓↓↓developへのマージを優先させたいので、未テストのロジックはコメントアウトする - # tsv_tabs = self._get_tsv_last_row_tab_count(local_file_name) - # expect_tabs = mapper.get_file_column_separators(target["condkey"]) - # if tsv_tabs != expect_tabs: - # msg = f"受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした local_file_name: {local_file_name}" - # raise BatchOperationException(msg) - # TODO: ↑↑↑developへのマージを優先させたいので、未テストのロジックはコメントアウトする + # memo: + # 対向元システムで生成されるファイルは稀に途中欠落が発生することがある。 + # これを、ファイルMySQL8.0のLOADステートメントで発生するWARNING/ERRORでは検知不可能なので、 + # LOADステートメント実行前に、物理的に途中欠落があるかを検知してエラーとすることが目的。 + tsv_tabs = self._get_tsv_last_row_tab_count(local_file_name) + expect_tabs = mapper.get_file_column_separators(target["condkey"]) + if tsv_tabs != expect_tabs: + msg = [ + "受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした", + f"local_file_name: {local_file_name}", + f"末尾行のtab数: {tsv_tabs}", + f"tab想定数: {expect_tabs}" + ] + raise BatchOperationException(' '.join(msg)) # データベース登録 self._import_to_db(local_file_name, target["condkey"]) diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py index 19f70067..21264c42 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py @@ -2,6 +2,9 @@ import textwrap class VjskReceiveFileMapper: + """ + Ｖ実消化データファイルI/Fマッピング定義クラス + """ CONDKEY_SLIP_DATA = "SLIP_DATA" # 販売実績データ CONDKEY_HLD_MST = "HLD_MST" # Ｖ卸ホールディングスマスタ CONDKEY_WHS_MST = "WHS_MST" # Ｖ卸マスタ @@ -1489,27 +1492,75 @@ class VjskReceiveFileMapper: return ret def get_data_name(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、ファイル論理名を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_DATA_NAME) def get_file_prefix(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、ファイル名接頭辞を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_FILE_PREFIX) def get_file_suffix(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、ファイル拡張子を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_FILE_SUFFIX) def get_file_column_separators(self, condkey: str) -> int: + """ + 概要 + 受領ファイルI/Fが想定する、1行あたりのタブ文字数を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return int(self._get_interface_property(condkey, self._KEY_FILE_COLUMN_SEPARATORS)) def get_org_table(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、LOAD先ロードスキーマテーブル名を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_ORG_TABLE) def get_src_table(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、登録先蓄積スキーマテーブル名を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_SRC_TABLE) def get_upsert_sql(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、upsert (ロードスキーマ→蓄積スキーマ) CMLステートメントを取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_UPSERT_SQL) def get_condkey_by_s3_file_path(self, s3_file_path: str) -> str: + """ + 概要 + S3受領バケットに受領したファイル名から、I/F想定に該当する condkey 値を取得する + 引数 + s3_file_path: S3受領バケットにある受領したファイルパス + """ ret = None filename = s3_file_path[s3_file_path.rfind("/") + 1:] for condkey in self._VJSK_INTERFACE_MAPPING: diff --git a/ecs/jskult-batch-daily/src/db/database.py b/ecs/jskult-batch-daily/src/db/database.py index 03c9c068..280a9d8b 100644 --- a/ecs/jskult-batch-daily/src/db/database.py +++ b/ecs/jskult-batch-daily/src/db/database.py @@ -83,7 +83,9 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。\n diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py index dccdd0df..bf9b2629 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py @@ -23,7 +23,6 @@ def receive_folder(): return os.environ["VJSK_DATA_RECEIVE_FOLDER"] -# TODO 共通fixtureにして15個固定でput/delete、各個別fixtureで15個から引き算でdeleteする @pytest.fixture def init_check_received_files_ok1(s3_client, bucket_name, receive_folder): # setup diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py index 2dbe2ef5..6b45f96e 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py @@ -17,6 +17,9 @@ from tests.testing_vjsk_utility import (assert_table_results, class TestImportFileToDb: + """ + Ｖ実消化データ取込－データベース登録処理 テストクラス + """ db: Database batch_context: BatchContext test_file_path_import_all: str @@ -1163,56 +1166,6 @@ class TestImportFileToDb: key = f"{receive_folder}/{test_file}" s3_client.delete_object(Bucket=bucket_name, Key=key) - def test_load_data_error(self, mapper): - """ - 観点 - 異常系 : 日付型矛盾のデータ ※製品価格マスタファイルで確認 - 期待値 - 例外が発生する - """ - - # setup - self.batch_context.is_vjsk_stock_import_day = True - - # 処理実行 - target_dict = { - "condkey": mapper.CONDKEY_PHM_PRICE_MST, - "src_file_path": path.join(self.test_file_path_load_individual, "phm_price_mst_dataerror.tsv") - } - - with pytest.raises(BatchOperationException) as e: - VjskDataLoadManager.load(target_dict) - - # 検証 - assert str(e.value).startswith("SQL Error:") > 0 - - # teardown - - def test_load_format_error(self, mapper): - """ - 観点 - 異常系 : tsvファイルが途中で欠落している - 期待値 - 例外が発生する - """ - - # setup - self.batch_context.is_vjsk_stock_import_day = True - - # 処理実行 - target_dict = { - "condkey": mapper.CONDKEY_PHM_PRICE_MST, - "src_file_path": path.join(self.test_file_path_load_individual, "phm_price_mst_formaterror.tsv") - } - - with pytest.raises(BatchOperationException) as e: - VjskDataLoadManager.load(target_dict) - - # 検証 - assert str(e.value).startswith("LOAD文実行時にWARNINGが発生しました。") > 0 - - # teardown - def test_s3backup_ok(self, s3_client, bucket_name, receive_folder, mapper): """ 観点 @@ -1293,3 +1246,783 @@ class TestImportFileToDb: assert str(e.value) == "An error occurred (404) when calling the HeadObject operation: Not Found" # teardown + + def test_load_format_error_01(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_STOCK_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "stock_slip_data_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_02(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "slip_data_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_03(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_ORG_CNV_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "org_cnv_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_04(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_VOP_HCO_MERGE, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "vop_hco_merge_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_05(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_WHS_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "whs_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_06(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_HLD_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "hld_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_07(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_FCL_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "fcl_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_08(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_MKR_ORG_HORIZON, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "mkr_org_horizon_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_09(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_TRAN_KBN_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "tran_kbn_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_10(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_PHM_PRD_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "phm_prd_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_11(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_PHM_PRICE_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "phm_price_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_12(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_WHS_CUSTOMER_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "whs_customer_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_13(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_MDB_CONV_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "mdb_conv_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_14(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_BIO_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "bio_slip_data_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_15(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_LOT_NUM_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "lot_num_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_no_data_ok_01(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_STOCK_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "stock_slip_data_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_02(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "slip_data_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_03(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_ORG_CNV_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "org_cnv_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_04(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_VOP_HCO_MERGE, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "vop_hco_merge_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_05(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_WHS_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "whs_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_06(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_HLD_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "hld_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_07(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_FCL_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "fcl_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_08(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_MKR_ORG_HORIZON, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "mkr_org_horizon_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_09(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_TRAN_KBN_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "tran_kbn_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_10(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_PHM_PRD_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "phm_prd_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_11(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_PHM_PRICE_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "phm_price_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_12(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_WHS_CUSTOMER_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "whs_customer_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_13(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_MDB_CONV_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "mdb_conv_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_14(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_BIO_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "bio_slip_data_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_15(self, mapper): + """ + 観点 + 正常系 : tsvファイルレコード0件の取込 + 期待値 + 例外が発生しないこと + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_LOT_NUM_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "lot_num_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/bio_slip_data_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/bio_slip_data_nodatarecord.tsv new file mode 100644 index 00000000..e2dccedb --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/bio_slip_data_nodatarecord.tsv @@ -0,0 +1 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_whs_org_cd" "rec_cust_cd" "rec_comm_cd" "rec_tran_kbn" "rec_hsdnYmd_wrk" "rec_hsdnYmd_srk" "rec_urag_no" "rec_comm_nm" "rec_nnskFcl_nm" "rec_nnsk_fcl_addr" "rec_lot_num1" "rec_amt1" "rec_lot_num2" "rec_amt2" "rec_lot_num3" "rec_amt3" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "hsdn_ymd" "exec_dt" "v_tran_cd" "tran_kbn_nm" "whs_org_cd" "v_whsOrg_cd" "whs_org_nm" "whs_org_kn" "v_whs_cd" "whs_nm" "nnsk_cd" "fcl_cd" "fcl_nm" "fcl_kn" "fcl_addr_v" "comm_cd" "comm_nm" "htdnYmd_err_kbn" "prd_exis_kbn" "fcl_exis_kbn" "amt1" "amt2" "amt3" "slip_org_kbn" "bef_slip_mgt_no" "whs_rep_comm_nm" "whs_rep_nnskFcl_nm" "whs_rep_nnsk_fcl_addr" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "err_flg11" "err_flg12" "err_flg13" "err_flg14" "err_flg15" "err_flg16" "err_flg17" "err_flg18" "err_flg19" "err_flg20" "kjyo_ym" "tksNbk_kbn" "fcl_exec_kbn" "rec_sts_kbn" "ins_dt" "ins_usr" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/fcl_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/fcl_mst_nodatarecord.tsv new file mode 100644 index 00000000..ccd44bbd --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/fcl_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"fcl_cd" "sub_no" "start_dt" "end_dt" "closed_dt" "nm" "kn_nm" "sht_nm" "sht_kn_nm" "mkr_cd" "jsk_proc_kbn" "fmt_addr" "fmt_kn_addr" "post_cd" "prft_cd" "prft_nm" "city_nm" "addr_line_1" "tel_no" "admin_kbn" "fcl_type" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/hld_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/hld_mst_nodatarecord.tsv new file mode 100644 index 00000000..27e3a2b4 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/hld_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"v_hld_cd" "sub_no" "nm" "kn_nm" "sht_nm" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/lot_num_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/lot_num_mst_nodatarecord.tsv new file mode 100644 index 00000000..73c23a1f --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/lot_num_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"ser_no" "lot_num" "expr_dt" "frst_mov_dt" "ins_dt" "ins_usr" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mdb_conv_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mdb_conv_mst_nodatarecord.tsv new file mode 100644 index 00000000..d8868820 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mdb_conv_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"hco_vid__v" "sub_no" "mdb_cd" "reliability" "start_dt" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mkr_org_horizon_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mkr_org_horizon_nodatarecord.tsv new file mode 100644 index 00000000..afd4d124 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mkr_org_horizon_nodatarecord.tsv @@ -0,0 +1 @@ +"vid_kind_1" "v_cd_1" "nm_1" "dsp_odr_1" "vid_kind_2" "v_cd_2" "nm_2" "dsp_odr_2" "vid_kind_3" "v_cd_3" "nm_3" "dsp_odr_3" "vid_kind_4" "v_cd_4" "nm_4" "dsp_odr_4" "vid_kind_5" "v_cd_5" "nm_5" "dsp_odr_5" "vid_kind_6" "v_cd_6" "nm_6" "dsp_odr_6" "vid_kind_7" "v_cd_7" "nm_7" "dsp_odr_7" "vid_kind_8" "v_cd_8" "nm_8" "dsp_odr_8" "vid_kind_9" "v_cd_9" "nm_9" "dsp_odr_9" "vid_kind_10" "v_cd_10" "nm_10" "dsp_odr_10" "v_whs_cd" "start_dt" "end_dt" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/org_cnv_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/org_cnv_mst_nodatarecord.tsv new file mode 100644 index 00000000..a4c52b43 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/org_cnv_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"whs_cd" "whs_sub_cd" "org_cd" "sub_no" "v_org_cd" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_prd_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_prd_mst_nodatarecord.tsv new file mode 100644 index 00000000..a9ce545e --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_prd_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"prd_cd" "sub_no" "prd_nm" "prd_e_nm" "mkr_cd" "mkr_inf_1" "mkr_inf_2" "phm_itm_cd" "itm_nm" "itm_sht_nm" "form_cd" "form_nm" "vol_cd" "vol_nm" "cont_cd" "cont_nm" "pkg_cd" "pkg_nm" "cnv_num" "jsk_start_dt" "prd_sale_kbn" "jsk_proc_kbn" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_price_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_price_mst_nodatarecord.tsv new file mode 100644 index 00000000..362c753c --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_price_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"phm_prd_cd" "phm_price_kind" "sub_no" "price" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/slip_data_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/slip_data_nodatarecord.tsv new file mode 100644 index 00000000..8957ebc7 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/slip_data_nodatarecord.tsv @@ -0,0 +1 @@ +"recvdata" "rec_whs_cd" "rec_whs_sub_cd" "rec_whs_org_cd" "rec_cust_cd" "rec_comm_cd" "rec_tran_kbn" "rev_hsdnYmd_wrk" "rev_hsdnYmd_srk" "rec_urag_no" "rec_amt" "rec_unit_price" "rec_price" "rec_comm_nm" "rec_nnskFcl_nm" "free_item" "rec_nnsk_fcl_addr" "rec_nnsk_fcl_post" "rec_nnsk_fcl_tel" "rec_bef_hsdn_ymd" "rec_bef_slip_no" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "hsdn_ymd" "exec_dt" "v_tran_cd" "tran_kbn_nm" "whs_org_cd" "v_whsOrg_cd" "whs_org_nm" "whs_org_kn" "v_whs_cd" "whs_nm" "nnsk_cd" "fcl_cd" "fcl_kn" "fcl_nm" "fcl_addr_v" "comm_cd" "comm_nm" "nn_amt" "nn_unitPrice" "nn_price" "unit_price" "unit_amt" "drag_price" "drag_amt" "whsPos_err_kbn" "htdnYmd_err_kbn" "prd_exis_kbn" "fcl_exis_kbn" "bef_hsdn_ymd" "bef_slip_no" "slip_org_kbn" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "err_flg11" "err_flg12" "err_flg13" "err_flg14" "err_flg15" "err_flg16" "err_flg17" "err_flg18" "err_flg19" "err_flg20" "kjyo_ym" "tksNbk_kbn" "fcl_exec_kbn" "rec_sts_kbn" "ins_dt" "ins_usr" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/stock_slip_data_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/stock_slip_data_nodatarecord.tsv new file mode 100644 index 00000000..4dcd83cb --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/stock_slip_data_nodatarecord.tsv @@ -0,0 +1 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_sto_place" "rec_stock_ymd" "rec_comm_cd" "rec_amt" "rev_stok_no_sign" "rev_jan_cd" "rec_free_item" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "exec_dt" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "rec_sts_kbn" "ins_dt" "ins_usr" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/tran_kbn_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/tran_kbn_mst_nodatarecord.tsv new file mode 100644 index 00000000..67a2ed3d --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/tran_kbn_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"v_tran_cd" "sub_no" "nm" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/vop_hco_merge_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/vop_hco_merge_nodatarecord.tsv new file mode 100644 index 00000000..e707d5cb --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/vop_hco_merge_nodatarecord.tsv @@ -0,0 +1 @@ +"hco_vid__v" "hco_vid__v_merge" "apply_dt" "merge_reason" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_customer_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_customer_mst_nodatarecord.tsv new file mode 100644 index 00000000..dc76587c --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_customer_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"whs_cd" "whs_sub_cd" "customer_cd" "sub_no" "start_dt" "end_dt" "org_cd" "src_org_cd" "nm" "kn_nm" "addr" "kn_addr" "zip_cd" "tel_no" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_mst_nodatarecord.tsv new file mode 100644 index 00000000..6f760e14 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"v_whs_cd" "sub_no" "nm" "kn_nm" "sht_nm" "zip_cd" "addr" "kn_addr" "tel_no" "v_hld_cd" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/bio_slip_data_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/bio_slip_data_formaterror.tsv new file mode 100644 index 00000000..0b359d23 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/bio_slip_data_formaterror.tsv @@ -0,0 +1,5 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_whs_org_cd" "rec_cust_cd" "rec_comm_cd" "rec_tran_kbn" "rec_hsdnYmd_wrk" "rec_hsdnYmd_srk" "rec_urag_no" "rec_comm_nm" "rec_nnskFcl_nm" "rec_nnsk_fcl_addr" "rec_lot_num1" "rec_amt1" "rec_lot_num2" "rec_amt2" "rec_lot_num3" "rec_amt3" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "hsdn_ymd" "exec_dt" "v_tran_cd" "tran_kbn_nm" "whs_org_cd" "v_whsOrg_cd" "whs_org_nm" "whs_org_kn" "v_whs_cd" "whs_nm" "nnsk_cd" "fcl_cd" "fcl_nm" "fcl_kn" "fcl_addr_v" "comm_cd" "comm_nm" "htdnYmd_err_kbn" "prd_exis_kbn" "fcl_exis_kbn" "amt1" "amt2" "amt3" "slip_org_kbn" "bef_slip_mgt_no" "whs_rep_comm_nm" "whs_rep_nnskFcl_nm" "whs_rep_nnsk_fcl_addr" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "err_flg11" "err_flg12" "err_flg13" "err_flg14" "err_flg15" "err_flg16" "err_flg17" "err_flg18" "err_flg19" "err_flg20" "kjyo_ym" "tksNbk_kbn" "fcl_exec_kbn" "rec_sts_kbn" "ins_dt" "ins_usr" +"D452960211JD1111311102503851400002304016427519111 496350122ﾊﾞﾍﾞﾝﾁｵﾃﾝﾃｷ200MG 1V ｿｳｺﾞｳﾒﾃﾞｲｶﾙﾆﾎﾝｺｳｶﾝﾋﾞﾂｸﾊﾞｼ ﾀｶｻｷ 753 BAVB007 000003 000000 000000 " "296" "02" "11JD11113111025" "0385140000" "496350122" "111" "230401" "20230401" "6427519" "ﾊﾞﾍﾞﾝﾁｵﾃﾝﾃｷ200MG 1V " "ｿｳｺﾞｳﾒﾃﾞｲｶﾙﾆﾎﾝｺｳｶﾝﾋﾞ" "ﾂｸﾊﾞｼ ﾀｶｻｷ 753 " "BAVB007 " "000003" " " "000000" " " "000000" "20230403" "J" "VJSK-BIO_J_MERCK_2023040300.txt" "J2023040300000126" "129" "20230401" "202305082041" "110" "売上" "11JD" "300001370" "川崎南支店" "" "200000007" "アルフレッサ株式会社" "0385140000" "670235967013012526" "医療法人社団こうかん会　日本鋼管病院" "ｲﾘｮｳﾎｳｼﾞﾝｼｬﾀﾞﾝｺｳｶﾝｶｲ ﾆﾎﾝｺｳｶﾝﾋﾞｮｳｲﾝ" "210-0852 神奈川県川崎市川崎区鋼管通１−２−１" "496350122" "ﾊﾞﾍﾞﾝﾁｵ 注射剤 200mg 1VIAL" "" "1" "" "3" "0" "0" "J" "" "" "" "" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202304" "" "" "0" "23-04-03 20:42:11" "system" +"D452960211G11111377452402930640002304016433215111 496300127ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ ｾｺﾑ)ｵｷﾞｸﾎﾞﾋﾞﾖｳｲﾝ ﾄｳｷﾖｳﾄ ｼﾌﾞﾔｸ ｼﾞﾝｸﾞｳﾏｴ 1-5-1 GF4C001 000002 000000 000000 " "296" "02" "11G111113774524" "0293064000" "496300127" "111" "230401" "20230401" "6433215" "ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ " "ｾｺﾑ)ｵｷﾞｸﾎﾞﾋﾞﾖｳｲﾝ " "ﾄｳｷﾖｳﾄ ｼﾌﾞﾔｸ ｼﾞﾝｸﾞｳﾏｴ 1-5-1 " "GF4C001 " "000002" " " "000000" " " "000000" "20230403" "J" "VJSK-BIO_J_MERCK_2023040300.txt" "J2023040300000127" "130" "20230401" "202305082041" "110" "売上" "11G1" "300001351" "杉並・中野支店" "" "200000007" "アルフレッサ株式会社" "0293064000" "670234652241314835" "医療法人財団荻窪病院　荻窪病院" "ｲﾘｮｳﾎｳｼﾞﾝｻﾞｲﾀﾞﾝｵｷﾞｸﾎﾞﾋﾞｮｳｲﾝ ｵｷﾞｸﾎﾞﾋﾞｮｳｲﾝ" "167-0035 東京都杉並区今川３−１−２４" "496300127" "ｺﾞﾅｰﾙｴﾌ 皮下注ﾍﾟﾝ 450IU 1PEN" "" "1" "" "2" "0" "0" "J" "" "" "" "" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202304" "" "" "0" "23-04-03 20:42:11" "system" +"D452960211V11121120604204799500002304016461276111 496300134ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ ﾆﾁｲｻ-ﾋﾞｽｷﾕｳｼﾕｳﾌｸｵｶｻﾝﾌｸｵｶｼ ｻﾜﾗｸ ﾓﾓﾁﾊﾏ 1-7-5 7F GF9C002 000010 000000 000000 " "296" "02" "11V111211206042" "0479950000" "496300134" "111" "230401" "20230401" "6461276" "ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ " "ﾆﾁｲｻ-ﾋﾞｽｷﾕｳｼﾕｳﾌｸｵｶｻﾝ" "ﾌｸｵｶｼ ｻﾜﾗｸ ﾓﾓﾁﾊﾏ 1-7-5 7F " "GF9C002 " "000010" " " "000000" " " "000000" "20230403" "J" "VJSK-BIO_J_MERCK_2023040300.txt" "J2023040300000128" "131" "20230401" "202305082041" "110" "売上" "11V1" "300001491" "福岡第一支店" "" "200000007" "アルフレッサ株式会社" "0479950000" "670235883412145206" "医療法人社団高邦会　福岡山王病院" "ｲﾘｮｳﾎｳｼﾞﾝｼｬﾀﾞﾝｺｳﾎｳｶｲ ﾌｸｵｶｻﾝﾉｳﾋﾞｮｳｲﾝ" "814-0001 福岡県福岡市早良区百道浜３−６−４５" "496300134" "ｺﾞﾅｰﾙｴﾌ 皮下注ﾍﾟﾝ 900IU 1PEN" "" "1" "" "10" "0" "0" "J" "" "" "" "" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202304" "" "" "0" "23-04-03 20:42:11" "system" +"D452960211JB1121309300202875030002304016523689111 496300127ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ ｲﾂｶﾝﾄﾞｳﾔﾂｷﾖｸ0561 ｶﾜｻｷｼ ｱｻｵｸ ﾌﾙｻﾜ 172-1 GF4C001 000001 000000 000000 " "296" "02" "11JB11213093002" "02875 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/fcl_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/fcl_mst_formaterror.tsv new file mode 100644 index 00000000..0125ec7d --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/fcl_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"fcl_cd" "sub_no" "start_dt" "end_dt" "closed_dt" "nm" "kn_nm" "sht_nm" "sht_kn_nm" "mkr_cd" "jsk_proc_kbn" "fmt_addr" "fmt_kn_addr" "post_cd" "prft_cd" "prft_nm" "city_nm" "addr_line_1" "tel_no" "admin_kbn" "fcl_type" "rec_sts_kbn" "ins_dt" "upd_dt" +"670229430760653825" "0" "20000101" "99991231" "" "駅前町歯科診療所" "ｴｷﾏｴﾁｮｳｼｶｼﾝﾘｮｳｼﾞｮ" "駅前町歯科診療所" "ｴｷﾏｴﾁｮｳｼｶｼﾝﾘｮｳｼﾞｮ" "" "0" "700-0023 岡山県岡山市北区駅前町１−６−２０" "ｵｶﾔﾏｹﾝ ｵｶﾔﾏｼｷﾀｸ ｴｷﾏｴﾁｮｳ1-6-20" "700-0023" "33" "岡山県" "岡山市北区" "駅前町１−６−２０" "0862236468" "33101" "30" "1" "" "" +"670229435466662922" "0" "20000101" "99991231" "" "医療法人社団仁卓会　ほりかわ歯科クリニック" "ｲﾘｮｳﾎｳｼﾞﾝｼｬﾀﾞﾝｼﾞﾝﾀｸｶｲ ﾎﾘｶﾜｼｶｸﾘﾆｯｸ" "ほりかわ歯科クリニック　（医社）" "ﾎﾘｶﾜｼｶｸﾘﾆｯｸ (ｲｼｬ)" "" "0" "675-0101 兵庫県加古川市平岡町新在家１５７３−１−４Ｆ" "ﾋｮｳｺﾞｹﾝ ｶｺｶﾞﾜｼ ﾋﾗｵｶﾁｮｳｼﾝｻﾞｲｹ1573-1-4F" "675-0101" "28" "兵庫県" "加古川市" "平岡町新在家１５７３−１−４Ｆ" "0794244617" "28210" "30" "1" "" "" +"670229435785430019" "0" "20000101" "99991231" "" "株式会社コミュニティメディカル　なつめ薬局　千歳船橋店" "ｶﾌﾞｼｷｶﾞｲｼｬｺﾐｭﾆﾃｨﾒﾃﾞｨｶﾙ ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ" "なつめ薬局　千歳船橋店　（株）" "ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ (ｶ)" "" "0" "156-0054 東京都世田谷区桜丘２−２４−２" "ﾄｳｷｮｳﾄ ｾﾀｶﾞﾔｸ ｻｸﾗｶﾞｵｶ2-24-2" "156-0054" "13" "東京都" "世田谷区" "桜丘２−２４−２" "0364136189" "13112" "20" "1" "" "" +"670229447437206529" "0" "20000101" "20230407" "2023-04-07" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "" "0" "670-0955 兵庫県 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/hld_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/hld_mst_formaterror.tsv new file mode 100644 index 00000000..9060a7da --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/hld_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"v_hld_cd" "sub_no" "nm" "kn_nm" "sht_nm" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"10001" "1" "卸ホールディングス名01-1" "オロシホールディングスメイ01-1" "卸名01-1" "20230101" "99991231" "1" "0" "23-05-09 12:00:01" "23-05-09 13:00:01" +"10001" "2" "卸ホールディングス名01-2" "オロシホールディングスメイ01-2" "卸名01-2" "20230102" "99991231" "1" "0" "23-05-09 12:00:02" "23-05-09 13:00:02" +"10001" "3" "卸ホールディングス名01-3" "オロシホールディングスメイ01-3" "卸名01-3" "20230103" "99991231" "1" "0" "23-05-09 12:00:03" "23-05-09 13:00:03" +"10001" "4" "卸ホールディングス名01-4" "オロシホールディングスメイ01-4" "卸 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/lot_num_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/lot_num_mst_formaterror.tsv new file mode 100644 index 00000000..6978f030 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/lot_num_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"ser_no" "lot_num" "expr_dt" "frst_mov_dt" "ins_dt" "ins_usr" +"F0110601" "BAVA001" "20230331" "20210510" "23-05-08 20:40:41" "batch" +"F0110601" "BAVA002" "20230331" "20210615" "23-05-08 20:40:41" "batch" +"F0110601" "BAVA003" "20231031" "20210719" "23-05-08 20:40:41" "batch" +"F0110601" "BAVA004" "2 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mdb_conv_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mdb_conv_mst_formaterror.tsv new file mode 100644 index 00000000..5b4d688d --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mdb_conv_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"hco_vid__v" "sub_no" "mdb_cd" "reliability" "start_dt" "rec_sts_kbn" "ins_dt" "upd_dt" +"670229780011959315" "1" "003410424" "0" "20020601" "0" "22-03-09 13:56:19" "22-03-09 13:56:19" +"670230081112654862" "0" "004101420" "0" "20000101" "9" "17-10-17 17:06:52" "22-03-09 14:17:34" +"670230081112654862" "1" "004104997" "2" "20000101" "0" "22-03-09 14:17:33" "22-03-09 14:17:33" +"670230100414841865" "0" "003622111" "3" \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mkr_org_horizon_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mkr_org_horizon_formaterror.tsv new file mode 100644 index 00000000..6f108bcf --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mkr_org_horizon_formaterror.tsv @@ -0,0 +1,5 @@ +"vid_kind_1" "v_cd_1" "nm_1" "dsp_odr_1" "vid_kind_2" "v_cd_2" "nm_2" "dsp_odr_2" "vid_kind_3" "v_cd_3" "nm_3" "dsp_odr_3" "vid_kind_4" "v_cd_4" "nm_4" "dsp_odr_4" "vid_kind_5" "v_cd_5" "nm_5" "dsp_odr_5" "vid_kind_6" "v_cd_6" "nm_6" "dsp_odr_6" "vid_kind_7" "v_cd_7" "nm_7" "dsp_odr_7" "vid_kind_8" "v_cd_8" "nm_8" "dsp_odr_8" "vid_kind_9" "v_cd_9" "nm_9" "dsp_odr_9" "vid_kind_10" "v_cd_10" "nm_10" "dsp_odr_10" "v_whs_cd" "start_dt" "end_dt" "rec_sts_kbn" "ins_dt" "upd_dt" +"3" "300003202" "その他営業本部卸" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "200000007" "20190401" "99991231" "0" "19-04-11 11:30:59" "23-04-12 17:52:38" +"3" "300003138" "北関東甲信越営業本部" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "200000007" "20190401" "99991231" "0" "19-04-11 11:30:59" "23-04-12 17:52:38" +"3" "300003202" "その他営業本部卸" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "200000007" "20190401" "99991231" "0" "19-04-11 11:30:59" "23-04-12 17:52:38" +"a" "300003144" "メディカル営業本部1" "1" "b" "300003202" "東海スタッフ医療2" "2" "c" "300003203" "東海スタ \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/org_cnv_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/org_cnv_mst_formaterror.tsv new file mode 100644 index 00000000..6151cd4e --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/org_cnv_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"whs_cd" "whs_sub_cd" "org_cd" "sub_no" "v_org_cd" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"042" "01" "361007" "0" "300006657" "20230401" "99991231" "0" "0" "23-04-12 11:24:06" "23-04-12 11:24:06" +"042" "01" "381207" "0" "300006658" "20230401" "99991231" "0" "0" "23-04-12 11:24:27" "23-04-12 11:24:27" +"080" "00" "02780" "0" "300006526" "20220401" "99991231" "0" "9" "22-04-11 15:57:35" "23-04-12 10:46:48" +"080" "00" "02780" "1" "300006526" "20220401" \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_prd_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_prd_mst_formaterror.tsv new file mode 100644 index 00000000..8324689b --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_prd_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"prd_cd" "sub_no" "prd_nm" "prd_e_nm" "mkr_cd" "mkr_inf_1" "mkr_inf_2" "phm_itm_cd" "itm_nm" "itm_sht_nm" "form_cd" "form_nm" "vol_cd" "vol_nm" "cont_cd" "cont_nm" "pkg_cd" "pkg_nm" "cnv_num" "jsk_start_dt" "prd_sale_kbn" "jsk_proc_kbn" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"496201110" "0" "ｾﾛﾌｪﾝ 錠剤 50mg 30TAB" "" "F21206A0" "セロフェン 錠 50MG" "SEROPHENE TAB. 50 MG. (30)" "001" "セロフェン" "SP" "F003" "錠剤" "0000" "" "V009" "50mg" "P007" "30TAB" "30" "" "0" "0" "20080101" "20190930" "140" "0" "17-11-08 16:52:41" "19-09-19 11:42:45" +"496201127" "0" "ｾﾛﾌｪﾝ 錠剤 50mg 30TAB" "" "F21206A0" "セロフェン 錠 50MG" "SEROPHENE TAB. 50 MG. (30)" "001" "セロフェン" "SP" "F003" "錠剤" "0000" "" "V009" "50mg" "P007" "30TAB" "30" "" "0" "0" "20070401" "20190930" "150" "0" "17-11-08 16:52:41" "19-09-19 11:42:45" +"496300110" "2" "ｺﾞﾅｰﾙｴﾌ 皮下注ﾍﾟﾝ 300IU 1PEN" "" "F1990608" "ゴナールエフ皮下注ペン 300" "GONAL-F PEN 300IU (1) - JPN" "005" "セロスティム" "ST" "F005" "皮下注ペン" "0000" "" "V017" "300IU" "P011" "1PEN" "1" "" "0" "0" "20190501" "20190930" "100" "9" "19-04-23 16:35:36" "19-04-23 16:40:38" +"496300127" "2" "ｺﾞﾅｰﾙｴﾌ 皮下注ﾍﾟﾝ 450IU 1PEN" "" "F19D0608" "ゴナールエフ皮下注ペン450" "Gonalef Pen 450 (1)" "008" "BDマイクロファインプラス" "MF" "F005" "皮下注ペン" "0000" "" "V018" "450IU" "P011" "1PEN" "1" "" "0" "0" "20190501" "20190930" "120" "9" "19-04-23 1 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_price_mst_formaterror.tsv similarity index 100% rename from ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_formaterror.tsv rename to ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_price_mst_formaterror.tsv diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/slip_data_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/slip_data_formaterror.tsv new file mode 100644 index 00000000..7b4b20b0 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/slip_data_formaterror.tsv @@ -0,0 +1,5 @@ +"recvdata" "rec_whs_cd" "rec_whs_sub_cd" "rec_whs_org_cd" "rec_cust_cd" "rec_comm_cd" "rec_tran_kbn" "rev_hsdnYmd_wrk" "rev_hsdnYmd_srk" "rec_urag_no" "rec_amt" "rec_unit_price" "rec_price" "rec_comm_nm" "rec_nnskFcl_nm" "free_item" "rec_nnsk_fcl_addr" "rec_nnsk_fcl_post" "rec_nnsk_fcl_tel" "rec_bef_hsdn_ymd" "rec_bef_slip_no" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "hsdn_ymd" "exec_dt" "v_tran_cd" "tran_kbn_nm" "whs_org_cd" "v_whsOrg_cd" "whs_org_nm" "whs_org_kn" "v_whs_cd" "whs_nm" "nnsk_cd" "fcl_cd" "fcl_kn" "fcl_nm" "fcl_addr_v" "comm_cd" "comm_nm" "nn_amt" "nn_unitPrice" "nn_price" "unit_price" "unit_amt" "drag_price" "drag_amt" "whsPos_err_kbn" "htdnYmd_err_kbn" "prd_exis_kbn" "fcl_exis_kbn" "bef_hsdn_ymd" "bef_slip_no" "slip_org_kbn" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "err_flg11" "err_flg12" "err_flg13" "err_flg14" "err_flg15" "err_flg16" "err_flg17" "err_flg18" "err_flg19" "err_flg20" "kjyo_ym" "tksNbk_kbn" "fcl_exec_kbn" "rec_sts_kbn" "ins_dt" "ins_usr" +"D4420202011611A4 0183733 23030133625911102303 4963500230000020003110000000622000000000000000000000000 ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1Vﾊｺﾀﾞﾃﾁﾕｳｵｳﾋﾞﾖｳｲﾝ 00000408585ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "202" "02" "011611A4 " "0183733 " "496350023" "110" "230301" "20230301" "3362591" "000002" "00031100" "0000062200" "ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1V" "ﾊｺﾀﾞﾃﾁﾕｳｵｳﾋﾞﾖｳｲﾝ " "0408585ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "0408585" "" "000000" " " "20230222" "J" "VJSK_J_MERCK_2023022" "J2023022200000022" "29" "20230301" "202303142041" "110" "売上" "01161" "300000383" "函館支店" "" "200000016" "株式会社スズケン" "0183733 " "670234934576694289" "ｼｬｶｲﾌｸｼﾎｳｼﾞﾝﾊｺﾀﾞﾃｺｳｾｲｲﾝ ﾊｺﾀﾞﾃﾁｭｳｵｳﾋﾞｮｳｲﾝ" "社会福祉法人函館厚生院　函館中央病院" "040-0011 北海道函館市本町３３−２" "496350023" "ｱｰﾋﾞﾀｯｸｽ 注射剤 100mg 1VIAL" "2" "31100" "62200" "31438" "62876" "35309" "70618" "" "" "1" "" "" "" "J" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202303" "" "" "0" "23-03-14 20:41:26" "SYSTEM" +"D4420202011611A4 0183733 23030133625921102303 4963500230000080003110000002488000000000000000000000000 ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1Vﾊｺﾀﾞﾃﾁﾕｳｵｳﾋﾞﾖｳｲﾝ 00000408585ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "202" "02" "011611A4 " "0183733 " "496350023" "110" "230301" "20230301" "3362592" "000008" "00031100" "0000248800" "ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1V" "ﾊｺﾀﾞﾃﾁﾕｳｵｳﾋﾞﾖｳｲﾝ " "0408585ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "0408585" "" "000000" " " "20230222" "J" "VJSK_J_MERCK_2023022" "J2023022200000023" "30" "20230301" "202303142041" "110" "売上" "01161" "300000383" "函館支店" "" "200000016" "株式会社スズケン" "0183733 " "670234934576694289" "ｼｬｶｲﾌｸｼﾎｳｼﾞﾝﾊｺﾀﾞﾃｺｳｾｲｲﾝ ﾊｺﾀﾞﾃﾁｭｳｵｳﾋﾞｮｳｲﾝ" "社会福祉法人函館厚生院　函館中央病院" "040-0011 北海道函館市本町３３−２" "496350023" "ｱｰﾋﾞﾀｯｸｽ 注射剤 100mg 1VIAL" "8" "31100" "248800" "31438" "251504" "35309" "282472" "" "" "1" "" "" "" "J" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202303" "" "" "0" "23-03-14 20:41:26" "SYSTEM" +"D4416101311101A8 5140013 23030173719811122303 4963500230000120002738100003285720000000000000000000000 ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1Vﾄﾂﾄﾘﾆﾂｾｷﾋﾞﾖｳｲﾝ 00006808517ﾄﾂﾄﾘｹﾝﾄﾂﾄﾘｼｼﾖｳﾄｸﾁﾖｳ117 " "161" "01" "311101A8 " "5140013 " "496350023" "112" "230301" "20230301" "7371981" "000012" "00027381" "0000328572" "ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1V" "ﾄﾂﾄﾘﾆﾂｾｷﾋﾞﾖｳｲﾝ " "6808517ﾄﾂﾄﾘｹﾝﾄﾂﾄﾘｼｼﾖｳﾄｸﾁﾖｳ117 " "ﾄﾂﾄﾘｹﾝﾄﾂﾄﾘｼｼﾖｳﾄｸﾁﾖｳ117 " "6808517" "" "000000" " " "20230224" "J" "VJSK_J_MERCK_2023022" "J2023022400000011" "16" "20230301" "202303142041" "110" "売上" "31110" "300000391" "鳥取支店" "" "200000015" "株式会社サンキ" "5140013 " "670237031040828444" "ﾆﾎﾝｾｷｼﾞｭｳｼﾞｼｬ ﾄｯﾄﾘｾｷｼﾞｭｳｼﾞﾋﾞｮｳｲﾝ" "日本赤十字社　鳥取赤十字病院" "680-0017 鳥取県鳥取市尚徳町１１７" "496350023" "ｱｰﾋﾞﾀｯｸｽ 注射剤 100mg 1VIAL" "12" "27381" "328572" "31438" "377256" "35309" "423708" "" "" "1" "" "" "" "J" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202303" "" "" "0" "23-03-14 20:41:26" "SYSTEM" +"D4416101311 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/stock_slip_data_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/stock_slip_data_formaterror.tsv new file mode 100644 index 00000000..aa3cb22e --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/stock_slip_data_formaterror.tsv @@ -0,0 +1,5 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_sto_place" "rec_stock_ymd" "rec_comm_cd" "rec_amt" "rev_stok_no_sign" "rev_jan_cd" "rec_free_item" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "exec_dt" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "rec_sts_kbn" "ins_dt" "ins_usr" +"D463630101 23022849630021900003500000 ｾﾄﾛﾀｲﾄﾞﾁﾕｳｼﾔﾖｳ0.25MG 1V" "363" "01" "01 " "230228" "496300219" "000035" "0" "0000" " ｾﾄﾛﾀｲﾄﾞﾁﾕｳｼﾔﾖｳ0.25MG 1V" "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000059" "59" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"D4625301026 2302284963001270000040000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "253" "01" "026 " "230228" "496300127" "000004" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000060" "60" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"D4625301026 2302284963001340000220000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "253" "01" "026 " "230228" "496300134" "000022" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000061" "61" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"D4625301026 2302284963004170000500000001ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "2 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/tran_kbn_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/tran_kbn_mst_formaterror.tsv new file mode 100644 index 00000000..34692f31 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/tran_kbn_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"v_tran_cd" "sub_no" "nm" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"10001" "1" "Veeva取引区分名01-1" "20230101" "99991231" "1" "0" "2023-05-09 12:00:01" "2023-05-09 13:00:01" +"10001" "2" "Veeva取引区分名01-2" "20230102" "99991231" "1" "0" "2023-05-09 12:00:02" "2023-05-09 13:00:02" +"10001" "3" "Veeva取引区分名01-3" "20230103" "99991231" "1" "0" "2023-05-09 12:00:03" "2023-05-09 13:00:03" +"10001" "4" "Veeva取引区分名01-4" "20230104" "9 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/vop_hco_merge_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/vop_hco_merge_formaterror.tsv new file mode 100644 index 00000000..f4e8956e --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/vop_hco_merge_formaterror.tsv @@ -0,0 +1,5 @@ +"hco_vid__v" "hco_vid__v_merge" "apply_dt" "merge_reason" +"100000001" "900000001" "20230509" "事由01" +"100000002" "900000002" "20230509" "事由02" +"100000003" "900000003" "20230509" "事由03" +"100000004" "900000004" \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_customer_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_customer_mst_formaterror.tsv new file mode 100644 index 00000000..0ec0c4cb --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_customer_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"whs_cd" "whs_sub_cd" "customer_cd" "sub_no" "start_dt" "end_dt" "org_cd" "src_org_cd" "nm" "kn_nm" "addr" "kn_addr" "zip_cd" "tel_no" "rec_sts_kbn" "ins_dt" "upd_dt" +"006" "01" "1002900000" "0" "20000101" "99991231" "11" "1131A2283316" "辻内科小児科医院　　　　　　　　　　　　　　　" "ﾂｼﾞﾅｲｶｼﾖｳﾆｶ ｲｲﾝ" "長崎県　佐世保市皆瀬町２９　　　　　　　　　　　　　　　　　　　" "ﾅｶﾞｻｷｹﾝ ｻｾﾎﾞｼｶｲｾﾞﾁﾖｳ 29" "8570144" "0956492319" "0" "23-04-14 11:53:14" "23-04-14 11:53:14" +"006" "01" "1005400000" "0" "20000101" "99991231" "12" "1211C3415515" "医療法人　愛恵会　佐世保愛恵病院　　　　　　　" "ｲﾘﾖｳﾎｳｼﾞﾝｱｲｹｲｶｲｻｾﾎﾞｱｲｹｲﾋﾞ" "長崎県　佐世保市瀬戸越４丁目　２−１５　　　　　　　　　　　　　" "ﾅｶﾞｻｷｹﾝ ｻｾﾎﾞｼｾﾄｺﾞｼ 4ﾁﾖｳﾒ 2-15" "8570134" "0956493335" "0" "23-04-14 11:53:14" "23-04-14 11:53:14" +"006" "01" "1007200000" "0" "20000101" "99991231" "11" "1131A2407312" "医療法人　山祇診療所　　　　　　　　　　　　　" "ｲﾘﾖｳﾎｳｼﾞﾝﾔﾏｽﾞﾐｼﾝﾘﾖｳｼﾞﾖ" "長崎県　佐世保市山祇町　１９−３６　　　　　　　　　　　　　　　" "ﾅｶﾞｻｷｹﾝ ｻｾﾎﾞｼﾔﾏｽﾞﾐﾁｮｳ" "8570822" "0956313633" "0" "23-04-14 11:53:14" "23-04-14 11:53:14" +"006" "01" "1007800000" "0" "20000101" "999912 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_mst_formaterror.tsv new file mode 100644 index 00000000..82239889 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"v_whs_cd" "sub_no" "nm" "kn_nm" "sht_nm" "zip_cd" "addr" "kn_addr" "tel_no" "v_hld_cd" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"200000002" "0" "株式会社モロオ" "カナ01" "モロオ" "1110001" "住所01" "ジュウショ01" "00-0000-0001" "0" "20000101" "99991231" "20" "0" "16-04-15 16:25:33" "16-04-15 16:25:33" +"200000005" "0" "岩渕薬品株式会社" "カナ02" "岩渕薬品" "1110002" "住所02" "ジュウショ02" "00-0000-0002" "0" "20000101" "99991231" "50" "0" "16-04-15 16:25:33" "16-04-15 16:25:33" +"200000009" "0" "株式会社マルタケ" "カナ03" "マルタケ" "1110003" "住所03" "ジュウショ03" "00-0000-0003" "0" "20000101" "99991231" "90" "0" "16-04-15 16:25:33" "16-04-15 16:25:33" +"200000010" "0" "株式会社ファイネス" "カナ04" "ファイネス" "1110004" "住所04" "ジュ \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_dataerror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_dataerror.tsv deleted file mode 100644 index 3df124c1..00000000 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_dataerror.tsv +++ /dev/null @@ -1,3 +0,0 @@ -"phm_prd_cd" "phm_price_kind" "sub_no" "price" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" -"123456701" "01" "1" "12345.6" "yyyy0401" "20190930" "10" "0" "18-03-07 09:48:00" "19-09-19 11:23:47" -"123456701" "02" "1" "12587.8" "20191001" "99991231" "10" "0" "19-09-19 11:24:05" "19-09-19 11:24:05" diff --git a/ecs/jskult-batch-monthly/src/db/database.py b/ecs/jskult-batch-monthly/src/db/database.py index f67a21b9..d2171fb5 100644 --- a/ecs/jskult-batch-monthly/src/db/database.py +++ b/ecs/jskult-batch-monthly/src/db/database.py @@ -1,11 +1,10 @@ from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, create_engine, text) from sqlalchemy.engine.url import URL -from tenacity import retry, stop_after_attempt, wait_exponential - from src.error.exceptions import DBException from src.logging.get_logger import get_logger from src.system_var import environment +from tenacity import retry, stop_after_attempt, wait_exponential logger = get_logger(__name__) @@ -74,7 +73,9 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。 diff --git a/ecs/jskult-dbdump/.dockerignore b/ecs/jskult-dbdump/.dockerignore new file mode 100644 index 00000000..8b9da402 --- /dev/null +++ b/ecs/jskult-dbdump/.dockerignore @@ -0,0 +1,12 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md +*.sql diff --git a/ecs/jskult-dbdump/.env.example b/ecs/jskult-dbdump/.env.example new file mode 100644 index 00000000..6db2d461 --- /dev/null +++ b/ecs/jskult-dbdump/.env.example @@ -0,0 +1,15 @@ +DB_HOST=************ +DB_PORT=3306 +DB_USERNAME=************ +DB_PASSWORD=************ +DB_SCHEMA=src05 + +JSKULT_BACKUP_BUCKET=mbj-newdwh2021-staging-backup-jskult + +DUMP_BACKUP_FOLDER=dump + +LOG_LEVEL=INFO +DB_CONNECTION_MAX_RETRY_ATTEMPT=************ +DB_CONNECTION_RETRY_INTERVAL_INIT=************ +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=************ +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=************ diff --git a/ecs/jskult-dbdump/.gitignore b/ecs/jskult-dbdump/.gitignore new file mode 100644 index 00000000..cf44449e --- /dev/null +++ b/ecs/jskult-dbdump/.gitignore @@ -0,0 +1,11 @@ +.vscode/settings.json +.env +my.cnf + +# python +__pycache__ + +# python test +.pytest_cache +.coverage +.report/ \ No newline at end of file diff --git a/ecs/jskult-dbdump/.vscode/launch.json b/ecs/jskult-dbdump/.vscode/launch.json new file mode 100644 index 00000000..a0178c26 --- /dev/null +++ b/ecs/jskult-dbdump/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "(DEBUG)jskult batch dbdump", + "type": "python", + "request": "launch", + "program": "entrypoint.py", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/jskult-dbdump/.vscode/recommended_settings.json b/ecs/jskult-dbdump/.vscode/recommended_settings.json new file mode 100644 index 00000000..b5e79d73 --- /dev/null +++ b/ecs/jskult-dbdump/.vscode/recommended_settings.json @@ -0,0 +1,31 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + // 自身の環境に合わせて変えてください + "python.defaultInterpreterPath": "", + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--max-line-length=200", + "--ignore=F541" + ], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Path": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", "200", + "--ignore=F541" + ], + "python.testing.pytestArgs": [ + "tests/batch/ultmarc" + ], + + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ecs/jskult-dbdump/Dockerfile b/ecs/jskult-dbdump/Dockerfile new file mode 100644 index 00000000..3c410b72 --- /dev/null +++ b/ecs/jskult-dbdump/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3.9 + +ENV TZ="Asia/Tokyo" + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +# mysql-apt-config をdpkgでインストールする際に標準出力に渡す文字列ファイルをコピー +COPY mysql_dpkg_selection.txt ./ +# 必要なパッケージインストール +RUN apt update && apt install -y less vim curl wget gzip unzip sudo lsb-release + +# mysqlをインストール +RUN \ + wget https://dev.mysql.com/get/mysql-apt-config_0.8.25-1_all.deb && \ + dpkg -i mysql-apt-config_0.8.25-1_all.deb < mysql_dpkg_selection.txt && \ + apt update && \ + apt install -y mysql-client + +# aws cli v2 のインストール +RUN \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + sudo ./aws/install + +# python関連のライブラリインストール +RUN \ + pip install --upgrade pip wheel setuptools && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv + +# パッケージのセキュリティアップデートのみを適用するコマンドを実行 +RUN \ + apt install -y unattended-upgrades && \ + unattended-upgrades + +COPY src ./src +COPY entrypoint.py entrypoint.py + +CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-dbdump/Pipfile b/ecs/jskult-dbdump/Pipfile new file mode 100644 index 00000000..65b30a51 --- /dev/null +++ b/ecs/jskult-dbdump/Pipfile @@ -0,0 +1,19 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +sqlalchemy = "*" +tenacity = "*" +pymysql = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.9" + +[pipenv] +allow_prereleases = true diff --git a/ecs/jskult-dbdump/Pipfile.lock b/ecs/jskult-dbdump/Pipfile.lock new file mode 100644 index 00000000..58713cc6 --- /dev/null +++ b/ecs/jskult-dbdump/Pipfile.lock @@ -0,0 +1,206 @@ +{ + "_meta": { + "hash": { + "sha256": "e2e2efd7ebd6ad719931983f277105dd94c4ebb6363f7c00e75dd8ca05523713" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "greenlet": { + "hashes": [ + "sha256:0a9dfcadc1d79696e90ccb1275c30ad4ec5fd3d1ab3ae6671286fac78ef33435", + "sha256:0f313771cb8ee0a04dfdf586b7d4076180d80c94be09049daeea018089b5b957", + "sha256:17503397bf6cbb5e364217143b6150c540020c51a3f6b08f9a20cd67c25e2ca8", + "sha256:180ec55cb127bc745669eddc9793ffab6e0cf7311e67e1592f183d6ca00d88c1", + "sha256:1b3f3568478bc21b85968e8038c4f98f4bf0039a692791bc324b5e0d1522f4b1", + "sha256:1bd4ea36f0aeb14ca335e0c9594a5aaefa1ac4e2db7d86ba38f0be96166b3102", + "sha256:21ebcb570e0d8501457d6a2695a44c5af3b6c2143dc6644ec73574beba067c90", + "sha256:24071eee113d75fedebaeb86264d94f04b5a24e311c5ba3e8003c07d00112a7e", + "sha256:270432cfdd6a50016b8259b3bbf398a3f7c06a06f2c68c7b93e49f53bc193bcf", + "sha256:271ed380389d2f7e4c1545b6e0837986e62504ab561edbaff05da9c9f3f98f96", + "sha256:2840187a94e258445e62ff1545e34f0b1a14aef4d0078e5c88246688d2b6515e", + "sha256:2cda110faee67613fed221f90467003f477088ef1cc84c8fc88537785a5b4de9", + "sha256:2e160a65cc6023a237be870f2072513747d512a1d018efa083acce0b673cccc0", + "sha256:2fcf7af83516db35af3d0ed5d182dea8585eddd891977adff1b74212f4bfd2fd", + "sha256:36cebce1f30964d5672fd956860e7e7b69772da69658d5743cb676b442eeff36", + "sha256:42bfe67824a9b53e73f568f982f0d1d4c7ac0f587d2e702a23f8a7b505d7b7c2", + "sha256:450a7e52a515402fd110ba807f1a7d464424bfa703be4effbcb97e1dfbfcc621", + "sha256:463d63ca5d8c236788284a9a44b9715372a64d5318a6b5eee36815df1ea0ba3d", + "sha256:4d0c0ffd732466ff324ced144fad55ed5deca36f6036c1d8f04cec69b084c9d6", + "sha256:4ff2a765f4861fc018827eab4df1992f7508d06c62de5d2fe8a6ac2233d4f1d0", + "sha256:53abf19b7dc62795c67b8d0a3d8ef866db166b21017632fff2624cf8fbf3481c", + "sha256:5552d7be37d878e9b6359bbffa0512d857bb9703616a4c0656b49c10739d5971", + "sha256:585810056a8adacd3152945ebfcd25deb58335d41f16ae4e0f3d768918957f9a", + "sha256:5942b1d6ba447cff1ec23a21ec525dde2288f00464950bc647f4e0f03bd537d1", + "sha256:5c355c99be5bb23e85d899b059a4f22fdf8a0741c57e7029425ee63eb436f689", + "sha256:5f61df4fe07864561f49b45c8bd4d2c42e3f03d2872ed05c844902a58b875028", + "sha256:665942d3a954c3e4c976581715f57fb3b86f4cf6bae3ac30b133f8ff777ac6c7", + "sha256:68368e908f14887fb202a81960bfbe3a02d97e6d3fa62b821556463084ffb131", + "sha256:6aac94ff957b5dea0216af71ab59c602e1b947b394e4f5e878a5a65643090038", + "sha256:889934aa8d72b6bfc46babd1dc4b817a56c97ec0f4a10ae7551fb60ab1f96fae", + "sha256:a00550757fca1b9cbc479f8eb1cf3514dbc0103b3f76eae46341c26ddcca67a9", + "sha256:a4a2d6ed0515c05afd5cc435361ced0baabd9ba4536ddfe8ad9a95bcb702c8ce", + "sha256:a8dd92fd76a61af2abc8ccad0c6c6069b3c4ebd4727ecc9a7c33aae37651c8c7", + "sha256:ab81f9ff3e3c2ca65e824454214c10985a846cd9bee5f4d04e15cd875d9fe13b", + "sha256:ac10196b8cde7a082e4e371ff171407270d3337c8d57ed43030094eb01d9c95c", + "sha256:b767930af686551dc96a5eb70af3736709d547ffa275c11a5e820bfb3ae61d8d", + "sha256:b9a1f4d256b81f59ba87bb7a29b9b38b1c018e052dba60a543cb0ddb5062d159", + "sha256:ba94c08321b5d345100fc64eb1ab235f42faf9aabba805cface55ebe677f1c2c", + "sha256:bab71f73001cd15723c4e2ca398f2f48e0a3f584c619eefddb1525e8986e06eb", + "sha256:bce5cf2b0f0b29680396c5c98ab39a011bd70f2dfa8b8a6811a69ee6d920cf9f", + "sha256:c02e514c72e745e49a3ae7e672a1018ba9b68460c21e0361054e956e5d595bc6", + "sha256:c3fb459ced6c5e3b2a895f23f1400f93e9b24d85c30fbe2d637d4f7706a1116b", + "sha256:cd31ab223e43ac64fd23f8f5dad249addadac2a459f040546200acbf7e84e353", + "sha256:ce70aa089ec589b5d5fab388af9f8c9f9dfe8fe4ad844820a92eb240d8628ddf", + "sha256:d47b2e1ad1429da9aa459ef189fbcd8a74ec28a16bc4c3f5f3cf3f88e36535eb", + "sha256:d61bad421c1f496f9fb6114dbd7c30a1dac0e9ff90e9be06f4472cbd8f7a1704", + "sha256:d7ba2e5cb119eddbc10874b41047ad99525e39e397f7aef500e6da0d6f46ab91", + "sha256:dde0ab052c7a1deee8d13d72c37f2afecee30ebdf6eb139790157eaddf04dd61", + "sha256:df34b52aa50a38d7a79f3abc9fda7e400791447aa0400ed895f275f6d8b0bb1f", + "sha256:e0fc20e6e6b298861035a5fc5dcf9fbaa0546318e8bda81112591861a7dcc28f", + "sha256:e20d5e8dc76b73db9280464d6e81bea05e51a99f4d4dd29c5f78dc79f294a5d3", + "sha256:e31d1a33dc9006b278f72cb0aacfe397606c2693aa2fdc0c2f2dcddbad9e0b53", + "sha256:e3a99f890f2cc5535e1b3a90049c6ca9ff9da9ec251cc130c8d269997f9d32ee", + "sha256:e7b192c3df761d0fdd17c2d42d41c28460f124f5922e8bd524018f1d35610682", + "sha256:ed0f4fad4c3656e34d20323a789b6a2d210a6bb82647d9c86dded372f55c58a1", + "sha256:f34ec09702be907727fd479046193725441aaaf7ed4636ca042734f469bb7451", + "sha256:f3530c0ec1fc98c43d5b7061781a8c55bd0db44f789f8152e19d9526cbed6021", + "sha256:f5672082576d0e9f52fa0fa732ff57254d65faeb4a471bc339fe54b58b3e79d2", + "sha256:ffb9f8969789771e95d3c982a36be81f0adfaa7302a1d56e29f168ca15e284b8" + ], + "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", + "version": "==3.0.0a1" + }, + "pymysql": { + "hashes": [ + "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96", + "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:00aa050faf24ce5f2af643e2b86822fa1d7149649995f11bc1e769bbfbf9010b", + "sha256:09397a18733fa2a4c7680b746094f980060666ee549deafdb5e102a99ce4619b", + "sha256:0f7fdcce52cd882b559a57b484efc92e108efeeee89fab6b623aba1ac68aad2e", + "sha256:10514adc41fc8f5922728fbac13d401a1aefcf037f009e64ca3b92464e33bf0e", + "sha256:10e001a84f820fea2640e4500e12322b03afc31d8f4f6b813b44813b2a7c7e0d", + "sha256:194f2d5a7cb3739875c4d25b3fe288ab0b3dc33f7c857ba2845830c8c51170a0", + "sha256:1aac42a21a7fa6c9665392c840b295962992ddf40aecf0a88073bc5c76728117", + "sha256:1fb792051db66e09c200e7bc3bda3b1eb18a5b8eb153d2cedb2b14b56a68b8cb", + "sha256:2756485f49e7df5c2208bdc64263d19d23eba70666f14ad12d6d8278a2fff65f", + "sha256:2b791577c546b6bbd7b43953565fcb0a2fec63643ad605353dd48afbc3c48317", + "sha256:420bc6d06d4ae7fb6921524334689eebcbea7bf2005efef070a8562cc9527a37", + "sha256:45b07470571bda5ee7f5ec471271bbde97267cc8403fce05e280c36ea73f4754", + "sha256:4ebc542d2289c0b016d6945fd07a7e2e23f4abc41e731ac8ad18a9e0c2fd0ec2", + "sha256:556dc18e39b6edb76239acfd1c010e37395a54c7fde8c57481c15819a3ffb13e", + "sha256:589aba9a35869695b319ed76c6f673d896cd01a7ff78054be1596df7ad9b096f", + "sha256:5c95e3e7cc6285bf7ff263eabb0d3bfe3def9a1ff98124083d45e5ece72f4579", + "sha256:5dd574a37be388512c72fe0d7318cb8e31743a9b2699847a025e0c08c5bf579d", + "sha256:67fbb40db3985c0cfb942fe8853ad94a5e9702d2987dec03abadc2f3b6a24afb", + "sha256:6852cd34d96835e4c9091c1e6087325efb5b607b75fd9f7075616197d1c4688a", + "sha256:69ae0e9509c43474e33152abe1385b8954922544616426bf793481e1a37e094f", + "sha256:6c5bae4c288bda92a7550fe8de9e068c0a7cd56b1c5d888aae5b40f0e13b40bd", + "sha256:774bd401e7993452ba0596e741c0c4d6d22f882dd2a798993859181dbffadc62", + "sha256:79228a7b90d95957354f37b9d46f2cc8926262ae17b0d3ed8f36c892f2a37e06", + "sha256:7b8cba5a25e95041e3413d91f9e50616bcfaec95afa038ce7dc02efefe576745", + "sha256:7db97eabd440327c35b751d5ebf78a107f505586485159bcc87660da8bb1fdca", + "sha256:7ddd6d35c598af872f9a0a5bce7f7c4a1841684a72dab3302e3df7f17d1b5249", + "sha256:82edf3a6090554a83942cec79151d6b5eb96e63d143e80e4cf6671e5d772f6be", + "sha256:8b7b3ebfa9416c8eafaffa65216e229480c495e305a06ba176dcac32710744e6", + "sha256:8da677135eff43502b7afab5a1e641edfb2dc734ba7fc146e9b1b86817a728e2", + "sha256:908c850b98cac1e203ababd4ba76868d19ae0d7172cdc75d3f1b7829b16837d2", + "sha256:9da4ee8f711e077633730955c8f3cd2485c9abf5ea0f80aac23221a3224b9a8c", + "sha256:a6f1d8256d06f58e6ece150fbe05c63c7f9510df99ee8ac37423f5476a2cebb4", + "sha256:afb322ca05e2603deedbcd2e9910f11a3fd2f42bdeafe63018e5641945c7491c", + "sha256:b52c6741073de5a744d27329f9803938dcad5c9fee7e61690c705f72973f4175", + "sha256:ba633b51835036ff0f402c21f3ff567c565a22ff0a5732b060a68f4660e2a38f", + "sha256:bfa1a0f83bdf8061db8d17c2029454722043f1e4dd1b3d3d3120d1b54e75825a", + "sha256:bffd6cd47c2e68970039c0d3e355c9ed761d3ca727b204e63cd294cad0e3df90", + "sha256:d7a2c1e711ce59ac9d0bba780318bcd102d2958bb423209f24c6354d8c4da930", + "sha256:da46beef0ce882546d92b7b2e8deb9e04dbb8fec72945a8eb28b347ca46bc15a", + "sha256:ebdd2418ab4e2e26d572d9a1c03877f8514a9b7436729525aa571862507b3fea", + "sha256:fc44e50f9d5e96af1a561faa36863f9191f27364a4df3eb70bca66e9370480b6" + ], + "index": "pypi", + "version": "==2.0.18" + }, + "tenacity": { + "hashes": [ + "sha256:2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0", + "sha256:43af037822bd0029025877f3b2d97cc4d7bb0c2991000a3d59d71517c5c969e0" + ], + "index": "pypi", + "version": "==8.2.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" + ], + "markers": "python_version >= '3.7'", + "version": "==4.7.1" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:86e9303b5e5c8160872b2f5ef611161b2893e9bfe8ccc7e2f76385947d57a2f1", + "sha256:f9849cdd62108cb739dbcdbfb7fdcc9a30d1b63c4cc3e1c1f893b5360941b61c" + ], + "index": "pypi", + "version": "==2.0.2" + }, + "flake8": { + "hashes": [ + "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7", + "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181" + ], + "index": "pypi", + "version": "==6.0.0" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", + "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" + ], + "markers": "python_version >= '3.6'", + "version": "==2.10.0" + }, + "pyflakes": { + "hashes": [ + "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf", + "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.1" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + } + } +} diff --git a/ecs/jskult-dbdump/README.md b/ecs/jskult-dbdump/README.md new file mode 100644 index 00000000..7d8f0ccd --- /dev/null +++ b/ecs/jskult-dbdump/README.md @@ -0,0 +1,48 @@ +# 日次バッチ処理前DBダンプ取得　 + +## 概要 + +日次バッチ処理前DBダンプ取得処理。 + +## 環境情報 + +- Python 3.9 +- MySQL 8.23 +- VSCode + +## 環境構築 + +- Python の構築 + + - Merck_NewDWH 開発 2021 の Wiki、[Python 環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)を参照 + - 「Pipenv の導入」までを行っておくこと + - 構築完了後、プロジェクト配下で以下のコマンドを実行し、Python の仮想環境を作成する + - `pipenv install --dev --python ` + - この手順で出力される仮想環境のパスは、後述する VSCode の設定手順で使用するため、控えておく + +- MySQL の環境構築 + - Windows の場合、以下のリンクからダウンロードする + - + - Docker を利用する場合、「newsdwh-tools」リポジトリの MySQL 設定を使用すると便利 + - 「crm-table-to-ddl」フォルダ内で以下のコマンドを実行すると + - `docker-compose up -d` + - Docker の構築手順は、[Docker のセットアップ手順](https://nds-tyo.backlog.com/alias/wiki/1754332)を参照のこと + - データを投入する + - 立ち上げたデータベースに「src05」スキーマを作成する + - [ローカル開発用データ](https://ndstokyo.sharepoint.com/:f:/r/sites/merck-new-dwh-team/Shared%20Documents/03.NewDWH%E6%A7%8B%E7%AF%89%E3%83%95%E3%82%A7%E3%83%BC%E3%82%BA3/02.%E9%96%8B%E7%99%BA/90.%E9%96%8B%E7%99%BA%E5%85%B1%E6%9C%89/%E3%83%AD%E3%83%BC%E3%82%AB%E3%83%AB%E9%96%8B%E7%99%BA%E7%94%A8%E3%83%87%E3%83%BC%E3%82%BF?csf=1&web=1&e=VVcRUs)をダウンロードし、mysql コマンドを使用して復元する + - `mysql -h <ホスト名> -P <ポート> -u <ユーザー名> -p src05 < src05_dump.sql` +- 環境変数の設定 + - 「.env.example」ファイルをコピーし、「.env」ファイルを作成する + - 環境変数を設定する。設定内容は PRJ メンバーより共有を受けてください +- VSCode の設定 + - 「.vscode/recommended_settings.json」ファイルをコピーし、「settings.json」ファイルを作成する + - 「python.defaultInterpreterPath」を、Python の構築手順で作成した仮想環境のパスに変更する + +## 実行 + +- VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 +- 「entrypoint.py」が、バッチ処理のエントリーポイント。 +- 実際の処理は、「src/jobctrl_dbdump.py」で行っている。 + + +## フォルダ構成（工事中） diff --git a/ecs/jskult-dbdump/entrypoint.py b/ecs/jskult-dbdump/entrypoint.py new file mode 100644 index 00000000..543cb134 --- /dev/null +++ b/ecs/jskult-dbdump/entrypoint.py @@ -0,0 +1,10 @@ +"""実消化&アルトマーク 日次バッチ処理前DBダンプ取得のエントリーポイント""" +from src import jobctrl_dbdump + +if __name__ == '__main__': + try: + exit(jobctrl_dbdump.exec()) + except Exception: + # エラーが起きても、正常系のコードで返す。 + # エラーが起きた事実はbatch_process内でログを出す。 + exit(0) diff --git a/ecs/jskult-dbdump/mysql_dpkg_selection.txt b/ecs/jskult-dbdump/mysql_dpkg_selection.txt new file mode 100644 index 00000000..d3cb46f9 --- /dev/null +++ b/ecs/jskult-dbdump/mysql_dpkg_selection.txt @@ -0,0 +1,3 @@ +1 +1 +4 \ No newline at end of file diff --git a/ecs/jskult-dbdump/src/__init__.py b/ecs/jskult-dbdump/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/batch/batch_functions.py b/ecs/jskult-dbdump/src/batch/batch_functions.py new file mode 100644 index 00000000..87231e17 --- /dev/null +++ b/ecs/jskult-dbdump/src/batch/batch_functions.py @@ -0,0 +1,118 @@ +"""バッチ処理の共通関数""" +import logging +import textwrap + +from src.db.database import Database +from src.error.exceptions import BatchOperationException, DBException +from src.system_var import constants + + +def get_batch_statuses() -> tuple[str, str]: + """日付テーブルから、以下を取得して返す。 + - バッチ処理中フラグ + - dump取得状況区分 + + Raises: + BatchOperationException: 日付テーブルが取得できないとき、何らかのエラーが発生したとき + + Returns: + tuple[str, str]: [0]バッチ処理中フラグ，[1]dump取得状況区分 + """ + db = Database.get_instance() + sql = 'SELECT bch_actf, dump_sts_kbn FROM src05.hdke_tbl' + try: + db.connect() + hdke_tbl_result = db.execute_select(sql) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + if len(hdke_tbl_result) == 0: + raise BatchOperationException('日付テーブルが取得できませんでした') + + # 必ず1件取れる + hdke_tbl_record = hdke_tbl_result[0] + batch_processing_flag = hdke_tbl_record['bch_actf'] + dump_status_kbn = hdke_tbl_record['dump_sts_kbn'] + + return batch_processing_flag, dump_status_kbn + + +def update_dump_status_kbn_in_processing() -> None: + """dump取得状況区分を処理中に更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = 'UPDATE src05.hdke_tbl SET dump_sts_kbn = :in_processing' + try: + db.connect() + db.execute(sql, {'in_processing': constants.BATCH_ACTF_BATCH_IN_PROCESSING}) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def update_dump_status_kbn_error() -> None: + """dump取得状況区分をエラーに更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = """\ + UPDATE src05.hdke_tbl + SET + dump_sts_kbn = :dump_unprocessed + """ + try: + db.connect() + db.execute(sql, { + 'dump_unprocessed': constants.DUMP_STATUS_KBN_ERROR + }) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def update_dump_status_kbn_complete() -> None: + """dump取得状況区分を正常終了に更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = """\ + UPDATE src05.hdke_tbl + SET + dump_sts_kbn = :dump_unprocessed + """ + try: + db.connect() + db.execute(sql, { + 'dump_unprocessed': constants.DUMP_STATUS_KBN_COMPLETE + }) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def logging_sql(logger: logging.Logger, sql: str) -> None: + """SQL文をデバッグログで出力する + + Args: + logger (logging.Logger): ロガー + sql (str): SQL文 + """ + logger.debug(f'\n{"-" * 15}\n{textwrap.dedent(sql)[1:-1]}\n{"-" * 15}') diff --git a/ecs/jskult-dbdump/src/batch/common/__init__.py b/ecs/jskult-dbdump/src/batch/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/db/__init__.py b/ecs/jskult-dbdump/src/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/db/database.py b/ecs/jskult-dbdump/src/db/database.py new file mode 100644 index 00000000..b675381c --- /dev/null +++ b/ecs/jskult-dbdump/src/db/database.py @@ -0,0 +1,177 @@ +from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, + create_engine, text) +from sqlalchemy.engine.url import URL +from tenacity import retry, stop_after_attempt, wait_exponential + +from src.error.exceptions import DBException +from src.system_var import environment + + +class Database: + """データベース操作クラス""" + __connection: Connection = None + __engine: Engine = None + __host: str = None + __port: str = None + __username: str = None + __password: str = None + __schema: str = None + __connection_string: str = None + + def __init__(self, username: str, password: str, host: str, port: int, schema: str) -> None: + """このクラスの新たなインスタンスを初期化します + + Args: + username (str): DBユーザー名 + password (str): DBパスワード + host (str): DBホスト名 + port (int): DBポート + schema (str): DBスキーマ名 + """ + self.__username = username + self.__password = password + self.__host = host + self.__port = int(port) + self.__schema = schema + + self.__connection_string = URL.create( + drivername='mysql+pymysql', + username=self.__username, + password=self.__password, + host=self.__host, + port=self.__port, + database=self.__schema, + query={"charset": "utf8mb4"} + ) + + self.__engine = create_engine( + self.__connection_string, + pool_timeout=5, + poolclass=QueuePool + ) + + @classmethod + def get_instance(cls): + """インスタンスを取得します + + Returns: + Database: DB操作クラスインスタンス + """ + return cls( + username=environment.DB_USERNAME, + password=environment.DB_PASSWORD, + host=environment.DB_HOST, + port=environment.DB_PORT, + schema=environment.DB_SCHEMA + ) + + @retry( + wait=wait_exponential( + multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT, + min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, + max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS + ), + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) + def connect(self): + """ + DBに接続します。接続に失敗した場合、リトライします。 + Raises: + DBException: 接続失敗 + """ + try: + self.__connection = self.__engine.connect() + except Exception as e: + raise DBException(e) + + def execute_select(self, select_query: str, parameters=None) -> list[dict]: + """SELECTクエリを実行します。 + + Args: + select_query (str): SELECT文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + list[dict]: カラム名: 値の辞書リスト + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(select_query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(select_query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + result_rows = result.mappings().all() + return result_rows + + def execute(self, query: str, parameters=None) -> CursorResult: + """SQLクエリを実行します。 + + Args: + query (str): SQL文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + CursorResult: 取得結果 + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + return result + + def begin(self): + """トランザクションを開始します。""" + if not self.__connection.in_transaction(): + self.__connection.begin() + + def commit(self): + """トランザクションをコミットします""" + if self.__connection.in_transaction(): + self.__connection.commit() + + def rollback(self): + """トランザクションをロールバックします""" + if self.__connection.in_transaction(): + self.__connection.rollback() + + def disconnect(self): + """DB接続を切断します。""" + if self.__connection is not None: + self.__connection.close() + self.__connection = None + + def __execute_with_transaction(self, query: str, parameters: dict): + # トランザクションを開始してクエリを実行する + with self.__connection.begin(): + try: + result = self.__connection.execute(text(query), parameters=parameters) + except Exception as e: + self.__connection.rollback() + raise e + # ここでコミットされる + return result diff --git a/ecs/jskult-dbdump/src/error/__init__.py b/ecs/jskult-dbdump/src/error/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/error/exceptions.py b/ecs/jskult-dbdump/src/error/exceptions.py new file mode 100644 index 00000000..055c24f6 --- /dev/null +++ b/ecs/jskult-dbdump/src/error/exceptions.py @@ -0,0 +1,10 @@ +class MeDaCaException(Exception): + pass + + +class DBException(MeDaCaException): + pass + + +class BatchOperationException(MeDaCaException): + pass diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py new file mode 100644 index 00000000..3dac256f --- /dev/null +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -0,0 +1,128 @@ +"""日次バッチ処理前DBダンプ取得""" + +import datetime +import os +import subprocess +import textwrap + +from src.batch.batch_functions import (get_batch_statuses, + update_dump_status_kbn_complete, + update_dump_status_kbn_error, + update_dump_status_kbn_in_processing) +from src.error.exceptions import BatchOperationException +from src.logging.get_logger import get_logger +from src.system_var import constants, environment + +logger = get_logger('日次バッチ処理前DBダンプ取得') + + +def exec(): + try: + logger.info('日次バッチ処理前DBダンプ取得：開始') + try: + # 日次バッチ処置中フラグ、dump処理状態区分を取得 + batch_processing_flag, dump_status_kbn = get_batch_statuses() + except BatchOperationException as e: + logger.exception(f'日付テーブル取得エラー（異常終了）:{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # 日次バッチ処理中の場合、処理は行わない + if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: + logger.error('日次バッチ処理中の為、日次バッチ処理前DBダンプ取得を終了します。') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分が処理中の場合、処理は行わない + if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED: + logger.error(f'ダンプ取得中の為、日次バッチ処理前DBダンプ取得を終了します。 dump処理状態区分={dump_status_kbn}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分がエラーの場合、処理は行わない + if dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: + logger.error(f'ダンプ取得が実行不可能な状態の為、日次バッチ処理前DBダンプ取得を終了します。 dump処理状態区分={dump_status_kbn}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分を処理中に更新 + try: + update_dump_status_kbn_in_processing() + except BatchOperationException as e: + logger.exception(f'dump処理状態区分更新(未処理→処理中) エラー（異常終了）:{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # MySQL接続情報を作成する + my_cnf_file_content = f""" + [client] + user={environment.DB_USERNAME} + password={environment.DB_PASSWORD} + host={environment.DB_HOST} + """ + # my.cnfファイルのパス + my_cnf_path = os.path.join('my.cnf') + + # my.cnfファイルを生成する + with open(my_cnf_path, 'w') as f: + f.write(textwrap.dedent(my_cnf_file_content)[1:-1]) + + dt_now = datetime.datetime.now() + converted_value = dt_now.strftime('%Y%m%d%H%M%S') + dump_file_name = f'backup_rds_src05_{converted_value}.gz' + s3_file_path = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{dump_file_name}' + + # mysqldumpコマンドを実行し、dumpを取得する + command = [ + 'mysqldump', + f'--defaults-file={my_cnf_path}', + '-P', + f"{environment.DB_PORT}", + '--no-tablespaces', + '--skip-column-statistics', + '--single-transaction', + '--set-gtid-purged=OFF', + environment.DB_SCHEMA + ] + + mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # gzipコマンドを実行してdump結果を圧縮する + gzip_process = subprocess.Popen(['gzip', '-c'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # aws s3 cpコマンドを実行してアップロードする + s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_path], stdin=gzip_process.stdout, stderr=subprocess.PIPE) + # mysqldumpの標準出力をgzipに接続したため、標準出力をクローズする + mysqldump_process.stdout.close() + # gzipの標準出力をaws s3 cpに接続したため、標準出力をクローズする + gzip_process.stdout.close() + + # パイプラインを実行し、エラーハンドリング + _, error = mysqldump_process.communicate() + if mysqldump_process.returncode != 0: + logger.error(f'`mysqldump`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS + + _, error = gzip_process.communicate() + if gzip_process.returncode != 0: + logger.error(f'`gzip`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS + + _, error = s3_cp_process.communicate() + if s3_cp_process.returncode != 0: + logger.error(f'`aws s3 cp`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分を正常終了に更新 + try: + update_dump_status_kbn_complete() + except BatchOperationException as e: + logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）:{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + logger.info('日次バッチ処理前DBダンプ取得：終了（正常終了）') + logger.info(f'出力ファイルパス: {s3_file_path}') + return constants.BATCH_EXIT_CODE_SUCCESS + + except Exception as e: + # dump処理状態区分をエラーに更新 + try: + update_dump_status_kbn_error() + except BatchOperationException as e: + logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）:{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました :{e}') + return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-dbdump/src/logging/get_logger.py b/ecs/jskult-dbdump/src/logging/get_logger.py new file mode 100644 index 00000000..f36f1199 --- /dev/null +++ b/ecs/jskult-dbdump/src/logging/get_logger.py @@ -0,0 +1,37 @@ +import logging + +from src.system_var.environment import LOG_LEVEL + +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + + +def get_logger(log_name: str) -> logging.Logger: + """一意のログ出力モジュールを取得します。 + + Args: + log_name (str): ロガー名 + + Returns: + _type_: _description_ + """ + logger = logging.getLogger(log_name) + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + logger.setLevel(level) + + if not logger.hasHandlers(): + handler = logging.StreamHandler() + logger.addHandler(handler) + + formatter = logging.Formatter( + '%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in logger.handlers: + handler.setFormatter(formatter) + + return logger diff --git a/ecs/jskult-dbdump/src/system_var/__init__.py b/ecs/jskult-dbdump/src/system_var/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/system_var/constants.py b/ecs/jskult-dbdump/src/system_var/constants.py new file mode 100644 index 00000000..993e5a7d --- /dev/null +++ b/ecs/jskult-dbdump/src/system_var/constants.py @@ -0,0 +1,15 @@ +# バッチ正常終了コード +BATCH_EXIT_CODE_SUCCESS = 0 + +# バッチ処理中フラグ：未処理 +BATCH_ACTF_BATCH_UNPROCESSED = '0' +# バッチ処理中フラグ：処理中 +BATCH_ACTF_BATCH_IN_PROCESSING = '1' +# dump取得状態区分：未処理 +DUMP_STATUS_KBN_UNPROCESSED = '0' +# dump取得状態区分：処理中 +DUMP_STATUS_KBN_PROCESSED = '1' +# dump取得状態区分：正常終了 +DUMP_STATUS_KBN_COMPLETE = '2' +# dump取得状態区分：エラー +DUMP_STATUS_KBN_ERROR = '9' diff --git a/ecs/jskult-dbdump/src/system_var/environment.py b/ecs/jskult-dbdump/src/system_var/environment.py new file mode 100644 index 00000000..ff510581 --- /dev/null +++ b/ecs/jskult-dbdump/src/system_var/environment.py @@ -0,0 +1,20 @@ +import os + +# Database +DB_HOST = os.environ['DB_HOST'] +DB_PORT = int(os.environ['DB_PORT']) +DB_USERNAME = os.environ['DB_USERNAME'] +DB_PASSWORD = os.environ['DB_PASSWORD'] +DB_SCHEMA = os.environ['DB_SCHEMA'] + +# AWS +JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] +DUMP_BACKUP_FOLDER = os.environ['DUMP_BACKUP_FOLDER'] + +# 初期値がある環境変数 +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') +DB_CONNECTION_MAX_RETRY_ATTEMPT = int(os.environ.get('DB_CONNECTION_MAX_RETRY_ATTEMPT', 4)) +DB_CONNECTION_RETRY_INTERVAL_INIT = int(os.environ.get('DB_CONNECTION_RETRY_INTERVAL', 5)) +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5)) +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50)) + diff --git a/ecs/jskult-restore-backup/.dockerignore b/ecs/jskult-restore-backup/.dockerignore new file mode 100644 index 00000000..d826198a --- /dev/null +++ b/ecs/jskult-restore-backup/.dockerignore @@ -0,0 +1,13 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md +*.sql +*.gz diff --git a/ecs/jskult-restore-backup/.env.example b/ecs/jskult-restore-backup/.env.example new file mode 100644 index 00000000..2bda940a --- /dev/null +++ b/ecs/jskult-restore-backup/.env.example @@ -0,0 +1,9 @@ +DB_HOST=************ +DB_PORT=3306 +DB_USERNAME=************ +DB_PASSWORD=************ +DB_SCHEMA=src05 + +JSKULT_BACKUP_DUMP_S3_FILE_PATH=******************* + +LOG_LEVEL=INFO diff --git a/ecs/jskult-restore-backup/.gitignore b/ecs/jskult-restore-backup/.gitignore new file mode 100644 index 00000000..339d80cc --- /dev/null +++ b/ecs/jskult-restore-backup/.gitignore @@ -0,0 +1,16 @@ +.vscode/settings.json +.env + +# python +__pycache__ + +# python test +.pytest_cache +.coverage +.report/ + +# mysql config file +my.cnf + +# compress file +*.gz \ No newline at end of file diff --git a/ecs/jskult-restore-backup/.vscode/launch.json b/ecs/jskult-restore-backup/.vscode/launch.json new file mode 100644 index 00000000..a0178c26 --- /dev/null +++ b/ecs/jskult-restore-backup/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "(DEBUG)jskult batch dbdump", + "type": "python", + "request": "launch", + "program": "entrypoint.py", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/jskult-restore-backup/.vscode/recommended_settings.json b/ecs/jskult-restore-backup/.vscode/recommended_settings.json new file mode 100644 index 00000000..b5e79d73 --- /dev/null +++ b/ecs/jskult-restore-backup/.vscode/recommended_settings.json @@ -0,0 +1,31 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + // 自身の環境に合わせて変えてください + "python.defaultInterpreterPath": "", + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--max-line-length=200", + "--ignore=F541" + ], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Path": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", "200", + "--ignore=F541" + ], + "python.testing.pytestArgs": [ + "tests/batch/ultmarc" + ], + + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ecs/jskult-restore-backup/Dockerfile b/ecs/jskult-restore-backup/Dockerfile new file mode 100644 index 00000000..3c410b72 --- /dev/null +++ b/ecs/jskult-restore-backup/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3.9 + +ENV TZ="Asia/Tokyo" + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +# mysql-apt-config をdpkgでインストールする際に標準出力に渡す文字列ファイルをコピー +COPY mysql_dpkg_selection.txt ./ +# 必要なパッケージインストール +RUN apt update && apt install -y less vim curl wget gzip unzip sudo lsb-release + +# mysqlをインストール +RUN \ + wget https://dev.mysql.com/get/mysql-apt-config_0.8.25-1_all.deb && \ + dpkg -i mysql-apt-config_0.8.25-1_all.deb < mysql_dpkg_selection.txt && \ + apt update && \ + apt install -y mysql-client + +# aws cli v2 のインストール +RUN \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + sudo ./aws/install + +# python関連のライブラリインストール +RUN \ + pip install --upgrade pip wheel setuptools && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv + +# パッケージのセキュリティアップデートのみを適用するコマンドを実行 +RUN \ + apt install -y unattended-upgrades && \ + unattended-upgrades + +COPY src ./src +COPY entrypoint.py entrypoint.py + +CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-restore-backup/Pipfile b/ecs/jskult-restore-backup/Pipfile new file mode 100644 index 00000000..f032c7ea --- /dev/null +++ b/ecs/jskult-restore-backup/Pipfile @@ -0,0 +1,16 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.9" + +[pipenv] +allow_prereleases = true diff --git a/ecs/jskult-restore-backup/Pipfile.lock b/ecs/jskult-restore-backup/Pipfile.lock new file mode 100644 index 00000000..f948e3df --- /dev/null +++ b/ecs/jskult-restore-backup/Pipfile.lock @@ -0,0 +1,69 @@ +{ + "_meta": { + "hash": { + "sha256": "cc5f54bfb2073051a26f113ceac64e12fdd0bf8faa36f1a42210cc9c921c134b" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": { + "autopep8": { + "hashes": [ + "sha256:86e9303b5e5c8160872b2f5ef611161b2893e9bfe8ccc7e2f76385947d57a2f1", + "sha256:f9849cdd62108cb739dbcdbfb7fdcc9a30d1b63c4cc3e1c1f893b5360941b61c" + ], + "index": "pypi", + "version": "==2.0.2" + }, + "flake8": { + "hashes": [ + "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7", + "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181" + ], + "index": "pypi", + "version": "==6.0.0" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", + "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" + ], + "markers": "python_version >= '3.6'", + "version": "==2.10.0" + }, + "pyflakes": { + "hashes": [ + "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf", + "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.1" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + } + } +} diff --git a/ecs/jskult-restore-backup/README.md b/ecs/jskult-restore-backup/README.md new file mode 100644 index 00000000..e9dfa8f7 --- /dev/null +++ b/ecs/jskult-restore-backup/README.md @@ -0,0 +1,67 @@ +# 実消化＆アルトマークダンプ復元スクリプト + +## 概要 + +実消化＆アルトマークダンプ復元スクリプト + +## 環境情報 + +- Python 3.9 +- MySQL 8.23 +- VSCode + +## 環境構築 + +- Python の構築 + + - Merck_NewDWH 開発 2021 の Wiki、[Python 環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)を参照 + - 「Pipenv の導入」までを行っておくこと + - 構築完了後、プロジェクト配下で以下のコマンドを実行し、Python の仮想環境を作成する + - `pipenv install --dev --python ` + - この手順で出力される仮想環境のパスは、後述する VSCode の設定手順で使用するため、控えておく + +- MySQL の環境構築 + - Windows の場合、以下のリンクからダウンロードする + - + - Docker を利用する場合、「newsdwh-tools」リポジトリの MySQL 設定を使用すると便利 + - 「crm-table-to-ddl」フォルダ内で以下のコマンドを実行すると + - `docker-compose up -d` + - Docker の構築手順は、[Docker のセットアップ手順](https://nds-tyo.backlog.com/alias/wiki/1754332)を参照のこと + - データを投入する + - 立ち上げたデータベースに「src05」スキーマを作成する + - [ローカル開発用データ](https://ndstokyo.sharepoint.com/:f:/r/sites/merck-new-dwh-team/Shared%20Documents/03.NewDWH%E6%A7%8B%E7%AF%89%E3%83%95%E3%82%A7%E3%83%BC%E3%82%BA3/02.%E9%96%8B%E7%99%BA/90.%E9%96%8B%E7%99%BA%E5%85%B1%E6%9C%89/%E3%83%AD%E3%83%BC%E3%82%AB%E3%83%AB%E9%96%8B%E7%99%BA%E7%94%A8%E3%83%87%E3%83%BC%E3%82%BF?csf=1&web=1&e=VVcRUs)をダウンロードし、mysql コマンドを使用して復元する + - `mysql -h <ホスト名> -P <ポート> -u <ユーザー名> -p src05 < src05_dump.sql` +- 環境変数の設定 + - 「.env.example」ファイルをコピーし、「.env」ファイルを作成する + - 環境変数を設定する。設定内容は PRJ メンバーより共有を受けてください +- VSCode の設定 + - 「.vscode/recommended_settings.json」ファイルをコピーし、「settings.json」ファイルを作成する + - 「python.defaultInterpreterPath」を、Python の構築手順で作成した仮想環境のパスに変更する + +## 実行 + +- VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 +- 「entrypoint.py」が、バッチ処理のエントリーポイント。 +- 実際の処理は、「src/restore_backup.py」で行っている。 + +## フォルダ構成 + +```txt +. +├── .env.example -- ローカル実行用の環境変数のサンプル値。 +├── .dockerignore -- docker build時のコンテキストに含めるファイルの抑制リスト +├── .gitignore -- Git差分管理除外リスト +├── Dockerfile -- Dockerイメージ作成用 +├── Pipfile -- pythonの依存関係管理 +├── Pipfile.lock -- 依存関係バージョン固定 +├── README.md -- 当ファイル +├── entrypoint.py -- エントリーポイントとなるファイル +├── mysql_dpkg_selection.txt -- Dockerイメージでdpkgを使うときに外部から注入する選択値 +└── src -- ソースコードフォルダ + ├── logging + │ └── get_logger.py -- ロガー + ├── restore_backup.py -- dump復元処理本体 + └── system_var + ├── constants.py -- 定数ファイル + └── environment.py -- 環境変数ファイル +``` diff --git a/ecs/jskult-restore-backup/entrypoint.py b/ecs/jskult-restore-backup/entrypoint.py new file mode 100644 index 00000000..70f48060 --- /dev/null +++ b/ecs/jskult-restore-backup/entrypoint.py @@ -0,0 +1,10 @@ +"""実消化&アルトマーク DBダンプ復元のエントリーポイント""" +from src import restore_backup + +if __name__ == '__main__': + try: + exit(restore_backup.exec()) + except Exception: + # エラーが起きても、正常系のコードで返す。 + # エラーが起きた事実はbatch_process内でログを出す。 + exit(0) diff --git a/ecs/jskult-restore-backup/mysql_dpkg_selection.txt b/ecs/jskult-restore-backup/mysql_dpkg_selection.txt new file mode 100644 index 00000000..d3cb46f9 --- /dev/null +++ b/ecs/jskult-restore-backup/mysql_dpkg_selection.txt @@ -0,0 +1,3 @@ +1 +1 +4 \ No newline at end of file diff --git a/ecs/jskult-restore-backup/src/__init__.py b/ecs/jskult-restore-backup/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-restore-backup/src/logging/get_logger.py b/ecs/jskult-restore-backup/src/logging/get_logger.py new file mode 100644 index 00000000..f36f1199 --- /dev/null +++ b/ecs/jskult-restore-backup/src/logging/get_logger.py @@ -0,0 +1,37 @@ +import logging + +from src.system_var.environment import LOG_LEVEL + +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + + +def get_logger(log_name: str) -> logging.Logger: + """一意のログ出力モジュールを取得します。 + + Args: + log_name (str): ロガー名 + + Returns: + _type_: _description_ + """ + logger = logging.getLogger(log_name) + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + logger.setLevel(level) + + if not logger.hasHandlers(): + handler = logging.StreamHandler() + logger.addHandler(handler) + + formatter = logging.Formatter( + '%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in logger.handlers: + handler.setFormatter(formatter) + + return logger diff --git a/ecs/jskult-restore-backup/src/restore_backup.py b/ecs/jskult-restore-backup/src/restore_backup.py new file mode 100644 index 00000000..439fe4cf --- /dev/null +++ b/ecs/jskult-restore-backup/src/restore_backup.py @@ -0,0 +1,69 @@ +"""実消化＆アルトマークダンプ復元スクリプト""" + +import os +import re +import subprocess +import textwrap + +from src.logging.get_logger import get_logger +from src.system_var import constants, environment + +logger = get_logger('実消化＆アルトマークダンプ復元スクリプト') + + +def exec(): + try: + logger.info('実消化＆アルトマークダンプ復元スクリプト：開始') + + # MySQL接続情報を作成する + my_cnf_file_content = f""" + [client] + user={environment.DB_USERNAME} + password={environment.DB_PASSWORD} + host={environment.DB_HOST} + """ + # my.cnfファイルのパス + my_cnf_path = os.path.join('my.cnf') + + # my.cnfファイルを生成する + with open(my_cnf_path, 'w') as f: + f.write(textwrap.dedent(my_cnf_file_content)[1:-1]) + + # 復元対象のダンプファイルを特定 + s3_file_path = environment.JSKULT_BACKUP_DUMP_S3_FILE_PATH + + # S3 URIになっているか確認 + s3_file_path_pattern = r"s3://(.+)/dump/(\d{4})/(\d{2})/(\d{2})/backup_rds_src05_(\d+).gz" + if re.match(s3_file_path_pattern, s3_file_path) is None: + logger.warning('環境変数[JSKULT_BACKUP_DUMP_S3_FILE_PATH]に指定された、S3 URIの形式が不正です。') + return constants.BATCH_EXIT_CODE_SUCCESS + + # aws s3 cpコマンドを実行してdumpファイルをローカルにダウンロードする + s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', s3_file_path, './dump.gz'], stderr=subprocess.PIPE) + _, error = s3_cp_process.communicate() + if s3_cp_process.returncode != 0: + logger.error(f'`aws s3 cp`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS + # S3コマンドの標準エラーはクローズしておく + s3_cp_process.stderr.close() + + # gzipコマンドを実行してdumpファイルを解凍する + gzip_process = subprocess.Popen(['gunzip', '-c', './dump.gz'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # mysqlコマンドを実行し、dumpを復元する + mysql_process = subprocess.Popen( + ['mysql', f'--defaults-file={my_cnf_path}', '-P', f"{environment.DB_PORT}", environment.DB_SCHEMA], + stdin=gzip_process.stdout, stderr=subprocess.PIPE + ) + # gzipの標準出力をmysqlに接続したため、標準出力をクローズする + gzip_process.stdout.close() + + _, error = mysql_process.communicate() + if mysql_process.returncode != 0: + logger.error(f'コマンド実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS + logger.info('[NOTICE]実消化＆アルトマークダンプ復元スクリプト：終了（正常終了）') + return constants.BATCH_EXIT_CODE_SUCCESS + + except Exception as e: + logger.exception(f'実消化＆アルトマークダンプ復元スクリプト中に想定外のエラーが発生しました :{e}') + return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-restore-backup/src/system_var/__init__.py b/ecs/jskult-restore-backup/src/system_var/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-restore-backup/src/system_var/constants.py b/ecs/jskult-restore-backup/src/system_var/constants.py new file mode 100644 index 00000000..8a555af3 --- /dev/null +++ b/ecs/jskult-restore-backup/src/system_var/constants.py @@ -0,0 +1,2 @@ +# バッチ正常終了コード +BATCH_EXIT_CODE_SUCCESS = 0 diff --git a/ecs/jskult-restore-backup/src/system_var/environment.py b/ecs/jskult-restore-backup/src/system_var/environment.py new file mode 100644 index 00000000..ee0e9be3 --- /dev/null +++ b/ecs/jskult-restore-backup/src/system_var/environment.py @@ -0,0 +1,14 @@ +import os + +# Database +DB_HOST = os.environ['DB_HOST'] +DB_PORT = int(os.environ['DB_PORT']) +DB_USERNAME = os.environ['DB_USERNAME'] +DB_PASSWORD = os.environ['DB_PASSWORD'] +DB_SCHEMA = os.environ['DB_SCHEMA'] + +# AWS +JSKULT_BACKUP_DUMP_S3_FILE_PATH = os.environ['JSKULT_BACKUP_DUMP_S3_FILE_PATH'] + +# 初期値がある環境変数 +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') diff --git a/ecs/jskult-webapp/src/db/database.py b/ecs/jskult-webapp/src/db/database.py index 85759f96..f700c016 100644 --- a/ecs/jskult-webapp/src/db/database.py +++ b/ecs/jskult-webapp/src/db/database.py @@ -1,10 +1,28 @@ from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, - create_engine, text) + create_engine, event, exc, text) from sqlalchemy.engine.url import URL +from sqlalchemy.pool import Pool from src.error.exceptions import DBException +from src.logging.get_logger import get_logger from src.system_var import environment +logger = get_logger('DB接続') + + +@event.listens_for(Pool, 'checkout') +def ping_connection(dbapi_connection, connection_record, connection_proxy): + """コネクションが切れた場合、再接続""" + cursor = dbapi_connection.cursor() + try: + cursor.execute("SELECT 1") + except Exception as e: + logger.info(f'DB接続に失敗したため、リトライします: {e}') + # raise DisconnectionError - pool will try + # connecting again up to three times before raising. + raise exc.DisconnectionError() + cursor.close() + class Database: """データベース操作クラス"""