From 97462420a1b1954a65d82f0b6c98fe0130b6b9bd Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Sat, 16 Sep 2023 15:25:36 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20V=E6=96=BD=E8=A8=AD=E3=83=9E=E3=82=B9?= =?UTF-8?q?=E3=82=BF=E3=81=AE=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/vjsk/vjsk_recv_file_mapper.py | 82 +++++++++---------- .../batch/vjsk/vjsk_load/test_vjsk_load.py | 14 ++-- .../testdata/fcl_mst_202304280000.tsv | 2 + .../testdata/fcl_mst_202304290000.tsv | 4 +- .../tests/testing_vjsk_utility.py | 7 +- 5 files changed, 59 insertions(+), 50 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py index 025276dc..51b51cd0 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py @@ -720,57 +720,57 @@ class VjskReceiveFileMapper: ,dwh_upd_dt ) SELECT - t.v_inst_cd + t.v_inst_cd ,t.sub_num ,t.start_date ,t.end_date - ,t.closed_dt - ,t.fcl_name - ,t.fcl_kn_name - ,t.fcl_abb_name - ,t.fcl_abb_kn_name - ,t.mkr_cd + ,medaca_common.convert_empty_to_null(t.closed_dt) + ,medaca_common.convert_empty_to_null(t.fcl_name) + ,medaca_common.convert_empty_to_null(t.fcl_kn_name) + ,medaca_common.convert_empty_to_null(t.fcl_abb_name) + ,medaca_common.convert_empty_to_null(t.fcl_abb_kn_name) + ,medaca_common.convert_empty_to_null(t.mkr_cd) ,t.jsk_proc_kbn - ,t.fmt_addr - ,t.fmt_kn_addr - ,t.postal_cd - ,t.prft_cd - ,t.prft_name - ,t.city_name - ,t.addr_line_1 - ,t.tel_num - ,t.admin_kbn - ,t.fcl_type + ,medaca_common.convert_empty_to_null(t.fmt_addr) + ,medaca_common.convert_empty_to_null(t.fmt_kn_addr) + ,medaca_common.convert_empty_to_null(t.postal_cd) + ,medaca_common.convert_empty_to_null(t.prft_cd) + ,medaca_common.convert_empty_to_null(t.prft_name) + ,medaca_common.convert_empty_to_null(t.city_name) + ,medaca_common.convert_empty_to_null(t.addr_line_1) + ,medaca_common.convert_empty_to_null(t.tel_num) + ,medaca_common.convert_empty_to_null(t.admin_kbn) + ,medaca_common.convert_empty_to_null(t.fcl_type) ,t.rec_sts_kbn ,nullif(t.ins_dt, 0) -- 受領データがブランクだった場合にゼロ日付で取得されるので明示的にNULL値に変換する ,nullif(t.upd_dt, 0) -- 受領データがブランクだった場合にゼロ日付で取得されるので明示的にNULL値に変換する ,SYSDATE() FROM org05.fcl_mst_v AS t ON DUPLICATE KEY UPDATE - v_inst_cd=t.v_inst_cd - ,sub_num=t.sub_num - ,start_date=t.start_date - ,end_date=t.end_date - ,closed_dt=t.closed_dt - ,fcl_name=t.fcl_name - ,fcl_kn_name=t.fcl_kn_name - ,fcl_abb_name=t.fcl_abb_name - ,fcl_abb_kn_name=t.fcl_abb_kn_name - ,mkr_cd=t.mkr_cd - ,jsk_proc_kbn=t.jsk_proc_kbn - ,fmt_addr=t.fmt_addr - ,fmt_kn_addr=t.fmt_kn_addr - ,postal_cd=t.postal_cd - ,prft_cd=t.prft_cd - ,prft_name=t.prft_name - ,city_name=t.city_name - ,addr_line_1=t.addr_line_1 - ,tel_num=t.tel_num - ,admin_kbn=t.admin_kbn - ,fcl_type=t.fcl_type - ,rec_sts_kbn=t.rec_sts_kbn - ,ins_dt=nullif(t.ins_dt, 0) - ,upd_dt=nullif(t.upd_dt, 0) + v_inst_cd=VALUES(v_inst_cd) + ,sub_num=VALUES(sub_num) + ,start_date=VALUES(start_date) + ,end_date=VALUES(end_date) + ,closed_dt=VALUES(closed_dt) + ,fcl_name=VALUES(fcl_name) + ,fcl_kn_name=VALUES(fcl_kn_name) + ,fcl_abb_name=VALUES(fcl_abb_name) + ,fcl_abb_kn_name=VALUES(fcl_abb_kn_name) + ,mkr_cd=VALUES(mkr_cd) + ,jsk_proc_kbn=VALUES(jsk_proc_kbn) + ,fmt_addr=VALUES(fmt_addr) + ,fmt_kn_addr=VALUES(fmt_kn_addr) + ,postal_cd=VALUES(postal_cd) + ,prft_cd=VALUES(prft_cd) + ,prft_name=VALUES(prft_name) + ,city_name=VALUES(city_name) + ,addr_line_1=VALUES(addr_line_1) + ,tel_num=VALUES(tel_num) + ,admin_kbn=VALUES(admin_kbn) + ,fcl_type=VALUES(fcl_type) + ,rec_sts_kbn=VALUES(rec_sts_kbn) + ,ins_dt=VALUES(ins_dt) + ,upd_dt=VALUES(upd_dt) ,dwh_upd_dt=SYSDATE() ; """) diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py index 109ac100..05921299 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py @@ -546,7 +546,7 @@ class TestImportFileToDb: self.batch_context.is_vjsk_stock_import_day = True self.db.execute(f"truncate table {table_name_src}") - # assertion1 (insert 4row) + # assertion1 (insert 6row) # 処理実行 target_dict = { @@ -561,13 +561,15 @@ class TestImportFileToDb: result_org = self.db.execute_select(f"select * from {table_name_org}") # 突合から除外する項目 ignore_columns = ['dwh_upd_dt'] + # 日付型に変換して比較する項目 force_cast_to_str_columns = ['closed_dt'] # orgテーブル結果が期待値通りかを突合 - assert_table_results(result_org, assert_list, ignore_columns, force_cast_to_str_columns) + assert_table_results(result_org, assert_list, ignore_columns, force_cast_to_str_columns, is_loaddata=True) # srcテーブル結果を取得 result_src = self.db.execute_select(f"select * from {table_name_src}") # 突合から除外する項目 ignore_columns = ['dwh_upd_dt'] + # 日付型に変換して比較する項目 force_cast_to_str_columns = ['closed_dt'] # srcテーブル結果が期待値通りかを突合 assert_table_results(result_src, assert_list, ignore_columns, force_cast_to_str_columns) @@ -587,21 +589,23 @@ class TestImportFileToDb: result_org = self.db.execute_select(f"select * from {table_name_org}") # 突合から除外する項目 ignore_columns = ['dwh_upd_dt'] + # 日付型に変換して比較する項目 force_cast_to_str_columns = ['closed_dt'] # orgテーブル結果が期待値通りかを突合 - assert_table_results(result_org, assert_list, ignore_columns, force_cast_to_str_columns) + assert_table_results(result_org, assert_list, ignore_columns, force_cast_to_str_columns, is_loaddata=True) # srcテーブル結果(orgテーブル結果のPK値で一致するもの)を取得 result_src = self.db.execute_select( - f"select * from {table_name_src} s inner join {table_name_org} o on (s.v_inst_cd = o.v_inst_cd and s.sub_num = o.sub_num)") + f"select s.* from {table_name_src} s inner join {table_name_org} o on (s.v_inst_cd = o.v_inst_cd and s.sub_num = o.sub_num)") # 突合から除外する項目 ignore_columns = ['dwh_upd_dt'] + # 日付型に変換して比較する項目 force_cast_to_str_columns = ['closed_dt'] # srcテーブル結果が期待値通りかを突合 assert_table_results(result_src, assert_list, ignore_columns, force_cast_to_str_columns) # srcテーブル結果のレコード件数 (insert 4row + update 2row + insert 2row = 6row) result_src_count = self.db.execute_select(f"select count(*) from {table_name_src} ") - assert result_src_count[0]['count(*)'] == 6 + assert result_src_count[0]['count(*)'] == 8 # teardown diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/fcl_mst_202304280000.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/fcl_mst_202304280000.tsv index 9615f392..705bfe65 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/fcl_mst_202304280000.tsv +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/fcl_mst_202304280000.tsv @@ -3,3 +3,5 @@ "670229435466662922" "0" "20000101" "99991231" "" "医療法人社団仁卓会　ほりかわ歯科クリニック" "ｲﾘｮｳﾎｳｼﾞﾝｼｬﾀﾞﾝｼﾞﾝﾀｸｶｲ ﾎﾘｶﾜｼｶｸﾘﾆｯｸ" "ほりかわ歯科クリニック　（医社）" "ﾎﾘｶﾜｼｶｸﾘﾆｯｸ (ｲｼｬ)" "" "0" "675-0101 兵庫県加古川市平岡町新在家１５７３−１−４Ｆ" "ﾋｮｳｺﾞｹﾝ ｶｺｶﾞﾜｼ ﾋﾗｵｶﾁｮｳｼﾝｻﾞｲｹ1573-1-4F" "675-0101" "28" "兵庫県" "加古川市" "平岡町新在家１５７３−１−４Ｆ" "0794244617" "28210" "30" "1" "" "" "670229435785430019" "0" "20000101" "99991231" "" "株式会社コミュニティメディカル　なつめ薬局　千歳船橋店" "ｶﾌﾞｼｷｶﾞｲｼｬｺﾐｭﾆﾃｨﾒﾃﾞｨｶﾙ ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ" "なつめ薬局　千歳船橋店　（株）" "ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ (ｶ)" "" "0" "156-0054 東京都世田谷区桜丘２−２４−２" "ﾄｳｷｮｳﾄ ｾﾀｶﾞﾔｸ ｻｸﾗｶﾞｵｶ2-24-2" "156-0054" "13" "東京都" "世田谷区" "桜丘２−２４−２" "0364136189" "13112" "20" "1" "" "" "670229447437206529" "0" "20000101" "20230407" "2023-04-07" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "" "0" "670-0955 兵庫県姫路市安田４−４７−８−１Ｆ" "ﾋｮｳｺﾞｹﾝ ﾋﾒｼﾞｼ ﾔｽﾀﾞ4-47-8-1F" "670-0955" "28" "兵庫県" "姫路市" "安田４−４７−８−１Ｆ" "0792846396" "28201" "20" "1" "" "" +"670229447437206530" "0" "20000101" "20230407" "" "" "" "" "" "" "0" "" "" "" "" "" "" "" "" "" "1" "" "" +"670229447437206531" "0" "20000101" "20230407" "2023-04-07" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "" "0" "670-0955 兵庫県姫路市安田４−４７−８−１Ｆ" "ﾋｮｳｺﾞｹﾝ ﾋﾒｼﾞｼ ﾔｽﾀﾞ4-47-8-1F" "670-0955" "28" "兵庫県" "姫路市" "安田４−４７−８−１Ｆ" "0792846396" "28201" "20" "1" "" "" \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/fcl_mst_202304290000.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/fcl_mst_202304290000.tsv index b4c05930..2cc40cb0 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/fcl_mst_202304290000.tsv +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/fcl_mst_202304290000.tsv @@ -1,5 +1,5 @@ "fcl_cd" "sub_no" "start_dt" "end_dt" "closed_dt" "nm" "kn_nm" "sht_nm" "sht_kn_nm" "mkr_cd" "jsk_proc_kbn" "fmt_addr" "fmt_kn_addr" "post_cd" "prft_cd" "prft_nm" "city_nm" "addr_line_1" "tel_no" "admin_kbn" "fcl_type" "rec_sts_kbn" "ins_dt" "upd_dt" -"670229435785430019" "0" "20000202" "99991231" "" "株式会社コミュニティメディカル　なつめ薬局　千歳船橋店" "ｶﾌﾞｼｷｶﾞｲｼｬｺﾐｭﾆﾃｨﾒﾃﾞｨｶﾙ ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ" "なつめ薬局　千歳船橋店　（株）" "ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ (ｶ)" "" "0" "156-0054 東京都世田谷区桜丘２−２４−２" "ﾄｳｷｮｳﾄ ｾﾀｶﾞﾔｸ ｻｸﾗｶﾞｵｶ2-24-2" "156-0054" "13" "東京都" "世田谷区" "桜丘２−２４−２" "0364136189" "13112" "20" "1" "" "" -"670229447437206529" "0" "20000202" "20230407" "2023-04-07" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "" "0" "670-0955 兵庫県姫路市安田４−４７−８−１Ｆ" "ﾋｮｳｺﾞｹﾝ ﾋﾒｼﾞｼ ﾔｽﾀﾞ4-47-8-1F" "670-0955" "28" "兵庫県" "姫路市" "安田４−４７−８−１Ｆ" "0792846396" "28201" "20" "1" "" "" +"670229447437206530" "0" "20000202" "99991231" "" "株式会社コミュニティメディカル　なつめ薬局　千歳船橋店" "ｶﾌﾞｼｷｶﾞｲｼｬｺﾐｭﾆﾃｨﾒﾃﾞｨｶﾙ ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ" "なつめ薬局　千歳船橋店　（株）" "ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ (ｶ)" "" "0" "156-0054 東京都世田谷区桜丘２−２４−２" "ﾄｳｷｮｳﾄ ｾﾀｶﾞﾔｸ ｻｸﾗｶﾞｵｶ2-24-2" "156-0054" "13" "東京都" "世田谷区" "桜丘２−２４−２" "0364136189" "13112" "20" "1" "" "" +"670229447437206531" "0" "20000101" "20230407" "" "" "" "" "" "" "0" "" "" "" "" "" "" "" "" "" "1" "" "" "670229463350395910" "0" "20000101" "99991231" "" "こうだ歯科" "ｺｳﾀﾞｼｶ" "こうだ歯科" "ｺｳﾀﾞｼｶ" "" "0" "770-0831 徳島県徳島市寺島本町西１−１０" "ﾄｸｼﾏｹﾝ ﾄｸｼﾏｼ ﾃﾗｼﾏﾎﾝﾁｮｳﾆｼ1-10" "770-0831" "36" "徳島県" "徳島市" "寺島本町西１−１０" "0886552625" "36201" "30" "1" "" "" "670229489380246545" "0" "20020521" "99991231" "" "社会医療法人社団埼玉巨樹の会　狭山中央病院" "ｼｬｶｲｲﾘｮｳﾎｳｼﾞﾝｼｬﾀﾞﾝｻｲﾀﾏｷｮｼﾞｭﾉｶｲ ｻﾔﾏﾁｭｳｵｳﾋﾞｮｳｲﾝ" "狭山中央病院　（社社）" "ｻﾔﾏﾁｭｳｵｳﾋﾞｮｳｲﾝ (ｼｬｼｬ)" "" "0" "350-1306 埼玉県狭山市富士見２−１９−３５" "ｻｲﾀﾏｹﾝ ｻﾔﾏｼ ﾌｼﾞﾐ2-19-35" "350-1306" "11" "埼玉県" "狭山市" "富士見２−１９−３５" "0429597111" "11215" "10" "1" "" "" diff --git a/ecs/jskult-batch-daily/tests/testing_vjsk_utility.py b/ecs/jskult-batch-daily/tests/testing_vjsk_utility.py index 52969c49..c93ba52d 100644 --- a/ecs/jskult-batch-daily/tests/testing_vjsk_utility.py +++ b/ecs/jskult-batch-daily/tests/testing_vjsk_utility.py @@ -62,13 +62,14 @@ def is_valid_date_format(date_str: str, date_format): return False -def assert_table_results(actual_rows: list[dict], expect_rows: list[dict], ignore_col_names: list = None, force_cast_to_str_columns: list = [], is_loaddata=False) -> None: +def assert_table_results(actual_rows: list[dict], expect_rows: list[dict], ignore_col_names: list = None, force_cast_to_str_columns: list = None, is_loaddata=False) -> None: """テーブル同士の取得結果突き合わせ Args: actual_rows (list[dict]): テスト結果の辞書リスト expect_rows (list[dict]): 期待値の辞書リスト ignore_col_name (list): 比較を無視するDBのカラム名. Default None. + force_cast_to_str_columns (list): 強制的に文字列にキャストする項目のリスト is_loaddata (bool): LOAD DATAで読み込むテーブルかどうかのフラグ。org05の比較をするときにONにする. Default False. """ # 取得件数が一致すること @@ -88,10 +89,12 @@ def assert_table_results(actual_rows: list[dict], expect_rows: list[dict], ignor expect_value = expect_row[expect_col_name] # 期待値を、DBのデータ型(リフレクションされたpythonのデータ型)にキャストする - if actual_col_name in force_cast_to_str_columns: + if force_cast_to_str_columns is not None and actual_col_name in force_cast_to_str_columns: # DB項目(varchar)に日付型としてキャスト可能な値が期待値である場合、force_cast_to_str_columnsに基づいて強制的に文字列キャストする if type(expect_value).__name__ == 'date': expect_value = expect_value.strftime('%Y-%m-%d') + elif not is_loaddata and expect_value == "": + expect_value = None elif isinstance(actual_value, (int)): # DB項目(int)の場合、期待値もintにキャストする expect_value = int(expect_value)