From 0c5eab44c233afce5454070d6044c57dc078d723 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 30 May 2025 08:58:34 +0900 Subject: [PATCH] =?UTF-8?q?fix:=20=E3=83=91=E3=83=95=E3=82=A9=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=B3=E3=82=B9=E3=83=86=E3=82=B9=E3=83=88=E5=AE=9F?= =?UTF-8?q?=E6=96=BD=E4=B8=AD=E3=81=AB=E7=99=BA=E8=A6=9A=E3=81=97=E3=81=9F?= =?UTF-8?q?=E3=83=90=E3=82=B0=E3=82=92=E4=BF=AE=E6=AD=A3=E3=80=82=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E5=86=85=E3=81=AE=E7=A9=BA=E6=96=87?= =?UTF-8?q?=E5=AD=97=E3=82=92NULL=E3=81=AB=E5=A4=89=E6=8F=9B=E3=80=82?= =?UTF-8?q?=E3=83=AD=E3=82=B0=E3=83=A1=E3=83=83=E3=82=BB=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E3=81=AE=E4=B8=8D=E5=82=99=E3=82=92=E4=BF=AE=E6=AD=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/chk.py | 2 +- ecs/dataimport/dataimport/main.py | 22 ++++++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index 87cdc9f9..91f59d04 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -187,7 +187,7 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i raise CheckError( f'E-CHK-03 - 投入データ末尾の項目数が一致しません 個別設定ファイル項目数：{settings_list[SETTINGS_ITEM["csvNumItems"]]} 投入データ項目数：{last_line_count}') print( - f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {INFO} I-CHK-09 - C-3 末尾行項目数チェック 正常終了') + f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {INFO} I-CHK-14 - C-3 末尾行項目数チェック 正常終了') # ⑦ チェック処理終了ログを出力する print( diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index adea2f8e..aa717847 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -480,9 +480,14 @@ def import_data_with_bulk( load_schema_name = settings_list[SETTINGS_ITEM["loadSchemaName"]] has_header = settings_list[SETTINGS_ITEM["headerFlag"]] or int( settings_list[SETTINGS_ITEM["headerFlag"]]) == 1 - settings_db_columu_list = settings_list[SETTINGS_ITEM["dbColumuName"]].rstrip( - ).split(',') enclosed_by = convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]) + settings_db_columu_list = settings_list[SETTINGS_ITEM["dbColumuName"]]\ + .rstrip().split(',') + + # データファイル内の空文字をNULLに変換するため、各カラムをパラメータ化 + variables = [f"@{col_name}" for col_name in settings_db_columu_list] + sets = [f"`{col}` = NULLIF(@{col}, '')" + for col in settings_db_columu_list] sql = f""" SET @file_row_cnt = 1; @@ -493,13 +498,14 @@ def import_data_with_bulk( INTO TABLE {load_schema_name} CHARACTER SET {MYSQL_CHARSET_CODE[settings_list[SETTINGS_ITEM["charCode"]]]} - FIELDS TERMINATED BY '{settings_list[SETTINGS_ITEM["delimiter"]]}' - ENCLOSED BY '{enclosed_by if enclosed_by != "'" else "\\'"}' - LINES TERMINATED BY '{LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]}' + FIELDS TERMINATED BY {repr(settings_list[SETTINGS_ITEM["delimiter"]])} + ENCLOSED BY {repr(enclosed_by)} + LINES TERMINATED BY {repr(LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]])} {'IGNORE 1 LINES' if has_header else ''} - ({','.join([column for column in settings_db_columu_list])}) + ({','.join(variables)}) SET - -- 取込ファイル名 + {','.join(sets)}, + -- 取込ファイル名 file_name = %s, -- 取込ファイル行番号 file_row_cnt = (@file_row_cnt := @file_row_cnt + 1), @@ -528,7 +534,7 @@ def import_data_with_bulk( conn.commit() except Exception as e: print( - f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {ERROR} 一括登録モードのSQL実行に失敗しました。エラー内容： {e}') + f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {ERROR} E-MAIN-02 - 一括登録モードのSQL実行に失敗しました。エラー内容： {e}') error(bucket_name, target_data_source, target_file_name, log_info) # 一括登録の場合、クエリ実行に成功したら、処理件数と成功件数は同じにする