diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index 263008cd..d90e2fb5 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -1,11 +1,13 @@ -from datetime import datetime -import boto3 -import io import csv +import io import sys +from datetime import datetime + +import boto3 + +from common import convert_quotechar, debug_log from end import end from error import error -from common import debug_log # 定数 DIRECTORY_WORK = '/work/' @@ -82,14 +84,18 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i work_obj = s3_resource.Object(bucket_name, work_key) work_response = work_obj.get() work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) - work_header_list = [] - for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): - work_header_list = line + work_csv_row = [] + for i, line in enumerate(csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]])): + # ヘッダあり、かつ、1行目の場合 + if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == 1 and i == 0: + work_csv_row.append(line) + continue + work_csv_row.append(line) break # ② C-0のデータ件数チェックを開始する print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-02 - C-0のチェックを開始します') - if not len(work_header_list): + if is_empty_file(work_csv_row, settings_list): print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-03 - 投入ファイルが0バイトのため処理を終了します') end(bucket_name, target_data_source, target_file_name, '', log_info, mode) print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-04 - 終了処理完了') @@ -98,16 +104,17 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i # ③ C-1の項目数チェックを開始する print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-06 - C-1のチェックを開始します') - work_header_list_len = len(work_header_list) - if work_header_list_len == int(settings_list[SETTINGS_ITEM["csvNumItems"]]): + work_csv_row_item_len = len(work_csv_row[0]) + if work_csv_row_item_len == int(settings_list[SETTINGS_ITEM["csvNumItems"]]): print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-07 - C-1:正常終了') else: - raise CheckError(f'E-CHK-01 - 項目数が一致しません 個別設定ファイル項目数:{settings_list[SETTINGS_ITEM["csvNumItems"]]} 投入データ項目数:{work_header_list_len}') + raise CheckError(f'E-CHK-01 - 項目数が一致しません 個別設定ファイル項目数:{settings_list[SETTINGS_ITEM["csvNumItems"]]} 投入データ項目数:{work_csv_row_item_len}') # ④ C-2の項目並び順チェック開始する if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-08 - C-2のチェックを開始します') settings_header_list = settings_list[SETTINGS_ITEM["csvNameItems"]].rstrip().split(',') + work_header_list = work_csv_row[0] for i in range(len(settings_header_list)): if not settings_header_list[i] == work_header_list[i]: raise CheckError(f'E-CHK-02 - 項目順序が一致しません {i + 1}番目の項目 個別設定ファイル項目:{settings_header_list[i]} 投入データ項目:{work_header_list[i]}') @@ -122,3 +129,22 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-CHK-99 - エラー内容:{e}') error(bucket_name, target_data_source, target_file_name, log_info) + + +def is_empty_file(work_csv_row: list, settings_list: list): + """② C-0のデータ件数チェック + ヘッダ行がある場合は、1行目を読み飛ばして判定する + + Args: + work_csv_row (list): CSVファイルの1行目(ヘッダを含む場合は2行目まで) + settings_list (list): 個別設定ファイルのリスト + + Returns: + bool: CSVファイルの1行目が0件だった場合はTrue + """ + has_header = int(settings_list[SETTINGS_ITEM["headerFlag"]]) == 1 + # ヘッダのみのファイルも0バイトファイルをみなす + if has_header: + return len(work_csv_row[1:]) == 0 + + return len(work_csv_row) == 0 diff --git a/ecs/dataimport/dataimport/common.py b/ecs/dataimport/dataimport/common.py index 8b8eeed6..1361ff53 100644 --- a/ecs/dataimport/dataimport/common.py +++ b/ecs/dataimport/dataimport/common.py @@ -11,3 +11,18 @@ MODE_TYPE = { def debug_log(log, log_info, mode): if MODE_TYPE['d'] == mode: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["d"]} {log}') + +def convert_quotechar(quotechar): + """csvモジュールの囲い文字を変換する + + Args: + quotechar : 項目囲い文字の設定値 + + Returns: + 空文字、空白文字の場合→None + それ以外→設定値をそのまま帰す + """ + if (quotechar.strip(' ') == ''): + return None + + return quotechar diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 94379370..23932c6f 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -6,7 +6,7 @@ from pymysql.constants import CLIENT import io import csv from error import error -from common import debug_log +from common import debug_log, convert_quotechar # 定数 DIRECTORY_WORK = '/work/' @@ -117,7 +117,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf settings_db_columu_list = settings_list[SETTINGS_ITEM["dbColumuName"]].rstrip().split(',') settings_replace_comma_list = settings_list[SETTINGS_ITEM["commaReplaceColumns"]].rstrip().split(',') - for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): + for line in csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): try: if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True and index == 0: index += 1 @@ -129,8 +129,8 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # SQL文生成 sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["loadSchemaName"]]} (' - for i in range(len(settings_db_columu_list)): - sql = f'{sql} {settings_db_columu_list[i]},' + for db_column in settings_db_columu_list: + sql = f'{sql} {db_column},' sql = f'{sql} file_name,' # システム項目:取込ファイル名 sql = f'{sql} file_row_cnt,' # システム項目:取込ファイル行番号 sql = f'{sql} delete_flg,' # システム項目:論理削除フラグ diff --git a/s3/data/SAP_fin/settings/CostReport.txt b/s3/data/SAP_fin/settings/CostReport.txt index 82e3a04b..be3dcc5e 100644 --- a/s3/data/SAP_fin/settings/CostReport.txt +++ b/s3/data/SAP_fin/settings/CostReport.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 18 diff --git a/s3/data/SAP_fin/settings/IOReport.txt b/s3/data/SAP_fin/settings/IOReport.txt index fb3dbda9..f7023c20 100644 --- a/s3/data/SAP_fin/settings/IOReport.txt +++ b/s3/data/SAP_fin/settings/IOReport.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 18 diff --git a/s3/data/SAP_fin/settings/Invoice.txt b/s3/data/SAP_fin/settings/Invoice.txt index 582e8df1..da1bb360 100644 --- a/s3/data/SAP_fin/settings/Invoice.txt +++ b/s3/data/SAP_fin/settings/Invoice.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 25 diff --git a/s3/data/SAP_fin/settings/WBSList.txt b/s3/data/SAP_fin/settings/WBSList.txt index ad66730e..1729a641 100644 --- a/s3/data/SAP_fin/settings/WBSList.txt +++ b/s3/data/SAP_fin/settings/WBSList.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 10 diff --git a/s3/data/SAP_fin/settings/WBSReport.txt b/s3/data/SAP_fin/settings/WBSReport.txt index 1d622c2e..1a899d2d 100644 --- a/s3/data/SAP_fin/settings/WBSReport.txt +++ b/s3/data/SAP_fin/settings/WBSReport.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 18 diff --git a/s3/data/SAP_sup/settings/ConfReport.txt b/s3/data/SAP_sup/settings/ConfReport.txt index 706e64d7..cd11e609 100644 --- a/s3/data/SAP_sup/settings/ConfReport.txt +++ b/s3/data/SAP_sup/settings/ConfReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 11 diff --git a/s3/data/SAP_sup/settings/GMReport.txt b/s3/data/SAP_sup/settings/GMReport.txt index 920ee02c..a9b15d1e 100644 --- a/s3/data/SAP_sup/settings/GMReport.txt +++ b/s3/data/SAP_sup/settings/GMReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 13 diff --git a/s3/data/SAP_sup/settings/GRReport.txt b/s3/data/SAP_sup/settings/GRReport.txt index 3ddaec9d..d43f139e 100644 --- a/s3/data/SAP_sup/settings/GRReport.txt +++ b/s3/data/SAP_sup/settings/GRReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 24 diff --git a/s3/data/SAP_sup/settings/MLCReport.txt b/s3/data/SAP_sup/settings/MLCReport.txt index 26953eac..bd2cbfd7 100644 --- a/s3/data/SAP_sup/settings/MLCReport.txt +++ b/s3/data/SAP_sup/settings/MLCReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 24 diff --git a/s3/data/SAP_sup/settings/POReport.txt b/s3/data/SAP_sup/settings/POReport.txt index 27dfbc37..ba46619b 100644 --- a/s3/data/SAP_sup/settings/POReport.txt +++ b/s3/data/SAP_sup/settings/POReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 12 diff --git a/s3/data/SAP_sup/settings/QAReport.txt b/s3/data/SAP_sup/settings/QAReport.txt index e9beb8ff..1cb09f0f 100644 --- a/s3/data/SAP_sup/settings/QAReport.txt +++ b/s3/data/SAP_sup/settings/QAReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 24 diff --git a/s3/data/SAP_sup/settings/StockList.txt b/s3/data/SAP_sup/settings/StockList.txt index f571083c..eb6cef3b 100644 --- a/s3/data/SAP_sup/settings/StockList.txt +++ b/s3/data/SAP_sup/settings/StockList.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 13