diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index a0bf2ced..ad1b9af7 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -23,6 +23,14 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, + 'removeCommas': 12, + 'reserved0': 13, + 'reserved1': 14, + 'reserved2': 15, + 'reserved3': 16, + 'reserved4': 17, + 'reserved5': 18, + 'reserved6': 19 } LINE_FEED_CODE = { 'CR': '\r', diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 730a7be8..94379370 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,4 +1,5 @@ from datetime import datetime +import re import boto3 import pymysql from pymysql.constants import CLIENT @@ -23,6 +24,14 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, + 'commaReplaceColumns': 12, + 'reserved0': 13, + 'reserved1': 14, + 'reserved2': 15, + 'reserved3': 16, + 'reserved4': 17, + 'reserved5': 18, + 'reserved6': 19 } LINE_FEED_CODE = { 'CR': '\r', @@ -76,12 +85,18 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-03 - タイムゾーンを変更しました') # ④ 個別設定ファイルのロードスキーマのテーブル名に記載されているテーブルをTRUNCATEする + # 個別設定ファイルの読み込み settings_obj = s3_resource.Object(bucket_name, settings_key) settings_response = settings_obj.get() settings_list = [] for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'): settings_list.append(line.rstrip('\n')) + # 設定ファイルに記載のない行を空文字として扱い、予約行とする + for _ in range(len(SETTINGS_ITEM) - len(settings_list)): + settings_list.append('') + + # ロードスキーマのTRUNCATE with conn.cursor() as cur: sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["loadSchemaName"]]}' cur.execute(sql_truncate) @@ -100,6 +115,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf warning_info = '' # ワーニング情報 index = 0 # ループインデックス settings_db_columu_list = settings_list[SETTINGS_ITEM["dbColumuName"]].rstrip().split(',') + settings_replace_comma_list = settings_list[SETTINGS_ITEM["commaReplaceColumns"]].rstrip().split(',') for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): try: @@ -125,13 +141,17 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf sql = f'{sql} VALUES (' for i in range(len(line)): # データ項目値が0桁より大きいかチェックする - if len(line[i]) > 0: - # 0桁より大きい場合 - replace_line = line[i].replace('\\', '\\\\') - sql = f'{sql} "{replace_line}",' - else: - # 上記以外の場合 + if len(line[i]) == 0: + # 0桁の場合 sql = f'{sql} NULL,' + continue + + # データ項目値の変換処理 + org_column_value = line[i] + current_settings_db_column_name = settings_db_columu_list[i] + column_value = convert_column_value(org_column_value, current_settings_db_column_name, settings_replace_comma_list) + sql = f'{sql} "{column_value}",' + sql = f'{sql} "{target_file_name}",' # システム項目：取込ファイル名 sql = f'{sql} "{index + 1}",' # システム項目：取込ファイル行番号 sql = f'{sql} "0",' # システム項目：論理削除フラグ @@ -272,3 +292,24 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) + +def convert_column_value(org_column_value, current_settings_db_column_name, settings_replace_comma_list): + """データ項目値変換処理 + + Args: + org_column_value : 投入データの値 + current_settings_db_column_name : 投入データのDBカラム物理名 + settings_replace_comma_list : 投入データの数値型のDBカラム物理名のリスト + + Returns: + converted_column_value:変換処理を行った投入データの値 + """ + # データ内の「\\」という文字がpythonで読んだ場合に「\\\\」となるため、「\\」に戻す + converted_column_value = org_column_value.replace('\\', '\\\\') + + # 投入データのDB物理カラム名が設定ファイルの数値型のDBカラム物理名に含まれている場合、データ項目値の「,」を取り除く + if current_settings_db_column_name in settings_replace_comma_list: + converted_column_value = converted_column_value.replace(',', '') + + return converted_column_value + \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/CostReport.txt b/s3/data/SAP_fin/settings/CostReport.txt index 98eb615b..a3b33f26 100644 --- a/s3/data/SAP_fin/settings/CostReport.txt +++ b/s3/data/SAP_fin/settings/CostReport.txt @@ -10,3 +10,4 @@ vt,fiscal_year,fiscal_period,cost_center,cost_element,cost_element_name,vbl_valu src03a.sapf_costreport org03a.sapf_costreport +vbl_value \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/IOReport.txt b/s3/data/SAP_fin/settings/IOReport.txt index 0157c6e9..a2ab0774 100644 --- a/s3/data/SAP_fin/settings/IOReport.txt +++ b/s3/data/SAP_fin/settings/IOReport.txt @@ -10,3 +10,4 @@ fiscal_year,fiscal_period,internal_order,cost_element,cost_element_name,vbl_valu src03a.sapf_ioreport org03a.sapf_ioreport +vbl_value \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/Invoice.txt b/s3/data/SAP_fin/settings/Invoice.txt index 61488a51..6fcac702 100644 --- a/s3/data/SAP_fin/settings/Invoice.txt +++ b/s3/data/SAP_fin/settings/Invoice.txt @@ -10,3 +10,4 @@ billing_type,condition_type,distribution_channel,invoiced_number,invoice_item,in src03b.sapf_invoice org03b.sapf_invoice +quantity_invoiced,extended_amount_invoiced_amount,accrual_value_rebate_1,accrual_percent_rebate_1,accrual_value_rebate_2,accrual_percent_rebate_2,unit_selling_price \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/WBSList.txt b/s3/data/SAP_fin/settings/WBSList.txt index a64e1390..9247ca7f 100644 --- a/s3/data/SAP_fin/settings/WBSList.txt +++ b/s3/data/SAP_fin/settings/WBSList.txt @@ -8,5 +8,4 @@ LF WBS element,Level,Profit center,Project Definition,Description,Control area,Responsibility,Basic finish date,Basic start date wbs_element,level,profit_center,project_definition,description,control_area,responsibility,basic_finish_date,basic_start_date src03a.sapf_wbslist -org03a.sapf_wbslist - +org03a.sapf_wbslist \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/WBSReport.txt b/s3/data/SAP_fin/settings/WBSReport.txt index 91c75389..2d29d606 100644 --- a/s3/data/SAP_fin/settings/WBSReport.txt +++ b/s3/data/SAP_fin/settings/WBSReport.txt @@ -10,3 +10,4 @@ fiscal_year,fiscal_period,wbs_element,cost_element,cost_element_name,vbl_value,o src03a.sapf_wbsreport org03a.sapf_wbsreport +vbl_value \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/ConfReport.txt b/s3/data/SAP_sup/settings/ConfReport.txt index 299632cf..7c7ecefe 100644 --- a/s3/data/SAP_sup/settings/ConfReport.txt +++ b/s3/data/SAP_sup/settings/ConfReport.txt @@ -10,3 +10,4 @@ process_order,operation_activity,yield,uom,posting_date,finish_execution_date,ma src04.saps_confreport org04.saps_confreport +yield \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/GMReport.txt b/s3/data/SAP_sup/settings/GMReport.txt index c20b6aa9..c254e937 100644 --- a/s3/data/SAP_sup/settings/GMReport.txt +++ b/s3/data/SAP_sup/settings/GMReport.txt @@ -10,3 +10,4 @@ process_order,material_document,material_document_item,material,material_descrip src04.saps_gmreport org04.saps_gmreport +qty \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/GRReport.txt b/s3/data/SAP_sup/settings/GRReport.txt index 3ed56da7..aecae348 100644 --- a/s3/data/SAP_sup/settings/GRReport.txt +++ b/s3/data/SAP_sup/settings/GRReport.txt @@ -10,3 +10,4 @@ plant,material,material_description,batch,posting_date,qty_in_un_of_entry,moveme src04.saps_grreport org04.saps_grreport +qty_in_un_of_entry,total_valuated_stock_before_the_posting,quantity,amount \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/MLCReport.txt b/s3/data/SAP_sup/settings/MLCReport.txt index ed71a72f..db3af777 100644 --- a/s3/data/SAP_sup/settings/MLCReport.txt +++ b/s3/data/SAP_sup/settings/MLCReport.txt @@ -10,3 +10,4 @@ plant,material,material_description,batch,posting_date,qty_in_un_of_entry,moveme src04.saps_mlcreport org04.saps_mlcreport +qty_in_un_of_entry,total_valuated_stock_before_the_posting,quantity,amount \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/POReport.txt b/s3/data/SAP_sup/settings/POReport.txt index eb6de20d..5fe24ce6 100644 --- a/s3/data/SAP_sup/settings/POReport.txt +++ b/s3/data/SAP_sup/settings/POReport.txt @@ -10,3 +10,4 @@ purchasing_document,vendor,item,material,short_text,document_date,order_quantity src04.saps_poreport org04.saps_poreport +order_quantity,net_order_value \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/QAReport.txt b/s3/data/SAP_sup/settings/QAReport.txt index 96b6f6aa..fca0ca72 100644 --- a/s3/data/SAP_sup/settings/QAReport.txt +++ b/s3/data/SAP_sup/settings/QAReport.txt @@ -10,3 +10,4 @@ plant,material,material_description,batch,posting_date,qty_in_un_of_entry,moveme src04.saps_qareport org04.saps_qareport +qty_in_un_of_entry,total_valuated_stock_before_the_posting,quantity,amount \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/StockList.txt b/s3/data/SAP_sup/settings/StockList.txt index 8bc8985d..f571083c 100644 --- a/s3/data/SAP_sup/settings/StockList.txt +++ b/s3/data/SAP_sup/settings/StockList.txt @@ -10,3 +10,4 @@ spl_stock_indic,material_num,material_desc,storage_location,batch_num,expired_da src04.saps_stocklist org04.saps_stocklist StockList_ex.sql +unrestricted_stock,in_quality_stock,blocked_stock,consign_stock,total_stock_quantity \ No newline at end of file