Merge pull request #39 feature-NEWDWH2021-582 into develop-8sap

ecs/dataimport/dataimport/chk.py

@@ -23,6 +23,14 @@ SETTINGS_ITEM = {
     'storageSchemaName': 9,
     'loadSchemaName': 10,
     'exSqlFileName': 11,
+    'removeCommas': 12,
+    'reserved0': 13,
+    'reserved1': 14,
+    'reserved2': 15,
+    'reserved3': 16,
+    'reserved4': 17,
+    'reserved5': 18,
+    'reserved6': 19
 }
 LINE_FEED_CODE = {
     'CR': '\r',

ecs/dataimport/dataimport/main.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+import re
 import boto3
 import pymysql
 from pymysql.constants import CLIENT
@@ -23,6 +24,14 @@ SETTINGS_ITEM = {
     'storageSchemaName': 9,
     'loadSchemaName': 10,
     'exSqlFileName': 11,
+    'commaReplaceColumns': 12,
+    'reserved0': 13,
+    'reserved1': 14,
+    'reserved2': 15,
+    'reserved3': 16,
+    'reserved4': 17,
+    'reserved5': 18,
+    'reserved6': 19
 }
 LINE_FEED_CODE = {
     'CR': '\r',
@@ -76,12 +85,18 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf
         print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-03 - タイムゾーンを変更しました')
 
         # ④ 個別設定ファイルのロードスキーマのテーブル名に記載されているテーブルをTRUNCATEする
+        # 個別設定ファイルの読み込み
         settings_obj = s3_resource.Object(bucket_name, settings_key)
         settings_response = settings_obj.get()
         settings_list = []
         for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'):
             settings_list.append(line.rstrip('\n'))
 
+        # 設定ファイルに記載のない行を空文字として扱い、予約行とする
+        for _ in range(len(SETTINGS_ITEM) - len(settings_list)):
+            settings_list.append('')
+    
+        # ロードスキーマのTRUNCATE
         with conn.cursor() as cur:
             sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["loadSchemaName"]]}'
             cur.execute(sql_truncate)
@@ -100,6 +115,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf
         warning_info = ''   # ワーニング情報
         index = 0           # ループインデックス
         settings_db_columu_list = settings_list[SETTINGS_ITEM["dbColumuName"]].rstrip().split(',')
+        settings_replace_comma_list = settings_list[SETTINGS_ITEM["commaReplaceColumns"]].rstrip().split(',')
 
         for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]):
             try:
@@ -125,13 +141,17 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf
                 sql = f'{sql} VALUES ('
                 for i in range(len(line)):
                     # データ項目値が0桁より大きいかチェックする
-                    if len(line[i]) > 0:
+                    if len(line[i]) == 0:
-                        # 0桁より大きい場合
+                        # 0桁の場合
-                        replace_line = line[i].replace('\\', '\\\\')
-                        sql = f'{sql} "{replace_line}",'
-                    else:
-                        # 上記以外の場合
                         sql = f'{sql} NULL,'
+                        continue
+                    
+                    # データ項目値の変換処理
+                    org_column_value = line[i]
+                    current_settings_db_column_name = settings_db_columu_list[i]
+                    column_value = convert_column_value(org_column_value, current_settings_db_column_name, settings_replace_comma_list)
+                    sql = f'{sql} "{column_value}",'
+
                 sql = f'{sql} "{target_file_name}",'    # システム項目：取込ファイル名
                 sql = f'{sql} "{index + 1}",'           # システム項目：取込ファイル行番号
                 sql = f'{sql} "0",'                     # システム項目：論理削除フラグ
@@ -272,3 +292,24 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo
     except Exception as e:
         print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}')
         error(bucket_name, target_data_source, target_file_name, log_info)
+
+def convert_column_value(org_column_value, current_settings_db_column_name, settings_replace_comma_list):
+    """データ項目値変換処理
+
+    Args:
+        org_column_value : 投入データの値
+        current_settings_db_column_name : 投入データのDBカラム物理名
+        settings_replace_comma_list : 投入データの数値型のDBカラム物理名のリスト
+
+    Returns:
+        converted_column_value:変換処理を行った投入データの値
+    """
+    # データ内の「\\」という文字がpythonで読んだ場合に「\\\\」となるため、「\\」に戻す
+    converted_column_value = org_column_value.replace('\\', '\\\\')
+    
+    # 投入データのDB物理カラム名が設定ファイルの数値型のDBカラム物理名に含まれている場合、データ項目値の「,」を取り除く    
+    if current_settings_db_column_name in settings_replace_comma_list: 
+        converted_column_value =  converted_column_value.replace(',', '')
+
+    return converted_column_value
+       

s3/data/SAP_fin/settings/CostReport.txt

@@ -10,3 +10,4 @@ vt,fiscal_year,fiscal_period,cost_center,cost_element,cost_element_name,vbl_valu
 src03a.sapf_costreport
 org03a.sapf_costreport
 
+vbl_value

s3/data/SAP_fin/settings/IOReport.txt

@@ -10,3 +10,4 @@ fiscal_year,fiscal_period,internal_order,cost_element,cost_element_name,vbl_valu
 src03a.sapf_ioreport
 org03a.sapf_ioreport
 
+vbl_value

s3/data/SAP_fin/settings/Invoice.txt

@@ -10,3 +10,4 @@ billing_type,condition_type,distribution_channel,invoiced_number,invoice_item,in
 src03b.sapf_invoice
 org03b.sapf_invoice
 
+quantity_invoiced,extended_amount_invoiced_amount,accrual_value_rebate_1,accrual_percent_rebate_1,accrual_value_rebate_2,accrual_percent_rebate_2,unit_selling_price

s3/data/SAP_fin/settings/WBSList.txt

@@ -9,4 +9,3 @@ WBS element,Level,Profit center,Project Definition,Description,Control area,Resp
 wbs_element,level,profit_center,project_definition,description,control_area,responsibility,basic_finish_date,basic_start_date
 src03a.sapf_wbslist
 org03a.sapf_wbslist
-

s3/data/SAP_fin/settings/WBSReport.txt

@@ -10,3 +10,4 @@ fiscal_year,fiscal_period,wbs_element,cost_element,cost_element_name,vbl_value,o
 src03a.sapf_wbsreport
 org03a.sapf_wbsreport
 
+vbl_value

s3/data/SAP_sup/settings/ConfReport.txt

@@ -10,3 +10,4 @@ process_order,operation_activity,yield,uom,posting_date,finish_execution_date,ma
 src04.saps_confreport
 org04.saps_confreport
 
+yield

s3/data/SAP_sup/settings/GMReport.txt

@@ -10,3 +10,4 @@ process_order,material_document,material_document_item,material,material_descrip
 src04.saps_gmreport
 org04.saps_gmreport
 
+qty

s3/data/SAP_sup/settings/GRReport.txt

@@ -10,3 +10,4 @@ plant,material,material_description,batch,posting_date,qty_in_un_of_entry,moveme
 src04.saps_grreport
 org04.saps_grreport
 
+qty_in_un_of_entry,total_valuated_stock_before_the_posting,quantity,amount

s3/data/SAP_sup/settings/MLCReport.txt

@@ -10,3 +10,4 @@ plant,material,material_description,batch,posting_date,qty_in_un_of_entry,moveme
 src04.saps_mlcreport
 org04.saps_mlcreport
 
+qty_in_un_of_entry,total_valuated_stock_before_the_posting,quantity,amount

s3/data/SAP_sup/settings/POReport.txt

@@ -10,3 +10,4 @@ purchasing_document,vendor,item,material,short_text,document_date,order_quantity
 src04.saps_poreport
 org04.saps_poreport
 
+order_quantity,net_order_value

s3/data/SAP_sup/settings/QAReport.txt

@@ -10,3 +10,4 @@ plant,material,material_description,batch,posting_date,qty_in_un_of_entry,moveme
 src04.saps_qareport
 org04.saps_qareport
 
+qty_in_un_of_entry,total_valuated_stock_before_the_posting,quantity,amount

s3/data/SAP_sup/settings/StockList.txt

@@ -10,3 +10,4 @@ spl_stock_indic,material_num,material_desc,storage_location,batch_num,expired_da
 src04.saps_stocklist
 org04.saps_stocklist
 StockList_ex.sql
+unrestricted_stock,in_quality_stock,blocked_stock,consign_stock,total_stock_quantity