diff --git a/.gitignore b/.gitignore index a85e0c0b..52cbf26c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,9 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* -__pycache__/ -.env \ No newline at end of file + +# ローカルの環境変数 +.env +# Python関連 +.venv +__pycache__ diff --git a/lambda/sap-data-decrypt/datadecrypt/main.py b/lambda/sap-data-decrypt/datadecrypt/main.py index ec4d24ae..49eb8db0 100644 --- a/lambda/sap-data-decrypt/datadecrypt/main.py +++ b/lambda/sap-data-decrypt/datadecrypt/main.py @@ -6,7 +6,6 @@ import datetime import logging from abc import * from zoneinfo import ZoneInfo -import traceback # 環境変数 SECRET_KEY_FILE_BUCKET_NAME = os.environ["SECRET_KEY_FILE_BUCKET_NAME"] @@ -23,6 +22,7 @@ LOG_LEVEL = os.environ["LOG_LEVEL"] # 定数 DIRECTORY_RECV = '/recv/' DIRECTORY_RECV_ERROR = '/recv_error/' +DIRECTORY_TARGET = '/target/' EXTENSION_ERROR = '.error' EXTENSION_DECRYPT_ERROR = '.decrypt_error' INDEX_EXTENSION_DELETE_NUM = -4 @@ -78,7 +78,7 @@ def handler(event, context): s3_client.download_file(s3_event.bucket_name, s3_event.file_path, PATH_TEMP_ENCRYPT_FILE) logger.info('I-03-02 PGP暗号化ファイルを読み込みました') except Exception as e: - logger.error(f'E-03-01 PGP暗号化ファイルの読み込みに失敗しました　エラー内容：{e}') + logger.exception(f'E-03-01 PGP暗号化ファイルの読み込みに失敗しました　エラー内容：{e}') raise EncryptFileReadException('E-03-01', EXTENSION_ERROR, e) # ④ S3から秘密鍵ファイルを読み込む @@ -87,7 +87,7 @@ def handler(event, context): s3_client.download_file(SECRET_KEY_FILE_BUCKET_NAME, SECRET_KEY_FILE_PATH, PATH_TEMP_PRIVATE_KEY) logger.info('I-04-02 秘密鍵ファイルを読み込みました') except Exception as e: - logger.error(f'E-04-01 秘密鍵ファイルの読み込みに失敗しました　エラー内容：{e}') + logger.exception(f'E-04-01 秘密鍵ファイルの読み込みに失敗しました　エラー内容：{e}') raise FileReadException('E-04-01', EXTENSION_ERROR, e) # ⑤ 「③」で読み込んだ秘密鍵ファイルをPGPライブラリにインポートを行う @@ -98,7 +98,7 @@ def handler(event, context): gpg.import_keys(key_file.read()) logger.info('I-05-02 秘密鍵ファイルをインポートしました') except Exception as e: - logger.error(f'E-05-01 秘密鍵ファイルのインポートに失敗しました　エラー内容：{e}') + logger.exception(f'E-05-01 秘密鍵ファイルのインポートに失敗しました　エラー内容：{e}') raise PrivateKeyImportException('E-05-01', EXTENSION_ERROR, e) # ⑥ 「④」で読み込んだPGP暗号化ファイルを「⑤」でインポートした秘密鍵ファイルで復号する @@ -109,7 +109,7 @@ def handler(event, context): decrypt_file = open(PATH_TEMP_DECRYPT_FILE, 'rb') logger.info('I-06-02 PGP暗号化ファイルを復号しました') except Exception as e: - logger.error(f'E-06-01 PGP暗号化ファイルの復号化に失敗しました　エラー内容：{e}') + logger.exception(f'E-06-01 PGP暗号化ファイルの復号化に失敗しました　エラー内容：{e}') raise DecryptException('E-06-01', EXTENSION_DECRYPT_ERROR, e) # ⑦ 各ファイルをS3に出力する @@ -126,7 +126,7 @@ def handler(event, context): decrypt_file.close logger.info('I-07-03 復号化ファイルをS3に出力しました') except Exception as e: - logger.error(f'E-07-01 復号化ファイルのS3出力に失敗しました　エラー内容：{e}') + logger.exception(f'E-07-01 復号化ファイルのS3出力に失敗しました　エラー内容：{e}') raise FileOutputException('E-07-01', EXTENSION_ERROR, e) # 「④」で読み込んだPGP暗号化ファイルを以下に移動（コピー削除）する @@ -142,23 +142,34 @@ def handler(event, context): s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path) logger.info('I-07-05 PGP暗号化ファイルをバックアップ用バケットに移動しました') except Exception as e: - logger.error(f'E-07-02 PGP暗号化ファイルのS3出力に失敗しました　エラー内容：{e}') + logger.exception(f'E-07-02 PGP暗号化ファイルのS3出力に失敗しました　エラー内容：{e}') + raise FileOutputException('E-07-02', EXTENSION_ERROR, e) + # 「⑥」で復号化したファイルをデータ取込用バケットに出力する + try: + copy_source = { + 'Bucket': decrypt_bucket_name, + 'Key': decrypt_file_key + } + import_file_folder = f'{s3_event.data_source_name}{DIRECTORY_TARGET}' + import_file_key = f'{import_file_folder}{decrypt_file_name}' + logger.info(f'I-07-06 復号化ファイル出力　ファイル名：{decrypt_file_name}　出力先：{import_file_folder}') + import_file_obj = s3_resource.Object(s3_event.bucket_name, import_file_key) + import_file_obj.copy(copy_source) + logger.info(f'I-07-07 復号化ファイルをS3に出力しました') + except Exception as e: + logger.exception(f'E-07-03 復号化ファイルのS3出力に失敗しました　エラー内容：{e}') raise FileOutputException('E-07-02', EXTENSION_ERROR, e) - # ⑧ 処理終了ログを出力する logger.info('I-08-01 処理終了　SAPデータ復号処理') except EncryptFileReadException as e: - traceback.print_exc() create_status_file(s3_event, e.extension) error_notice(e.id, e.arg) except CustomException as e: - traceback.print_exc() create_status_file(s3_event, e.extension) move_encrypt_file(s3_event) error_notice(e.id, e.arg) except Exception as e: - logger.error(f'E-99 想定外のエラーが発生しました　エラー内容：{e}') - traceback.print_exc() + logger.exception(f'E-99 想定外のエラーが発生しました　エラー内容：{e}') create_status_file(s3_event, EXTENSION_ERROR) move_encrypt_file(s3_event) error_notice('E-99', e) @@ -174,8 +185,7 @@ def create_status_file(s3_event, extension) -> None: result_error_obj.put(Body='') logger.error(f'E-ERR-01 recvディレクトリにエラーファイルを作成しました　ファイル名：{result_error_file_name}　出力先：{s3_event.bucket_name}/{result_error_key}') except Exception as e: - logger.error(f'E-96 エラーステータスファイルの作成に失敗しました　エラー内容：{e}') - traceback.print_exc() + logger.exception(f'E-96 エラーステータスファイルの作成に失敗しました　エラー内容：{e}') return @@ -193,8 +203,7 @@ def move_encrypt_file(s3_event) -> None: s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path) logger.error(f'E-ERR-02 recv_errorディレクトリにファイルを移動しました　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{s3_event.bucket_name}/{error_key}') except Exception as e: - logger.error(f'E-97 PGP暗号化ファイルの移動に失敗しました　エラー内容：{e}') - traceback.print_exc() + logger.exception(f'E-97 PGP暗号化ファイルの移動に失敗しました　エラー内容：{e}') return @@ -210,8 +219,7 @@ def error_notice(error_log_id, exception) -> None: sns_client.publish(**params) logger.error(f'E-ERR-03 処理異常通知の送信指示をしました　通知先トピック：{NDS_NOTICE_TOPIC}') except Exception as e: - logger.error(f'E-98 処理異常通知の送信指示に失敗しました　エラー内容：{e}') - traceback.print_exc() + logger.exception(f'E-98 処理異常通知の送信指示に失敗しました　エラー内容：{e}') return diff --git a/s3/data/SAP_fin/settings/CostReport.txt b/s3/data/SAP_fin/settings/CostReport.txt new file mode 100644 index 00000000..28877767 --- /dev/null +++ b/s3/data/SAP_fin/settings/CostReport.txt @@ -0,0 +1,11 @@ +SAP_fin + +utf-8 + +LF +1 +18 +VT,Fiscal Year,Fiscal Period,Cost Center,Cost element,Cost element name,Vbl.value,Offsetting account,Offsetting account name,Name,Document header text,Posting date,Document date,Purchasing document,Document number,Posting row,Ref doc number,Rev +vt,fiscal_year,fiscal_period,cost_center,cost_element,cost_element_name,vbl_value,offsetting_account,offsetting_account_name,name,document_header_text,posting_date,document_date,purchasing_document,document_number,posting_row,ref_doc_number,rev +src03a.sapf_costreport +org03a.sapf_costreport diff --git a/s3/data/SAP_fin/settings/IOReport.txt b/s3/data/SAP_fin/settings/IOReport.txt new file mode 100644 index 00000000..8c09e651 --- /dev/null +++ b/s3/data/SAP_fin/settings/IOReport.txt @@ -0,0 +1,11 @@ +SAP_fin + +utf-8 + +LF +1 +17 +Fiscal Year,Fiscal Period,Internal Order,Cost element,Cost element name,Vbl.value,Offsetting account,Offsetting account name,AuxAccAs1,Name,Document header text,Posting date,Document date,Purchasing document,Document number,Posting row,Ref doc number +fiscal_year,fiscal_period,internal_order,cost_element,cost_element_name,vbl_value,offsetting_account,offsetting_account_name,auxaccas1,name,document_header_text,posting_date,document_date,purchasing_document,document_number,posting_row,ref_doc_number +src03a.sapf_ioreport +org03a.sapf_ioreport diff --git a/s3/data/SAP_fin/settings/Invoice.txt b/s3/data/SAP_fin/settings/Invoice.txt new file mode 100644 index 00000000..13430d0d --- /dev/null +++ b/s3/data/SAP_fin/settings/Invoice.txt @@ -0,0 +1,11 @@ +SAP_fin + +utf-8 + +LF +1 +25 +Billing Type,Condition Type,Distribution channel,Invoiced number,Invoice item,Invoice date,Account number (Sold-to),Customer name,Ship-to,Ship-to(Name),Quantity invoiced,Extended amount (invoiced amount),Accrual value (rebate 1),Accrual % (rebate 1),Accrual value (rebate 2),Accrual % (rebate 2),Unit selling price,Item code,Item name,Tax,Order reason,Reference doc,Sales order No.,Rejected RSN,Batch number +billing_type,condition_type,distribution_channel,invoiced_number,invoice_item,invoice_date,account_number_sold_to,customer_name,ship_to,ship_to_name,quantity_invoiced,extended_amount_invoiced_amount,accrual_value_rebate_1,accrual_percent_rebate_1,accrual_value_rebate_2,accrual_percent_rebate_2,unit_selling_price,item_code,item_name,tax,order_reason,reference_doc,sales_order_no,rejected_rsn,batch_number +src03b.sapf_invoice +org03b.sapf_invoice diff --git a/s3/data/SAP_fin/settings/WBSList.txt b/s3/data/SAP_fin/settings/WBSList.txt new file mode 100644 index 00000000..9247ca7f --- /dev/null +++ b/s3/data/SAP_fin/settings/WBSList.txt @@ -0,0 +1,11 @@ +SAP_fin + +utf-8 + +LF +1 +9 +WBS element,Level,Profit center,Project Definition,Description,Control area,Responsibility,Basic finish date,Basic start date +wbs_element,level,profit_center,project_definition,description,control_area,responsibility,basic_finish_date,basic_start_date +src03a.sapf_wbslist +org03a.sapf_wbslist \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/WBSReport.txt b/s3/data/SAP_fin/settings/WBSReport.txt new file mode 100644 index 00000000..000959c5 --- /dev/null +++ b/s3/data/SAP_fin/settings/WBSReport.txt @@ -0,0 +1,11 @@ +SAP_fin + +utf-8 + +LF +1 +17 +Fiscal Year,Fiscal Period,WBS element,Cost element,Cost element name,Vbl.value,Offsetting account,Offsetting account name,AuxAccAs1,Name,Document header text,Posting date,Document date,Purchasing document,Document number,Posting row,Ref doc number +fiscal_year,fiscal_period,wbs_element,cost_element,cost_element_name,vbl_value,offsetting_account,offsetting_account_name,auxaccas1,name,document_header_text,posting_date,document_date,purchasing_document,document_number,posting_row,ref_doc_number +src03a.sapf_wbsreport +org03a.sapf_wbsreport diff --git a/s3/data/SAP_fin/settings/configmap.config b/s3/data/SAP_fin/settings/configmap.config new file mode 100644 index 00000000..5d466ebb --- /dev/null +++ b/s3/data/SAP_fin/settings/configmap.config @@ -0,0 +1,7 @@ +/* SAPデータ取込:Finance */ +CostReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) CostReport.txt +IOReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) IOReport.txt +WBSReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) WBSReport.txt +WBSList_[0-9]{8}_[0-9]{6}\.(TSV|tsv) WBSList.txt +Invoice_[0-9]{8}_[0-9]{6}\.(TSV|tsv) Invoice.txt + diff --git a/s3/data/SAP_sup/settings/ConfReport.txt b/s3/data/SAP_sup/settings/ConfReport.txt new file mode 100644 index 00000000..af59684d --- /dev/null +++ b/s3/data/SAP_sup/settings/ConfReport.txt @@ -0,0 +1,11 @@ +SAP_sup + +utf-8 + +LF +1 +11 +Process Order,Operation / activity,Yield,UoM,Posting date,Finish execution date,Material,Material description,Confirmation,cofirm.counter,cancelled. conf +process_order,operation_activity,yield,uom,posting_date,finish_execution_date,material,material_description,confirmation,cofirm_counter,cancelled_conf +src04.saps_gmreport +org04.saps_gmreport diff --git a/s3/data/SAP_sup/settings/GMReport.txt b/s3/data/SAP_sup/settings/GMReport.txt new file mode 100644 index 00000000..fe4fec3d --- /dev/null +++ b/s3/data/SAP_sup/settings/GMReport.txt @@ -0,0 +1,11 @@ +SAP_sup + +utf-8 + +LF +1 +12 +Process Order,Material document,Material document item,Material,Material description,Goods movement,Posting date,Movement type,D/C indicator,Storage location,Batch,Qty +process_order,material_document,material_document_item,material,material_description,goods_movement,posting_date,movement_type,d_c_indicator,storage_location,batch,qty +src04.saps_gmreport +org04.saps_gmreport \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/GRReport.txt b/s3/data/SAP_sup/settings/GRReport.txt new file mode 100644 index 00000000..5b9de358 --- /dev/null +++ b/s3/data/SAP_sup/settings/GRReport.txt @@ -0,0 +1,11 @@ +SAP_sup + +utf-8 + +LF +1 +24 +Plant,Material,Material Description,Batch,Posting Date,Qty in Un. of Entry,Movement Type,Unit of Entry,Entry Date,Material Doc. Year,Document Date,Total valuated stock before the posting,Base Unit of Measure,Quantity,Reference,Purchase Order,Customer,Amount,Amount in LC,Vendor,Item,Material Document,Storage Location,Movement Type Text +plant,material,material_description4,batch,posting_date,qty_in_un_of_entry,movement_type,unit_of_entry,entry_date,material_doc_year,document_date,total_valuated_stock_before_the_posting,base_unit_of_measure,quantity,reference,purchase_order,customer,amount,amount_in_lc,vendor,item,material_document,storage_location,movement_type_text +src04.saps_grreport +org04.saps_grreport diff --git a/s3/data/SAP_sup/settings/MLCReport.txt b/s3/data/SAP_sup/settings/MLCReport.txt new file mode 100644 index 00000000..0f67ad56 --- /dev/null +++ b/s3/data/SAP_sup/settings/MLCReport.txt @@ -0,0 +1,11 @@ +SAP_sup + +utf-8 + +LF +1 +24 +Plant,Material,Material Description,Batch,Posting Date,Qty in Un. of Entry,Movement Type,Unit of Entry,Entry Date,Material Doc. Year,Document Date,Total valuated stock before the posting,Base Unit of Measure,Quantity,Reference,Purchase Order,Customer,Amount,Amount in LC,Vendor,Item,Material Document,Storage Location,Movement Type Text +plant,material,material_description,batch,posting_date,qty_in_un_of_entry,movement_type,unit_of_entry,entry_date,material_doc_year,document_date,total_valuated_stock_before_the_posting,base_unit_of_measure,quantity,reference,purchase_order,customer,amount,amount_in_lc,vendor,item,material_document,storage_location,movement_type_text +src04.saps_mlcreport +org04.saps_mlcreport diff --git a/s3/data/SAP_sup/settings/POReport.txt b/s3/data/SAP_sup/settings/POReport.txt new file mode 100644 index 00000000..f6c0ca59 --- /dev/null +++ b/s3/data/SAP_sup/settings/POReport.txt @@ -0,0 +1,11 @@ +SAP_sup + +utf-8 + +LF +1 +12 +Purchasing Document,Vendor,Item,Material,Short Text,Document Date,Order Quantity,Order Unit,Net Order Value,Currency,Order Price Unit,Deletion Flag +purchasing_document,vendor,item,material,short_text,document_date,order_quantity,order_unit,net_order_value,currency,order_price_unit,deletion_flag +src04.saps_poreport +org04.saps_poreport diff --git a/s3/data/SAP_sup/settings/QAReport.txt b/s3/data/SAP_sup/settings/QAReport.txt new file mode 100644 index 00000000..794a23a0 --- /dev/null +++ b/s3/data/SAP_sup/settings/QAReport.txt @@ -0,0 +1,11 @@ +SAP_sup + +utf-8 + +LF +1 +24 +Plant,Material,Material Description,Batch,Posting Date,Qty in Un. of Entry,Movement Type,Unit of Entry,Entry Date,Material Doc. Year,Document Date,Total valuated stock before the posting,Base Unit of Measure,Quantity,Reference,Purchase Order,Customer,Amount,Amount in LC,Vendor,Item,Material Document,Storage Location,Movement Type Text +plant,material,material_description,batch,posting_date,qty_in_un_of_entry,movement_type,unit_of_entry,entry_date,material_doc_year,document_date,total_valuated_stock_before_the_posting,base_unit_of_measure,quantity,reference,purchase_order,customer,amount,amount_in_lc,vendor,item,material_document,storage_location,movement_type_text +src04.saps_qareport +org04.saps_qareport diff --git a/s3/data/SAP_sup/settings/StockList.txt b/s3/data/SAP_sup/settings/StockList.txt new file mode 100644 index 00000000..338898d8 --- /dev/null +++ b/s3/data/SAP_sup/settings/StockList.txt @@ -0,0 +1,12 @@ +SAP_sup + +utf-8 + +LF +1 +13 +SPL.stock Indic,Material Num.,Material Desc.,Storage Location,Batch Num.,Expired Date,Unrestricted Stock,In Quality Stock,Blocked Stock,Consign Stock,Total Stock Quantity,Sold to,Name +spl_stock_indic,material_num,material_desc,storage_location,batch_num,expired_date,unrestricted_stock,in_quality_stock,blocked_stock,consign_stock,total_stock_quantity,sold_to,name +src04.saps_stocklist +org04.saps_stocklist +StockList_ex.sql \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/StockList_ex.sql b/s3/data/SAP_sup/settings/StockList_ex.sql new file mode 100644 index 00000000..d3f3d10c --- /dev/null +++ b/s3/data/SAP_sup/settings/StockList_ex.sql @@ -0,0 +1,7 @@ +/* 蓄積スキーマ */ +/* execute_dateがnullのレコードを抽出し、取込ファイル名のyyyymmdd部分を切り出しセットする */ +update src04.saps_stocklist +set + execute_date = STR_TO_DATE(SUBSTRING(file_name,11,8),'%Y%m%d') +where + execute_date is null \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/configmap.config b/s3/data/SAP_sup/settings/configmap.config new file mode 100644 index 00000000..857e4047 --- /dev/null +++ b/s3/data/SAP_sup/settings/configmap.config @@ -0,0 +1,9 @@ +/* SAPデータ取込:SupplyChain */ +GRReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) GRReport.txt +QAReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) QAReport.txt +MLCReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) MLCReport.txt +MLCReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) MLCReport.txt +POReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) POReport.txt +StockList_[0-9]{8}_[0-9]{6}\.(TSV|tsv) StockList.txt +ConfReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) ConfReport.txt +GMReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv) GMReport.txt \ No newline at end of file