From 24149be0b1b00071ac8abbed876dc563ff887247 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:55:37 +0900 Subject: [PATCH] =?UTF-8?q?style:=20=E3=83=95=E3=82=A9=E3=83=BC=E3=83=9E?= =?UTF-8?q?=E3=83=83=E3=82=BF=E3=82=92=E9=81=A9=E7=94=A8=E3=80=82=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=81=AE=E6=9B=B8=E3=81=8D=E3=81=A3?= =?UTF-8?q?=E3=81=B7=E3=82=8A=E3=82=92=E3=82=92=E8=80=83=E6=85=AE=E3=81=97?= =?UTF-8?q?=E3=81=A61=E8=A1=8C150=E6=96=87=E5=AD=97=E3=81=BE=E3=81=A7?= =?UTF-8?q?=E3=81=AF=E8=A8=B1=E5=AE=B9=E3=81=99=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.vscode/recommend_settings.json | 34 +++---- ecs/crm-datafetch/src/aws/s3.py | 27 ++---- ecs/crm-datafetch/src/chk.py | 10 +- ecs/crm-datafetch/src/config/objects.py | 4 +- ecs/crm-datafetch/src/constants.py | 5 +- ecs/crm-datafetch/src/conv.py | 11 +-- ecs/crm-datafetch/src/converter/converter.py | 13 +-- ecs/crm-datafetch/src/csvbk.py | 13 +-- ecs/crm-datafetch/src/date.py | 20 ++-- ecs/crm-datafetch/src/end.py | 14 +-- ecs/crm-datafetch/src/environments.py | 97 +++++++++++++------ ecs/crm-datafetch/src/fetch.py | 39 +++----- ecs/crm-datafetch/src/parser/json_parse.py | 4 +- ecs/crm-datafetch/src/pre.py | 22 ++--- ecs/crm-datafetch/src/process.py | 24 +++-- ecs/crm-datafetch/src/resbk.py | 10 +- .../src/salesforce/salesforce_api.py | 37 +++---- ecs/crm-datafetch/src/upd.py | 11 +-- ecs/crm-datafetch/src/upld.py | 18 +--- ecs/crm-datafetch/src/util/logger.py | 1 + 20 files changed, 184 insertions(+), 230 deletions(-) diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index a29de1db..72dacf55 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -1,20 +1,16 @@ { - "[python]": { - "editor.defaultFormatter": null, - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true - } - }, - "python.linting.lintOnSave": true, - "python.linting.enabled": true, - "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": true, - "python.linting.flake8Args": ["--max-line-length=120"], - "python.formatting.provider": "autopep8", - "python.formatting.autopep8Args": [ - "--max-line-length", - "120" - ] - } - \ No newline at end of file + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": ["--max-line-length=150"], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Args": ["--max-line-length", "150"] +} diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index 229b8666..e8691e16 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,22 +1,13 @@ -import boto3 import json -from src.environments import ( - CRM_CONFIG_BUCKET, - CRM_BACKUP_BUCKET, - IMPORT_DATA_BUCKET, - OBJECT_INFO_FOLDER, - OBJECT_INFO_FILENAME, - PROCESS_RESULT_FOLDER, - CRM_IMPORT_DATA_FOLDER, - CRM_IMPORT_DATA_BACKUP_FOLDER, - LAST_FETCH_DATE_FOLDER, - RESPONSE_JSON_BACKUP_FOLDER -) -from src.constants import ( - AWS_RESOURCE_S3, - AWS_CLINET_S3, - S3_RESPONSE_BODY -) + +import boto3 +from src.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY +from src.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, + CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, + LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER, PROCESS_RESULT_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER) class S3Clinet: diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/chk.py index 077f0ee2..b842d19b 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/chk.py @@ -1,11 +1,7 @@ -from src.util.logger import logger_instance as logger -from src.constants import( - CHK_JP_NAME -) -from src.error.exceptions import( - InvalidConfigException -) from src.config.objects import TargetObject +from src.constants import CHK_JP_NAME +from src.error.exceptions import InvalidConfigException +from src.util.logger import logger_instance as logger def check_object_info(object_info, execute_datetime): diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index edce5b33..8d9f6f85 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,6 +1,4 @@ -from src.constants import ( - DATE_PATTERN_YYYYMMDDTHHMMSSTZ -) +from src.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ from src.util.dict_checker import DictCheck diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/constants.py index a394df16..fde2ec65 100644 --- a/ecs/crm-datafetch/src/constants.py +++ b/ecs/crm-datafetch/src/constants.py @@ -1,5 +1,6 @@ # environments(task settings file) -LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +LOG_LEVEL = "LOG_LEVEL" CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 @@ -70,4 +71,4 @@ END_JP_NAME = '取得処理実施結果アップロード処理' # CSVチェック CSV_TRUE_VALUE = '1' -CSV_FALSE_VALUE = '0' \ No newline at end of file +CSV_FALSE_VALUE = '0' diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py index eb17c4c3..cfcc6a8a 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/conv.py @@ -1,12 +1,9 @@ from datetime import datetime -from src.util.logger import logger_instance as logger -from src.constants import ( - CONV_JP_NAME, -) -from src.error.exceptions import( - DataConvertException -) + +from src.constants import CONV_JP_NAME from src.converter.converter import CSVStringConverter +from src.error.exceptions import DataConvertException +from src.util.logger import logger_instance as logger def convert_crm_csvdata(target_object, sf_object_jsons): diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 1b785c35..18fc2923 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -1,13 +1,10 @@ -import re -import io import csv +import io +import re from datetime import datetime -from src.constants import( - CSV_TRUE_VALUE, - CSV_FALSE_VALUE, - CRM_DATETIME_FORMAT, - YYYYMMDDHHMMSS -) + +from src.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, + CSV_TRUE_VALUE, YYYYMMDDHHMMSS) class CSVStringConverter: diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/csvbk.py index 75c5f780..375b273e 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/csvbk.py @@ -1,11 +1,7 @@ -from src.util.logger import logger_instance as logger from src.aws.s3 import BackupBucket -from src.constants import ( - CSVBK_JP_NAME -) -from src.error.exceptions import( - FileUploadException -) +from src.constants import CSVBK_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def backup_crm_csvdata(target_object, date_path, csv_object): @@ -24,7 +20,8 @@ def backup_crm_csvdata(target_object, date_path, csv_object): except Exception as e: raise FileUploadException( - 'E-CSVBK-01', CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') + 'E-CSVBK-01', + CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') # ③ CSVバックアップ処理の終了ログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/date.py index adcfdd4c..2896c366 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/date.py @@ -1,18 +1,10 @@ -from src.util.logger import logger_instance as logger -from src.constants import( - DATE_JP_NAME -) -from src.environments import ( - CRM_CONFIG_BUCKET, - LAST_FETCH_DATE_FOLDER -) -from src.error.exceptions import( - FileNotFoundException, - InvalidConfigException -) -from src.parser.json_parse import JsonParser -from src.config.objects import LastFetchDatetime from src.aws.s3 import ConfigBucket +from src.config.objects import LastFetchDatetime +from src.constants import DATE_JP_NAME +from src.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.error.exceptions import FileNotFoundException, InvalidConfigException +from src.parser.json_parse import JsonParser +from src.util.logger import logger_instance as logger def set_datetime_period(target_object, execute_datetime): diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/end.py index 7604b5b3..471effc2 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/end.py @@ -1,14 +1,8 @@ -from src.util.logger import logger_instance as logger -from src.constants import ( - END_JP_NAME -) -from src.environments import( - PROCESS_RESULT_FILENAME -) -from src.error.exceptions import( - FileUploadException -) from src.aws.s3 import BackupBucket +from src.constants import END_JP_NAME +from src.environments import PROCESS_RESULT_FILENAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def updload_result_data(process_result, date_path): diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/environments.py index 262a2fbd..4a09b2a2 100644 --- a/ecs/crm-datafetch/src/environments.py +++ b/ecs/crm-datafetch/src/environments.py @@ -1,38 +1,71 @@ import os + import src.constants as constants # environments(task settings file) -LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] # CRMのレコード取得処理のリトライ時の最大待ち秒数 +# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) +# CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] +# CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] +# CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] +# CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] +# CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] +# CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] +# CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] +# CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] +# CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] +# CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] +# CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] +# CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] +# CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] +# CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] +# CRMのレコード取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] # environments(ECS Task Enviroment) -CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] # CRMのAPI実行のための認証エンドポイントのドメイン -CRM_USER_NAME = os.environ[constants.CRM_USER_NAME] # CRMのAPI実行用ユーザ名 -CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD] # CRMのAPI実行用ユーザパスワード -CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN] # CRMのAPI実行用ユーザのセキュリティトークン -CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET] # CRMデータ取得用の設定ファイルを格納するバケット名 -CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] # CRMのバックアップデータを格納するバケット名 -IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] # CRMの取込データを格納するバケット名 -OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] # CRM取得対象オブジェクトの情報のファイル名 -PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] # CRMデータ取得結果を格納するファイル名 -LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] # CRMからの最終取得日時ファイルを格納するフォルダパス -CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] # CRMから取得し、取込用に変換したデータを格納するフォルダ -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] # CRMから取得した生データのバックアップを格納するフォルダパス -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +# CRMのAPI実行のための認証エンドポイントのドメイン +CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] +# CRMのAPI実行用ユーザ名 +CRM_USER_NAME = os.environ[constants.CRM_USER_NAME] +# CRMのAPI実行用ユーザパスワード +CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD] +# CRMのAPI実行用ユーザのセキュリティトークン +CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN] +# CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET] +# CRMのバックアップデータを格納するバケット名 +CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] +# CRMの取込データを格納するバケット名 +IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] +# CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] +# CRM取得対象オブジェクトの情報のファイル名 +OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] +# CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] +# CRMデータ取得結果を格納するファイル名 +PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] +# CRMからの最終取得日時ファイルを格納するフォルダパス +LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] +# CRMから取得し、取込用に変換したデータを格納するフォルダ +CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] +# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] +# CRMから取得した生データのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] +# CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py index 9670767a..893aaadc 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch.py @@ -1,31 +1,22 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential + +from src.constants import FETCH_JP_NAME +from src.environments import (CRM_AUTH_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT) +from src.error.exceptions import DataConvertException, SalesforceAPIException +from src.salesforce.salesforce_api import SalesForceCount, SalesForceData from src.util.logger import logger_instance as logger -from src.constants import( - FETCH_JP_NAME -) -from src.environments import( - CRM_AUTH_TIMEOUT, - CRM_GET_RECORD_COUNT_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL -) -from src.error.exceptions import( - SalesforceAPIException, - DataConvertException -) -from src.salesforce.salesforce_api import( - SalesForceCount, - SalesForceData -) def fetch_crm_data(target_object, last_fetch_datetime): diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py index 4273125d..f9cda105 100644 --- a/ecs/crm-datafetch/src/parser/json_parse.py +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -1,7 +1,9 @@ -import re import json +import re + from src.constants import EXCLUDE_SYMBOL + class JsonParser(): def __init__(self, json_str) -> None: self.__json_str = json_str diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/pre.py index c8190f12..adb6f68e 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/pre.py @@ -1,21 +1,13 @@ from datetime import datetime -from src.util.logger import logger_instance as logger -from src.constants import( - PRE_JP_NAME, - YYYYMMDDTHHMMSSTZ -) -from src.environments import( - CRM_CONFIG_BUCKET, - OBJECT_INFO_FOLDER, - OBJECT_INFO_FILENAME -) -from src.error.exceptions import( - FileNotFoundException, - InvalidConfigException -) + from src.aws.s3 import ConfigBucket -from src.parser.json_parse import JsonParser from src.config.objects import FetchTargetObjects +from src.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ +from src.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) +from src.error.exceptions import FileNotFoundException, InvalidConfigException +from src.parser.json_parse import JsonParser +from src.util.logger import logger_instance as logger def prepare_get_data(): diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/process.py index 09d938af..add1e98d 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/process.py @@ -1,17 +1,15 @@ -from src.pre import prepare_get_data # データ取得準備処理 -from src.chk import check_object_info # オブジェクト情報形式チェック処理 -from src.date import set_datetime_period # データ取得期間設定処理 -from src.fetch import fetch_crm_data # CRMデータ取得処理 -from src.resbk import backup_crm_data # CRM電文データバックアップ処理 -from src.conv import convert_crm_csvdata # CSV変換処理 -from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 -from src.upld import copy_crm_csvdata # CSVアップロード処理 -from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 -from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.chk import check_object_info # オブジェクト情報形式チェック処理 +from src.conv import convert_crm_csvdata # CSV変換処理 +from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 +from src.date import set_datetime_period # データ取得期間設定処理 +from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.error.exceptions import MeDaCaCRMDataFetchException +from src.fetch import fetch_crm_data # CRMデータ取得処理 +from src.pre import prepare_get_data # データ取得準備処理 +from src.resbk import backup_crm_data # CRM電文データバックアップ処理 +from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 +from src.upld import copy_crm_csvdata # CSVアップロード処理 from src.util.logger import logger_instance as logger -from src.error.exceptions import( - MeDaCaCRMDataFetchException -) def main() -> None: diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/resbk.py index 21d516ef..61bfe306 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/resbk.py @@ -1,11 +1,7 @@ -from src.util.logger import logger_instance as logger -from src.constants import( - RESBK_JP_NAME -) -from src.error.exceptions import( - FileUploadException -) from src.aws.s3 import BackupBucket +from src.constants import RESBK_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def backup_crm_data(object_name, sf_object_jsons, date_path): diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index 0bce5675..ac6151a9 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,26 +1,19 @@ from simple_salesforce import Salesforce -from src.environments import( - CRM_AUTH_DOMAIN, - CRM_USER_NAME, - CRM_USER_PASSWORD, - CRM_USER_SECURITY_TOKEN, - CRM_AUTH_TIMEOUT, - CRM_AUTH_MAX_RETRY_ATTEMPT, - CRM_AUTH_RETRY_INTERVAL, - CRM_AUTH_RETRY_MIN_INTERVAL, - CRM_AUTH_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL -) - +from src.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, + CRM_AUTH_RETRY_INTERVAL, + CRM_AUTH_RETRY_MAX_INTERVAL, + CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME, + CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN) FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} WHERE SystemModStamp > {last_update_datetime_from} diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py index f8dd4350..d69bddcc 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upd.py @@ -1,12 +1,9 @@ import json -from src.util.logger import logger_instance as logger + from src.aws.s3 import ConfigBucket -from src.constants import ( - UPD_JP_NAME -) -from src.error.exceptions import( - FileUploadException -) +from src.constants import UPD_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def updload_last_fetch_datetime(target_object, last_fetch_datetime): diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/upld.py index c3c3140f..2384ae57 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/upld.py @@ -1,17 +1,9 @@ -from src.util.logger import logger_instance as logger -from src.constants import ( - UPLD_JP_NAME -) -from src.environments import( - CRM_BACKUP_BUCKET, - IMPORT_DATA_BUCKET, - CRM_IMPORT_DATA_FOLDER, - CRM_IMPORT_DATA_BACKUP_FOLDER, -) -from src.error.exceptions import( - FileUploadException -) from src.aws.s3 import S3ResourceNonBucket +from src.constants import UPLD_JP_NAME +from src.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def copy_crm_csvdata(target_object, date_path): diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index ec0baf55..cb139c98 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -1,4 +1,5 @@ import logging + from src.environments import LOG_LEVEL for name in ["boto3", "botocore", "s3transfer", "urllib3"]: