From 440e6c8fd094f0800d3d8d9b7555361ae45fdf28 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 26 Jul 2022 09:54:16 +0900 Subject: [PATCH] =?UTF-8?q?fix:=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E5=90=8D=E5=A4=89=E6=9B=B4=E3=80=81=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=81=AE=E3=83=91=E3=82=B9=E3=82=92=E5=A4=89=E6=9B=B4?= =?UTF-8?q?=E3=81=97=E3=81=9F=E3=82=82=E3=81=AE=E3=82=92=E4=B8=80=E5=BA=A6?= =?UTF-8?q?=E3=82=B3=E3=83=9F=E3=83=83=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/launch.json | 16 ++++++++++++++ ecs/crm-datafetch/main.py | 4 ++-- ecs/crm-datafetch/src/aws/s3.py | 4 ++-- ...csvbk.py => backup_crm_csvdata_process.py} | 2 +- .../{resbk.py => backup_crm_data_process.py} | 2 +- .../{chk.py => check_object_info_process.py} | 2 +- ecs/crm-datafetch/src/config/objects.py | 2 +- .../src/{process.py => controller.py} | 22 +++++++++---------- ...conv.py => convert_crm_csvdata_process.py} | 2 +- ecs/crm-datafetch/src/converter/converter.py | 2 +- .../{upld.py => copy_crm_csvdata_process.py} | 4 ++-- .../{fetch.py => fetch_crm_data_process.py} | 4 ++-- ecs/crm-datafetch/src/parser/json_parse.py | 2 +- .../{pre.py => prepare_get_data_process.py} | 4 ++-- .../src/salesforce/salesforce_api.py | 2 +- ...date.py => set_datetime_period_process.py} | 4 ++-- .../src/{ => system_var}/constants.py | 0 .../src/{ => system_var}/environments.py | 2 +- ...{end.py => updload_result_data_process.py} | 4 ++-- ... => upload_last_fetch_datetime_process.py} | 2 +- ecs/crm-datafetch/src/util/logger.py | 2 +- 21 files changed, 52 insertions(+), 36 deletions(-) create mode 100644 ecs/crm-datafetch/.vscode/launch.json rename ecs/crm-datafetch/src/{csvbk.py => backup_crm_csvdata_process.py} (96%) rename ecs/crm-datafetch/src/{resbk.py => backup_crm_data_process.py} (96%) rename ecs/crm-datafetch/src/{chk.py => check_object_info_process.py} (95%) rename ecs/crm-datafetch/src/{process.py => controller.py} (87%) rename ecs/crm-datafetch/src/{conv.py => convert_crm_csvdata_process.py} (95%) rename ecs/crm-datafetch/src/{upld.py => copy_crm_csvdata_process.py} (92%) rename ecs/crm-datafetch/src/{fetch.py => fetch_crm_data_process.py} (98%) rename ecs/crm-datafetch/src/{pre.py => prepare_get_data_process.py} (95%) rename ecs/crm-datafetch/src/{date.py => set_datetime_period_process.py} (95%) rename ecs/crm-datafetch/src/{ => system_var}/constants.py (100%) rename ecs/crm-datafetch/src/{ => system_var}/environments.py (99%) rename ecs/crm-datafetch/src/{end.py => updload_result_data_process.py} (91%) rename ecs/crm-datafetch/src/{upd.py => upload_last_fetch_datetime_process.py} (97%) diff --git a/ecs/crm-datafetch/.vscode/launch.json b/ecs/crm-datafetch/.vscode/launch.json new file mode 100644 index 00000000..2e39a541 --- /dev/null +++ b/ecs/crm-datafetch/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + // エントリーポイントのファイルに変更すること + "program": "main.py", + "console": "integratedTerminal", + "justMyCode": true, + // 環境変数が必要な場合に読み込む環境変数ファイル + "envFile": "${workspaceFolder}/.env", + } + ] +} \ No newline at end of file diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index b0edf545..6e197d30 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -1,4 +1,4 @@ -import src.process as process +import src.controller as controller if __name__ == '__main__': - process.main() + controller.main() diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index e8691e16..dec0ec62 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,8 +1,8 @@ import json import boto3 -from src.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY -from src.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, +from src.system_var.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY +from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME, diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/backup_crm_csvdata_process.py similarity index 96% rename from ecs/crm-datafetch/src/csvbk.py rename to ecs/crm-datafetch/src/backup_crm_csvdata_process.py index 375b273e..1960e7b0 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/backup_crm_csvdata_process.py @@ -1,5 +1,5 @@ from src.aws.s3 import BackupBucket -from src.constants import CSVBK_JP_NAME +from src.system_var.constants import CSVBK_JP_NAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/backup_crm_data_process.py similarity index 96% rename from ecs/crm-datafetch/src/resbk.py rename to ecs/crm-datafetch/src/backup_crm_data_process.py index 61bfe306..130da769 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -1,5 +1,5 @@ from src.aws.s3 import BackupBucket -from src.constants import RESBK_JP_NAME +from src.system_var.constants import RESBK_JP_NAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/check_object_info_process.py similarity index 95% rename from ecs/crm-datafetch/src/chk.py rename to ecs/crm-datafetch/src/check_object_info_process.py index b842d19b..91551d21 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -1,5 +1,5 @@ from src.config.objects import TargetObject -from src.constants import CHK_JP_NAME +from src.system_var.constants import CHK_JP_NAME from src.error.exceptions import InvalidConfigException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 8d9f6f85..c00274bf 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,4 +1,4 @@ -from src.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ +from src.system_var.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ from src.util.dict_checker import DictCheck diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/controller.py similarity index 87% rename from ecs/crm-datafetch/src/process.py rename to ecs/crm-datafetch/src/controller.py index e914a4d4..ebd3b3cd 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/controller.py @@ -1,14 +1,14 @@ -from src.chk import check_object_info # オブジェクト情報形式チェック処理 -from src.conv import convert_crm_csvdata # CSV変換処理 -from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 -from src.date import set_datetime_period # データ取得期間設定処理 -from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.check_object_info_process import check_object_info # オブジェクト情報形式チェック処理 +from src.convert_crm_csvdata_process import convert_crm_csvdata # CSV変換処理 +from src.backup_crm_csvdata_process import backup_crm_csvdata # CSVバックアップ処理 +from src.set_datetime_period_process import set_datetime_period # データ取得期間設定処理 +from src.updload_result_data_process import updload_result_data # 取得処理実施結果アップロード処理 from src.error.exceptions import MeDaCaCRMDataFetchException -from src.fetch import fetch_crm_data # CRMデータ取得処理 -from src.pre import prepare_get_data # データ取得準備処理 -from src.resbk import backup_crm_data # CRM電文データバックアップ処理 -from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 -from src.upld import copy_crm_csvdata # CSVアップロード処理 +from src.fetch_crm_data_process import fetch_crm_data # CRMデータ取得処理 +from src.prepare_get_data_process import prepare_get_data # データ取得準備処理 +from src.backup_crm_data_process import backup_crm_data # CRM電文データバックアップ処理 +from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime # 前回取得日時ファイル更新 +from src.copy_crm_csvdata_process import copy_crm_csvdata # CSVアップロード処理 from src.util.logger import logger_instance as logger @@ -95,7 +95,7 @@ def main() -> None: # 12. 前回取得日時ファイル更新処理を呼びだす logger.info( f'I-CTRL-15 [{target_object.object_name}] 前回取得日時ファイル更新処理呼び出し') - updload_last_fetch_datetime(target_object, last_fetch_datetime) + upload_last_fetch_datetime(target_object, last_fetch_datetime) # 13. オブジェクト処理結果の更新 process_result[target_object.object_name] = 'success' diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/convert_crm_csvdata_process.py similarity index 95% rename from ecs/crm-datafetch/src/conv.py rename to ecs/crm-datafetch/src/convert_crm_csvdata_process.py index 2a4d3be8..4ef6dd11 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/convert_crm_csvdata_process.py @@ -1,4 +1,4 @@ -from src.constants import CONV_JP_NAME +from src.system_var.constants import CONV_JP_NAME from src.converter.converter import CSVStringConverter from src.error.exceptions import DataConvertException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 18fc2923..7b4adb1e 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -3,7 +3,7 @@ import io import re from datetime import datetime -from src.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, +from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, CSV_TRUE_VALUE, YYYYMMDDHHMMSS) diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/copy_crm_csvdata_process.py similarity index 92% rename from ecs/crm-datafetch/src/upld.py rename to ecs/crm-datafetch/src/copy_crm_csvdata_process.py index 2384ae57..ecebe9ea 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/copy_crm_csvdata_process.py @@ -1,6 +1,6 @@ from src.aws.s3 import S3ResourceNonBucket -from src.constants import UPLD_JP_NAME -from src.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, +from src.system_var.constants import UPLD_JP_NAME +from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py similarity index 98% rename from ecs/crm-datafetch/src/fetch.py rename to ecs/crm-datafetch/src/fetch_crm_data_process.py index baa0219f..eac5478b 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -2,8 +2,8 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential -from src.constants import FETCH_JP_NAME -from src.environments import (CRM_AUTH_TIMEOUT, +from src.system_var.constants import FETCH_JP_NAME +from src.system_var.environments import (CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, CRM_FETCH_RECORD_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py index f9cda105..5e7249b9 100644 --- a/ecs/crm-datafetch/src/parser/json_parse.py +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -1,7 +1,7 @@ import json import re -from src.constants import EXCLUDE_SYMBOL +from src.system_var.constants import EXCLUDE_SYMBOL class JsonParser(): diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/prepare_get_data_process.py similarity index 95% rename from ecs/crm-datafetch/src/pre.py rename to ecs/crm-datafetch/src/prepare_get_data_process.py index adb6f68e..e96de09c 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/prepare_get_data_process.py @@ -2,8 +2,8 @@ from datetime import datetime from src.aws.s3 import ConfigBucket from src.config.objects import FetchTargetObjects -from src.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ -from src.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, +from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ +from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, OBJECT_INFO_FOLDER) from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index ac6151a9..a6b9c458 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,5 +1,5 @@ from simple_salesforce import Salesforce -from src.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, +from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, CRM_AUTH_RETRY_INTERVAL, CRM_AUTH_RETRY_MAX_INTERVAL, CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT, diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/set_datetime_period_process.py similarity index 95% rename from ecs/crm-datafetch/src/date.py rename to ecs/crm-datafetch/src/set_datetime_period_process.py index 2896c366..ea281fba 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,7 +1,7 @@ from src.aws.s3 import ConfigBucket from src.config.objects import LastFetchDatetime -from src.constants import DATE_JP_NAME -from src.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.system_var.constants import DATE_JP_NAME +from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/system_var/constants.py similarity index 100% rename from ecs/crm-datafetch/src/constants.py rename to ecs/crm-datafetch/src/system_var/constants.py diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/system_var/environments.py similarity index 99% rename from ecs/crm-datafetch/src/environments.py rename to ecs/crm-datafetch/src/system_var/environments.py index 4a09b2a2..dfa87ae3 100644 --- a/ecs/crm-datafetch/src/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -1,6 +1,6 @@ import os -import src.constants as constants +import src.system_var.constants as constants # environments(task settings file) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/updload_result_data_process.py similarity index 91% rename from ecs/crm-datafetch/src/end.py rename to ecs/crm-datafetch/src/updload_result_data_process.py index 471effc2..e70f4679 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/updload_result_data_process.py @@ -1,6 +1,6 @@ from src.aws.s3 import BackupBucket -from src.constants import END_JP_NAME -from src.environments import PROCESS_RESULT_FILENAME +from src.system_var.constants import END_JP_NAME +from src.system_var.environments import PROCESS_RESULT_FILENAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py similarity index 97% rename from ecs/crm-datafetch/src/upd.py rename to ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index d6a8ef03..f606c9ad 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,7 +1,7 @@ import json from src.aws.s3 import ConfigBucket -from src.constants import UPD_JP_NAME +from src.system_var.constants import UPD_JP_NAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index cb139c98..689d9c36 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -1,6 +1,6 @@ import logging -from src.environments import LOG_LEVEL +from src.system_var.environments import LOG_LEVEL for name in ["boto3", "botocore", "s3transfer", "urllib3"]: logging.getLogger(name).setLevel(logging.WARNING)