nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: フォーマッタを適用。コメントの書きっぷりをを考慮して1行150文字までは許容するようにした

Browse Source
This commit is contained in:
shimoda.m@nds-tyo.co.jp 2022-07-21 19:55:37 +09:00
parent a53e60cbf1
commit 24149be0b1
20 changed files with 184 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
ecs/crm-datafetch/.vscode/recommend_settings.json vendored
View File

@ -1,20 +1,16 @@
{ {
"[python]": { "[python]": {
"editor.defaultFormatter": null, "editor.defaultFormatter": null,
"editor.formatOnSave": true, "editor.formatOnSave": true,
"editor.codeActionsOnSave": { "editor.codeActionsOnSave": {
"source.organizeImports": true "source.organizeImports": true
} }
}, },
"python.linting.lintOnSave": true, "python.linting.lintOnSave": true,
"python.linting.enabled": true, "python.linting.enabled": true,
"python.linting.pylintEnabled": false, "python.linting.pylintEnabled": false,
"python.linting.flake8Enabled": true, "python.linting.flake8Enabled": true,
"python.linting.flake8Args": ["--max-line-length=120"], "python.linting.flake8Args": ["--max-line-length=150"],
"python.formatting.provider": "autopep8", "python.formatting.provider": "autopep8",
"python.formatting.autopep8Args": [ "python.formatting.autopep8Args": ["--max-line-length", "150"]
"--max-line-length", }
"120"
]
}

27
ecs/crm-datafetch/src/aws/s3.py
View File

@ -1,22 +1,13 @@
import boto3
import json import json
from src.environments import (
CRM_CONFIG_BUCKET, import boto3
CRM_BACKUP_BUCKET, from src.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY
IMPORT_DATA_BUCKET, from src.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET,
OBJECT_INFO_FOLDER, CRM_IMPORT_DATA_BACKUP_FOLDER,
OBJECT_INFO_FILENAME, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET,
PROCESS_RESULT_FOLDER, LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME,
CRM_IMPORT_DATA_FOLDER, OBJECT_INFO_FOLDER, PROCESS_RESULT_FOLDER,
CRM_IMPORT_DATA_BACKUP_FOLDER, RESPONSE_JSON_BACKUP_FOLDER)
LAST_FETCH_DATE_FOLDER,
RESPONSE_JSON_BACKUP_FOLDER
)
from src.constants import (
AWS_RESOURCE_S3,
AWS_CLINET_S3,
S3_RESPONSE_BODY
)
class S3Clinet: class S3Clinet:

10
ecs/crm-datafetch/src/chk.py
View File

@ -1,11 +1,7 @@
from src.util.logger import logger_instance as logger
from src.constants import(
CHK_JP_NAME
)
from src.error.exceptions import(
InvalidConfigException
)
from src.config.objects import TargetObject from src.config.objects import TargetObject
from src.constants import CHK_JP_NAME
from src.error.exceptions import InvalidConfigException
from src.util.logger import logger_instance as logger
def check_object_info(object_info, execute_datetime): def check_object_info(object_info, execute_datetime):

4
ecs/crm-datafetch/src/config/objects.py
View File

@ -1,6 +1,4 @@
from src.constants import ( from src.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ
DATE_PATTERN_YYYYMMDDTHHMMSSTZ
)
from src.util.dict_checker import DictCheck from src.util.dict_checker import DictCheck

5
ecs/crm-datafetch/src/constants.py
View File

@ -1,5 +1,6 @@
# environments(task settings file) # environments(task settings file)
LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する
LOG_LEVEL = "LOG_LEVEL"
CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数
CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数
CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数
@ -70,4 +71,4 @@ END_JP_NAME = '取得処理実施結果アップロード処理'
# CSVチェック # CSVチェック
CSV_TRUE_VALUE = '1' CSV_TRUE_VALUE = '1'
CSV_FALSE_VALUE = '0' CSV_FALSE_VALUE = '0'

11
ecs/crm-datafetch/src/conv.py
View File

@ -1,12 +1,9 @@
from datetime import datetime from datetime import datetime
from src.util.logger import logger_instance as logger
from src.constants import ( from src.constants import CONV_JP_NAME
CONV_JP_NAME,
)
from src.error.exceptions import(
DataConvertException
)
from src.converter.converter import CSVStringConverter from src.converter.converter import CSVStringConverter
from src.error.exceptions import DataConvertException
from src.util.logger import logger_instance as logger
def convert_crm_csvdata(target_object, sf_object_jsons): def convert_crm_csvdata(target_object, sf_object_jsons):

13
ecs/crm-datafetch/src/converter/converter.py
View File

@ -1,13 +1,10 @@
import re
import io
import csv import csv
import io
import re
from datetime import datetime from datetime import datetime
from src.constants import(
CSV_TRUE_VALUE, from src.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE,
CSV_FALSE_VALUE, CSV_TRUE_VALUE, YYYYMMDDHHMMSS)
CRM_DATETIME_FORMAT,
YYYYMMDDHHMMSS
)
class CSVStringConverter: class CSVStringConverter:

13
ecs/crm-datafetch/src/csvbk.py
View File

@ -1,11 +1,7 @@
from src.util.logger import logger_instance as logger
from src.aws.s3 import BackupBucket from src.aws.s3 import BackupBucket
from src.constants import ( from src.constants import CSVBK_JP_NAME
CSVBK_JP_NAME from src.error.exceptions import FileUploadException
) from src.util.logger import logger_instance as logger
from src.error.exceptions import(
FileUploadException
)
def backup_crm_csvdata(target_object, date_path, csv_object): def backup_crm_csvdata(target_object, date_path, csv_object):
@ -24,7 +20,8 @@ def backup_crm_csvdata(target_object, date_path, csv_object):
except Exception as e: except Exception as e:
raise FileUploadException( raise FileUploadException(
'E-CSVBK-01', CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') 'E-CSVBK-01',
CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]')
# ③ CSVバックアップ処理の終了ログを出力する # ③ CSVバックアップ処理の終了ログを出力する
logger.info( logger.info(

20
ecs/crm-datafetch/src/date.py
View File

@ -1,18 +1,10 @@
from src.util.logger import logger_instance as logger
from src.constants import(
DATE_JP_NAME
)
from src.environments import (
CRM_CONFIG_BUCKET,
LAST_FETCH_DATE_FOLDER
)
from src.error.exceptions import(
FileNotFoundException,
InvalidConfigException
)
from src.parser.json_parse import JsonParser
from src.config.objects import LastFetchDatetime
from src.aws.s3 import ConfigBucket from src.aws.s3 import ConfigBucket
from src.config.objects import LastFetchDatetime
from src.constants import DATE_JP_NAME
from src.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER
from src.error.exceptions import FileNotFoundException, InvalidConfigException
from src.parser.json_parse import JsonParser
from src.util.logger import logger_instance as logger
def set_datetime_period(target_object, execute_datetime): def set_datetime_period(target_object, execute_datetime):

14
ecs/crm-datafetch/src/end.py
View File

@ -1,14 +1,8 @@
from src.util.logger import logger_instance as logger
from src.constants import (
END_JP_NAME
)
from src.environments import(
PROCESS_RESULT_FILENAME
)
from src.error.exceptions import(
FileUploadException
)
from src.aws.s3 import BackupBucket from src.aws.s3 import BackupBucket
from src.constants import END_JP_NAME
from src.environments import PROCESS_RESULT_FILENAME
from src.error.exceptions import FileUploadException
from src.util.logger import logger_instance as logger
def updload_result_data(process_result, date_path): def updload_result_data(process_result, date_path):

97
ecs/crm-datafetch/src/environments.py
View File

@ -1,38 +1,71 @@
import os import os
import src.constants as constants import src.constants as constants
# environments(task settings file) # environments(task settings file)
LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する
CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] # CRMへの認証処理のタイムアウト秒数 LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO)
CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] # CRMへの認証処理の最大リトライ試行回数 # CRMへの認証処理のタイムアウト秒数
CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] # CRMへの認証処理のリトライ時の初回待ち秒数 CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT]
CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] # CRMへの認証処理のリトライ時の最小待ち秒数 # CRMへの認証処理の最大リトライ試行回数
CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] # CRMへの認証処理のリトライ時の最大待ち秒数 CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT]
CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] # CRMのレコード件数取得処理のタイムアウト秒数 # CRMへの認証処理のリトライ時の初回待ち秒数
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] # CRMのレコード件数取得処理の最大リトライ試行回数 CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL]
CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 # CRMへの認証処理のリトライ時の最小待ち秒数
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL]
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 # CRMへの認証処理のリトライ時の最大待ち秒数
CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] # CRMのレコード取得処理のタイムアウト秒数 CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL]
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] # CRMのレコード取得処理の最大リトライ試行回数 # CRMのレコード件数取得処理のタイムアウト秒数
CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] # CRMのレコード取得処理のリトライ時の初回待ち秒数 CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT]
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] # CRMのレコード取得処理のリトライ時の最小待ち秒数 # CRMのレコード件数取得処理の最大リトライ試行回数
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] # CRMのレコード取得処理のリトライ時の最大待ち秒数 CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT]
# CRMのレコード件数取得処理のリトライ時の初回待ち秒数
CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL]
# CRMのレコード件数取得処理のリトライ時の最小待ち秒数
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL]
# CRMのレコード件数取得処理のリトライ時の最大待ち秒数
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL]
# CRMのレコード取得処理のタイムアウト秒数
CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT]
# CRMのレコード取得処理の最大リトライ試行回数
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT]
# CRMのレコード取得処理のリトライ時の初回待ち秒数
CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL]
# CRMのレコード取得処理のリトライ時の最小待ち秒数
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL]
# CRMのレコード取得処理のリトライ時の最大待ち秒数
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL]
# environments(ECS Task Enviroment) # environments(ECS Task Enviroment)
CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] # CRMのAPI実行のための認証エンドポイントのドメイン # CRMのAPI実行のための認証エンドポイントのドメイン
CRM_USER_NAME = os.environ[constants.CRM_USER_NAME] # CRMのAPI実行用ユーザ名 CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN]
CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD] # CRMのAPI実行用ユーザパスワード # CRMのAPI実行用ユーザ名
CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN] # CRMのAPI実行用ユーザのセキュリティトークン CRM_USER_NAME = os.environ[constants.CRM_USER_NAME]
CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET] # CRMデータ取得用の設定ファイルを格納するバケット名 # CRMのAPI実行用ユーザパスワード
CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] # CRMのバックアップデータを格納するバケット名 CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD]
IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] # CRMの取込データを格納するバケット名 # CRMのAPI実行用ユーザのセキュリティトークン
OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] # CRM取得対象オブジェクトの情報を格納するフォルダパス CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN]
OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] # CRM取得対象オブジェクトの情報のファイル名 # CRMデータ取得用の設定ファイルを格納するバケット名
PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] # CRMデータ取得結果を格納するフォルダパス CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET]
PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] # CRMデータ取得結果を格納するファイル名 # CRMのバックアップデータを格納するバケット名
LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] # CRMからの最終取得日時ファイルを格納するフォルダパス CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET]
CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] # CRMから取得し、取込用に変換したデータを格納するフォルダ # CRMの取込データを格納するバケット名
LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET]
RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] # CRMから取得した生データのバックアップを格納するフォルダパス # CRM取得対象オブジェクトの情報を格納するフォルダパス
CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER]
# CRM取得対象オブジェクトの情報のファイル名
OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME]
# CRMデータ取得結果を格納するフォルダパス
PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER]
# CRMデータ取得結果を格納するファイル名
PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME]
# CRMからの最終取得日時ファイルを格納するフォルダパス
LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER]
# CRMから取得し、取込用に変換したデータを格納するフォルダ
CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER]
# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス
LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER]
# CRMから取得した生データのバックアップを格納するフォルダパス
RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER]
# CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ
CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER]

39
ecs/crm-datafetch/src/fetch.py
View File

@ -1,31 +1,22 @@
from requests.exceptions import ConnectTimeout, ReadTimeout from requests.exceptions import ConnectTimeout, ReadTimeout
from tenacity import retry, stop_after_attempt from tenacity import retry, stop_after_attempt
from tenacity.wait import wait_exponential from tenacity.wait import wait_exponential
from src.constants import FETCH_JP_NAME
from src.environments import (CRM_AUTH_TIMEOUT,
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT,
CRM_FETCH_RECORD_RETRY_INTERVAL,
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL,
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL,
CRM_FETCH_RECORD_TIMEOUT,
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT,
CRM_GET_RECORD_COUNT_RETRY_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL,
CRM_GET_RECORD_COUNT_TIMEOUT)
from src.error.exceptions import DataConvertException, SalesforceAPIException
from src.salesforce.salesforce_api import SalesForceCount, SalesForceData
from src.util.logger import logger_instance as logger from src.util.logger import logger_instance as logger
from src.constants import(
FETCH_JP_NAME
)
from src.environments import(
CRM_AUTH_TIMEOUT,
CRM_GET_RECORD_COUNT_TIMEOUT,
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT,
CRM_GET_RECORD_COUNT_RETRY_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL,
CRM_FETCH_RECORD_TIMEOUT,
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT,
CRM_FETCH_RECORD_RETRY_INTERVAL,
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL,
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL
)
from src.error.exceptions import(
SalesforceAPIException,
DataConvertException
)
from src.salesforce.salesforce_api import(
SalesForceCount,
SalesForceData
)
def fetch_crm_data(target_object, last_fetch_datetime): def fetch_crm_data(target_object, last_fetch_datetime):

4
ecs/crm-datafetch/src/parser/json_parse.py
View File

@ -1,7 +1,9 @@
import re
import json import json
import re
from src.constants import EXCLUDE_SYMBOL from src.constants import EXCLUDE_SYMBOL
class JsonParser(): class JsonParser():
def __init__(self, json_str) -> None: def __init__(self, json_str) -> None:
self.__json_str = json_str self.__json_str = json_str

22
ecs/crm-datafetch/src/pre.py
View File

@ -1,21 +1,13 @@
from datetime import datetime from datetime import datetime
from src.util.logger import logger_instance as logger
from src.constants import(
PRE_JP_NAME,
YYYYMMDDTHHMMSSTZ
)
from src.environments import(
CRM_CONFIG_BUCKET,
OBJECT_INFO_FOLDER,
OBJECT_INFO_FILENAME
)
from src.error.exceptions import(
FileNotFoundException,
InvalidConfigException
)
from src.aws.s3 import ConfigBucket from src.aws.s3 import ConfigBucket
from src.parser.json_parse import JsonParser
from src.config.objects import FetchTargetObjects from src.config.objects import FetchTargetObjects
from src.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ
from src.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME,
OBJECT_INFO_FOLDER)
from src.error.exceptions import FileNotFoundException, InvalidConfigException
from src.parser.json_parse import JsonParser
from src.util.logger import logger_instance as logger
def prepare_get_data(): def prepare_get_data():

24
ecs/crm-datafetch/src/process.py
View File

@ -1,17 +1,15 @@
from src.pre import prepare_get_data # データ取得準備処理 from src.chk import check_object_info # オブジェクト情報形式チェック処理
from src.chk import check_object_info # オブジェクト情報形式チェック処理 from src.conv import convert_crm_csvdata # CSV変換処理
from src.date import set_datetime_period # データ取得期間設定処理 from src.csvbk import backup_crm_csvdata # CSVバックアップ処理
from src.fetch import fetch_crm_data # CRMデータ取得処理 from src.date import set_datetime_period # データ取得期間設定処理
from src.resbk import backup_crm_data # CRM電文データバックアップ処理 from src.end import updload_result_data # 取得処理実施結果アップロード処理
from src.conv import convert_crm_csvdata # CSV変換処理 from src.error.exceptions import MeDaCaCRMDataFetchException
from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 from src.fetch import fetch_crm_data # CRMデータ取得処理
from src.upld import copy_crm_csvdata # CSVアップロード処理 from src.pre import prepare_get_data # データ取得準備処理
from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 from src.resbk import backup_crm_data # CRM電文データバックアップ処理
from src.end import updload_result_data # 取得処理実施結果アップロード処理 from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新
from src.upld import copy_crm_csvdata # CSVアップロード処理
from src.util.logger import logger_instance as logger from src.util.logger import logger_instance as logger
from src.error.exceptions import(
MeDaCaCRMDataFetchException
)
def main() -> None: def main() -> None:

10
ecs/crm-datafetch/src/resbk.py
View File

@ -1,11 +1,7 @@
from src.util.logger import logger_instance as logger
from src.constants import(
RESBK_JP_NAME
)
from src.error.exceptions import(
FileUploadException
)
from src.aws.s3 import BackupBucket from src.aws.s3 import BackupBucket
from src.constants import RESBK_JP_NAME
from src.error.exceptions import FileUploadException
from src.util.logger import logger_instance as logger
def backup_crm_data(object_name, sf_object_jsons, date_path): def backup_crm_data(object_name, sf_object_jsons, date_path):

37
ecs/crm-datafetch/src/salesforce/salesforce_api.py
View File

@ -1,26 +1,19 @@
from simple_salesforce import Salesforce from simple_salesforce import Salesforce
from src.environments import( from src.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT,
CRM_AUTH_DOMAIN, CRM_AUTH_RETRY_INTERVAL,
CRM_USER_NAME, CRM_AUTH_RETRY_MAX_INTERVAL,
CRM_USER_PASSWORD, CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT,
CRM_USER_SECURITY_TOKEN, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT,
CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_RETRY_INTERVAL,
CRM_AUTH_MAX_RETRY_ATTEMPT, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL,
CRM_AUTH_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MIN_INTERVAL,
CRM_AUTH_RETRY_MIN_INTERVAL, CRM_FETCH_RECORD_TIMEOUT,
CRM_AUTH_RETRY_MAX_INTERVAL, CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT,
CRM_GET_RECORD_COUNT_TIMEOUT, CRM_GET_RECORD_COUNT_RETRY_INTERVAL,
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_INTERVAL, CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME,
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN)
CRM_FETCH_RECORD_TIMEOUT,
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT,
CRM_FETCH_RECORD_RETRY_INTERVAL,
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL,
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL
)
FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name}
WHERE SystemModStamp > {last_update_datetime_from} WHERE SystemModStamp > {last_update_datetime_from}

11
ecs/crm-datafetch/src/upd.py
View File

@ -1,12 +1,9 @@
import json import json
from src.util.logger import logger_instance as logger
from src.aws.s3 import ConfigBucket from src.aws.s3 import ConfigBucket
from src.constants import ( from src.constants import UPD_JP_NAME
UPD_JP_NAME from src.error.exceptions import FileUploadException
) from src.util.logger import logger_instance as logger
from src.error.exceptions import(
FileUploadException
)
def updload_last_fetch_datetime(target_object, last_fetch_datetime): def updload_last_fetch_datetime(target_object, last_fetch_datetime):

18
ecs/crm-datafetch/src/upld.py
View File

@ -1,17 +1,9 @@
from src.util.logger import logger_instance as logger
from src.constants import (
UPLD_JP_NAME
)
from src.environments import(
CRM_BACKUP_BUCKET,
IMPORT_DATA_BUCKET,
CRM_IMPORT_DATA_FOLDER,
CRM_IMPORT_DATA_BACKUP_FOLDER,
)
from src.error.exceptions import(
FileUploadException
)
from src.aws.s3 import S3ResourceNonBucket from src.aws.s3 import S3ResourceNonBucket
from src.constants import UPLD_JP_NAME
from src.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER,
CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET)
from src.error.exceptions import FileUploadException
from src.util.logger import logger_instance as logger
def copy_crm_csvdata(target_object, date_path): def copy_crm_csvdata(target_object, date_path):

1
ecs/crm-datafetch/src/util/logger.py
View File

@ -1,4 +1,5 @@
import logging import logging
from src.environments import LOG_LEVEL from src.environments import LOG_LEVEL
for name in ["boto3", "botocore", "s3transfer", "urllib3"]: for name in ["boto3", "botocore", "s3transfer", "urllib3"]: