nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #82 feature-NEWDWH2021-664 into develop-6crm

Browse Source
This commit is contained in:
朝倉 明日香 2022-08-25 11:02:46 +09:00
commit 997dbd6e1a
6 changed files with 244 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ecs/crm-datafetch/src/aws/s3.py
View File

@ -87,7 +87,7 @@ class BackupBucket:
def put_response_json(self, file_path: str, data: dict) -> None:
object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}'
self.__s3_resource.put_object(object_key, json.dumps(data))
self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False))
return
def put_csv(self, file_path: str, data: str) -> None:
@ -97,5 +97,5 @@ class BackupBucket:
def put_result_json(self, file_path: str, data: dict) -> None:
object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}'
self.__s3_resource.put_object(object_key, json.dumps(data))
self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False))
return

11
ecs/crm-datafetch/src/converter/convert_strategy.py
View File

@ -1,4 +1,6 @@
import json
import re
from collections import OrderedDict
from datetime import datetime
from dateutil.tz import gettz
@ -17,6 +19,7 @@ class ConvertStrategyFactory:
self.__datetime_convert_strategy = DatetimeConvertStrategy()
self.__int_convert_strategy = IntConvertStrategy()
self.__string_convert_strategy = StringConvertStrategy()
self.__dict_convert_strategy = DictConvertStrategy()
def create(self, value):
@ -35,6 +38,8 @@ class ConvertStrategyFactory:
elif type(value) == int:
convert_strategy = self.__int_convert_strategy
elif type(value) == dict or type(value) == OrderedDict:
convert_strategy = self.__dict_convert_strategy
else:
convert_strategy = self.__string_convert_strategy
@ -78,3 +83,9 @@ class StringConvertStrategy:
"""string型を変換せずに返す処理"""
# ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す
return convert_value
class DictConvertStrategy:
def convert_value(self, convert_value: dict):
"""dict型の項目を文字列に変換して返す処理"""
return json.dumps(convert_value, ensure_ascii=False)

101
ecs/crm-datafetch/tests/converter/test_convert_strategy.py
View File

@ -1,6 +1,9 @@
from collections import OrderedDict
from src.converter.convert_strategy import (BooleanConvertStrategy,
ConvertStrategyFactory,
DatetimeConvertStrategy,
DictConvertStrategy,
FloatConvertStrategy,
IntConvertStrategy,
NoneValueConvertStrategy,
@ -94,10 +97,10 @@ class TestConvertStrategyFactory:
# Expects
assert type(actual) == DatetimeConvertStrategy
def test_create_other_str(self):
def test_create_str(self):
"""
Cases:
引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること
引数にSalesforce日付型以外の文字列を指定した場合、StringConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
@ -111,10 +114,10 @@ class TestConvertStrategyFactory:
# Expects
assert type(actual) == StringConvertStrategy
def test_create_other_int(self):
def test_create_int(self):
"""
Cases:
引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること
引数に整数を指定した場合、IntConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
@ -128,6 +131,40 @@ class TestConvertStrategyFactory:
# Expects
assert type(actual) == IntConvertStrategy
def test_create_dict(self):
"""
Cases:
引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = ConvertStrategyFactory()
actual = sut.create({'key': 'value'})
# Expects
assert type(actual) == DictConvertStrategy
def test_create_ordered_dict_dict(self):
"""
Cases:
引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = ConvertStrategyFactory()
actual = sut.create(OrderedDict([('key', 'value')]))
# Expects
assert type(actual) == DictConvertStrategy
class TestNoneValueConvertStrategy:
@ -264,3 +301,59 @@ class TestStringConvertStrategy:
# Expects
assert actual == 'テストデータ'
class TestDictConvertStrategy:
def test_convert_value_dict(self):
"""
Cases:
引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = DictConvertStrategy()
actual = sut.convert_value({'テストデータキー': 'テストデータバリュー'})
# Expects
assert actual == '{"テストデータキー": "テストデータバリュー"}'
def test_convert_value_dict_in_line_break(self):
"""
Cases:
引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること(バリューに改行を含む)
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = DictConvertStrategy()
actual = sut.convert_value({'テストデータキー': 'テスト\nデータ\nバリュー'})
# Expects
assert actual == '{"テストデータキー": "テスト\\nデータ\\nバリュー"}'
def test_convert_value_ordered_dict(self):
"""
Cases:
引数に整列された辞書型のデータを指定した場合、JSONの文字列が返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = DictConvertStrategy()
actual = sut.convert_value(OrderedDict(
[('テストデータキー', 'テストデータバリュー')]
))
# Expects
assert actual == '{"テストデータキー": "テストデータバリュー"}'

110
ecs/crm-datafetch/tests/salesforce/test_salesforce.py
View File

@ -9,6 +9,8 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように
変更してしまった場合は各SOQLの取得日付とデータを修正してください
"""
from typing import OrderedDict
import pytest
from requests.exceptions import ConnectTimeout, ReadTimeout
from src.config.objects import LastFetchDatetime, TargetObject
@ -33,8 +35,8 @@ class TestSalesforceApiClient:
FROM
Account
WHERE
SystemModstamp > 2022-08-04T00:00:00.000Z AND
SystemModstamp <= 2022-08-06T00:00:00.000Z
SystemModstamp > 2022-08-23T01:56:39.000Z AND
SystemModstamp <= 2022-08-24T00:00:00.000Z
"""
sut = SalesforceApiClient()
@ -63,7 +65,7 @@ class TestSalesforceApiClient:
actual = sut.fetch_sf_count(soql)
assert actual >= 0
def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self):
def test_fetch_sf_count_by_soql_builder_system_modstamp_to_ge(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること
@ -79,8 +81,8 @@ class TestSalesforceApiClient:
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z',
'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z',
'last_fetch_datetime_from': '2022-08-23T02:38:59.000Z',
'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
@ -106,7 +108,7 @@ class TestSalesforceApiClient:
actual = sut.fetch_sf_count(soql)
assert actual == 1
def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self):
def test_fetch_sf_count_by_soql_builder_system_modstamp_to_lt(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること
@ -122,8 +124,8 @@ class TestSalesforceApiClient:
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z',
'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z',
'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z',
'last_fetch_datetime_to': '2022-08-23T02:39:01.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
@ -160,7 +162,7 @@ class TestSalesforceApiClient:
- LastFetchDatetimeのFromに2000年1月1日を指定する
- LastFetchDatetimeのToに2100年12月31日を指定する
Expects:
取得件数が16になる
取得件数が17になる
"""
execute_datetime = ExecuteDateTime()
@ -190,7 +192,7 @@ class TestSalesforceApiClient:
sut = SalesforceApiClient()
actual = sut.fetch_sf_count(soql)
assert actual == 16
assert actual == 17
def test_fetch_sf_data_one_record(self):
"""
@ -217,8 +219,8 @@ class TestSalesforceApiClient:
FROM
Account
WHERE
SystemModstamp > 2022-08-04T00:00:00.000Z AND
SystemModstamp <= 2022-08-06T00:00:00.000Z
SystemModstamp > 2022-08-23T01:56:39.000Z AND
SystemModstamp <= 2022-08-24T00:00:00.000Z
"""
sut = SalesforceApiClient()
@ -284,7 +286,7 @@ class TestSalesforceApiClient:
actual = sut.fetch_sf_data(soql)
assert len(actual) >= 0
def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self):
def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること
@ -300,8 +302,8 @@ class TestSalesforceApiClient:
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z',
'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z',
'last_fetch_datetime_from': '2022-08-23T03:48:39.000Z',
'last_fetch_datetime_to': '2022-08-23T03:48:40.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
@ -353,7 +355,7 @@ class TestSalesforceApiClient:
assert dict(actual[0]) == expect
def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self):
def test_fetch_sf_data_by_soql_builder_system_modstamp_to_gt(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること
@ -361,16 +363,16 @@ class TestSalesforceApiClient:
- SystemModStampのToが指定日付未満のレコードは取得できないこと
Arranges:
- SalesforceのAccountオブジェクトに、レコードを作成する
- LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定）
- LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定）
- LastFetchDatetimeのFromがSystemModstampより小さくなるように指定する（UTC指定）
- LastFetchDatetimeのToがSystemModstampより大きくなるように指定する（UTC指定）
Expects:
取得できたオブジェクト1件が期待値どおりであること
"""
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z',
'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z',
'last_fetch_datetime_from': '2022-08-23T03:42:24.000Z',
'last_fetch_datetime_to': '2022-08-23T03:42:26.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
@ -422,6 +424,69 @@ class TestSalesforceApiClient:
assert dict(actual[0]) == expect
def test_fetch_sf_data_by_soql_builder_address_item_check(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること
- できること
Arranges:
- SalesforceのAccountオブジェクトに、住所項目を持つレコードを作成する
- 住所項目を持つレコードだけが取れるよう日付を設定する
Expects:
取得できたオブジェクト件数が1件になる
住所項目(BillingAddress)が想定通りの値になっていること
"""
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z',
'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
'columns': [
'Id',
'Name',
'SystemModstamp',
'LastModifiedDate',
'BillingStreet',
'BillingCity',
'BillingState',
'BillingPostalCode',
'BillingCountry',
'BillingLatitude',
'BillingLongitude',
'BillingGeocodeAccuracy',
'BillingAddress',
]
}, execute_datetime)
soql_builder = SOQLBuilder(target_object, last_fetch_datetime)
soql = soql_builder.create_fetch_soql()
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) == 1
expect_address = OrderedDict([
("city", '〇〇区'),
("country", "日本"),
("geocodeAccuracy", None),
("latitude", None),
("longitude", None),
("postalCode", '999-9999'),
("state", '東京都'),
("street", '△△-✗✗'),
])
assert actual[0]['BillingAddress'] == expect_address
assert actual[0]['BillingCity'] == '〇〇区'
assert actual[0]['BillingCountry'] == '日本'
assert actual[0]['BillingGeocodeAccuracy'] is None
assert actual[0]['BillingLatitude'] is None
assert actual[0]['BillingLongitude'] is None
assert actual[0]['BillingPostalCode'] == '999-9999'
assert actual[0]['BillingState'] == '東京都'
assert actual[0]['BillingStreet'] == '△△-✗✗'
def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self):
"""
Cases:
@ -433,7 +498,7 @@ class TestSalesforceApiClient:
- LastFetchDatetimeのFromに2000年1月1日を指定する
- LastFetchDatetimeのToに2100年12月31日を指定する
Expects:
取得できたオブジェクト件数が16件になる
取得できたオブジェクト件数が17件になる
"""
execute_datetime = ExecuteDateTime()
@ -448,6 +513,7 @@ class TestSalesforceApiClient:
'Name',
'SystemModstamp',
'LastModifiedDate',
'BillingAddress',
'CustomItem1__c',
'CustomItem2__c',
'CustomItem3__c',
@ -463,7 +529,7 @@ class TestSalesforceApiClient:
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) == 16
assert len(actual) == 17
# 内容の確認は別のケースで行っているため省略
def test_raise_create_instance_cause_auth_failed(self, monkeypatch):

11
ecs/crm-datafetch/tests/test_backup_crm_data_process.py
View File

@ -44,7 +44,8 @@ class TestBackupCrmDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('Name', 'テスト取引先1')
]),
OrderedDict([
('attributes', OrderedDict([('type', 'Account'),
@ -54,7 +55,8 @@ class TestBackupCrmDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('Name', 'テスト取引先2')
]),
OrderedDict([
('attributes', OrderedDict([('type', 'Account'),
@ -64,7 +66,8 @@ class TestBackupCrmDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('Name', 'テスト取引先3')
]),
]
@ -96,7 +99,7 @@ class TestBackupCrmDataProcess:
actual = s3_client.get_object(
Bucket=bucket_name,
Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json')
assert actual['Body'].read().decode('utf-8') == json.dumps(response_json)
assert actual['Body'].read().decode('utf-8') == json.dumps(response_json, ensure_ascii=False)
# ログの確認
assert generate_log_message_tuple(

47
ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py
View File

@ -34,7 +34,17 @@ class TestConvertCrmCsvDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', True)
('IsDeleted', True),
('PersonMailingAddress', OrderedDict([
('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501
('PersonMailingCity', 'New york city'),
('PersonMailingState', 'Ohaio'),
('PersonMailingPostalCode', '999-9999'),
('PersonMailingCountry', 'US'),
('PersonMailingLatitude', 50.1234567),
('PersonMailingLongitude', 103.1234567),
('PersonMailingGeocodeAccuracy', 'Address'),
])),
]),
OrderedDict([
('attributes', OrderedDict([('type', 'Account'),
@ -44,7 +54,17 @@ class TestConvertCrmCsvDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('PersonMailingAddress', OrderedDict([
('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501
('PersonMailingCity', 'New york city'),
('PersonMailingState', 'Ohaio'),
('PersonMailingPostalCode', '999-9999'),
('PersonMailingCountry', 'US'),
('PersonMailingLatitude', 50.1234567),
('PersonMailingLongitude', 103.1234567),
('PersonMailingGeocodeAccuracy', 'Address'),
])),
]),
OrderedDict([
('attributes', OrderedDict([('type', 'Account'),
@ -54,7 +74,17 @@ class TestConvertCrmCsvDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('PersonMailingAddress', OrderedDict([
('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501
('PersonMailingCity', 'New york city'),
('PersonMailingState', 'Ohaio'),
('PersonMailingPostalCode', '999-9999'),
('PersonMailingCountry', 'US'),
('PersonMailingLatitude', 50.1234567),
('PersonMailingLongitude', 103.1234567),
('PersonMailingGeocodeAccuracy', 'Address'),
])),
]),
]
@ -66,7 +96,8 @@ class TestConvertCrmCsvDataProcess:
'LastModifiedDate',
'LastModifiedById',
'SystemModstamp',
'IsDeleted'
'IsDeleted',
'PersonMailingAddress'
]
}
@ -79,10 +110,10 @@ class TestConvertCrmCsvDataProcess:
# Assert
expect_csv_string = """\
"Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\
"TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\
"TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\
"TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\
"Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted","PersonMailingAddress"\r\n\
"TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\
"TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\
"TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\
"""
# 返り値の期待値チェック
assert isinstance(actual_csv_string, str), 'CSV文字列が返却される'