From 6b0d978378b21d3a02f483b1f97bb0c760637dfc Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 7 Sep 2022 18:16:51 +0900 Subject: [PATCH] Revert "Revert "Merge branch 'develop-6crm' of nds-tyo.git.backlog.com:/NEWDWH2021/newsdwh2021 into develop-6crm"" This reverts commit bae8447658a03ff8a0bd84948d493f332724bebb. --- ecs/crm-datafetch/Dockerfile | 2 +- ecs/crm-datafetch/src/aws/s3.py | 4 +- .../src/converter/convert_strategy.py | 21 +-- .../src/prepare_data_fetch_process.py | 6 +- .../src/system_var/environments.py | 8 +- .../tests/converter/test_convert_strategy.py | 145 ++++++++++++++---- .../tests/converter/test_converter.py | 10 +- .../tests/salesforce/test_salesforce.py | 110 ++++++++++--- .../tests/test_backup_crm_data_process.py | 11 +- .../test_convert_crm_csv_data_process.py | 51 ++++-- .../tests/test_fetch_crm_data_process.py | 60 ++++---- .../tests/test_prepare_data_fetch_process.py | 5 + ecs/crm-datafetch/tests/test_walk_through.py | 24 +-- .../crm/object_info/crm_object_list_diff.json | 3 - s3/config/crm/task_settings/task_settings.env | 7 +- .../settings/CRM_Event_Attendee_vod__c.txt | 6 +- s3/data/crm/settings/CRM_User.txt | 6 +- 17 files changed, 330 insertions(+), 149 deletions(-) diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile index 56087bf0..2d6318e7 100644 --- a/ecs/crm-datafetch/Dockerfile +++ b/ecs/crm-datafetch/Dockerfile @@ -14,6 +14,6 @@ RUN \ pip uninstall -y pipenv virtualenv-clone virtualenv COPY main.py ./ -COPY src ./ +COPY src ./src CMD [ "python", "./main.py" ] diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index d5bd03f0..7b63861f 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -87,7 +87,7 @@ class BackupBucket: def put_response_json(self, file_path: str, data: dict) -> None: object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data)) + self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) return def put_csv(self, file_path: str, data: str) -> None: @@ -97,5 +97,5 @@ class BackupBucket: def put_result_json(self, file_path: str, data: dict) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data)) + self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) return diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 22fca8fc..7c123d39 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,4 +1,6 @@ +import json import re +from collections import OrderedDict from datetime import datetime from dateutil.tz import gettz @@ -12,20 +14,17 @@ from src.system_var.environments import CONVERT_TZ class ConvertStrategyFactory: def __init__(self) -> None: self.__none_value_convert_strategy = NoneValueConvertStrategy() - self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__int_convert_strategy = IntConvertStrategy() self.__string_convert_strategy = StringConvertStrategy() + self.__dict_convert_strategy = DictConvertStrategy() def create(self, value): if value is None: convert_strategy = self.__none_value_convert_strategy - elif type(value) == float: - convert_strategy = self.__float_convert_strategy - elif type(value) == bool: convert_strategy = self.__boolean_convert_strategy @@ -35,6 +34,8 @@ class ConvertStrategyFactory: elif type(value) == int: convert_strategy = self.__int_convert_strategy + elif type(value) == dict or type(value) == OrderedDict: + convert_strategy = self.__dict_convert_strategy else: convert_strategy = self.__string_convert_strategy @@ -60,12 +61,6 @@ class DatetimeConvertStrategy: return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) -class FloatConvertStrategy: - def convert_value(self, convert_value: str) -> int: - """float型をint型に変換する処理""" - return int(convert_value) - - class IntConvertStrategy: def convert_value(self, convert_value: int): """int型を変換せずに返す処理""" @@ -78,3 +73,9 @@ class StringConvertStrategy: """string型を変換せずに返す処理""" # ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す return convert_value + + +class DictConvertStrategy: + def convert_value(self, convert_value: dict): + """dict型の項目を文字列に変換して返す処理""" + return json.dumps(convert_value, ensure_ascii=False) diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index d546c7b8..f14e85b8 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -34,9 +34,9 @@ def prepare_data_fetch_process(): try: # ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する - object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' - logger.debug( - f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') + object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' + logger.info( + f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') config_bucket = ConfigBucket() object_info_file_str = config_bucket.get_object_info_file() diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index 297cfb2b..c457386f 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -9,8 +9,8 @@ LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) # CRMのレコード件数取得処理のタイムアウト秒数 CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) -# CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3)) +# CRMのレコード件数取得処理の最大リトライ試行回数(処理の実施総回数) +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 4)) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 @@ -19,8 +19,8 @@ CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_R CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50)) # CRMのレコード取得処理のタイムアウト秒数 CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300)) -# CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3)) +# CRMのレコード取得処理の最大リトライ試行回数(処理の実施総回数) +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 4)) # CRMのレコード取得処理のリトライ時の初回待ち秒数 CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最小待ち秒数 diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 0e268fc4..129b3980 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,7 +1,9 @@ +from collections import OrderedDict + from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, DatetimeConvertStrategy, - FloatConvertStrategy, + DictConvertStrategy, IntConvertStrategy, NoneValueConvertStrategy, StringConvertStrategy) @@ -29,7 +31,7 @@ class TestConvertStrategyFactory: def test_create_float(self): """ Cases: - 引数に指数表記を指定した場合、FloatConvertStrategyインスタンスが返ってくること + 引数に指数表記を指定した場合、StringConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -41,7 +43,26 @@ class TestConvertStrategyFactory: actual = sut.create(1.2345678E7) # Expects - assert type(actual) == FloatConvertStrategy + # 変換しない + assert type(actual) == StringConvertStrategy + + def test_create_float_scale(self): + """ + Cases: + 引数に少数を指定した場合、StringConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(1.2345678) + + # Expects + # 変換しない + assert type(actual) == StringConvertStrategy def test_create_bool_true(self): """ @@ -94,10 +115,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == DatetimeConvertStrategy - def test_create_other_str(self): + def test_create_str(self): """ Cases: - 引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること + 引数にSalesforce日付型以外の文字列を指定した場合、StringConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -111,10 +132,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == StringConvertStrategy - def test_create_other_int(self): + def test_create_int(self): """ Cases: - 引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること + 引数に整数を指定した場合、IntConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -128,6 +149,40 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == IntConvertStrategy + def test_create_dict(self): + """ + Cases: + 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create({'key': 'value'}) + + # Expects + assert type(actual) == DictConvertStrategy + + def test_create_ordered_dict_dict(self): + """ + Cases: + 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(OrderedDict([('key', 'value')])) + + # Expects + assert type(actual) == DictConvertStrategy + class TestNoneValueConvertStrategy: @@ -206,26 +261,6 @@ class TestDatetimeConvertStrategy: assert actual == "2022-06-14 05:15:32" -class TestFloatConvertStrategy: - - def test_convert_value(self) -> int: - """ - Cases: - 引数に指数表記を指定した場合、整数で返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = FloatConvertStrategy() - actual = sut.convert_value(1.2345678E7) - - # Expects - assert actual == 12345678 - - class TestIntConvertStrategy: def test_convert_value(self): @@ -264,3 +299,59 @@ class TestStringConvertStrategy: # Expects assert actual == 'テストデータ' + + +class TestDictConvertStrategy: + + def test_convert_value_dict(self): + """ + Cases: + 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value({'テストデータキー': 'テストデータバリュー'}) + + # Expects + assert actual == '{"テストデータキー": "テストデータバリュー"}' + + def test_convert_value_dict_in_line_break(self): + """ + Cases: + 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること(バリューに改行を含む) + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value({'テストデータキー': 'テスト\nデータ\nバリュー'}) + + # Expects + assert actual == '{"テストデータキー": "テスト\\nデータ\\nバリュー"}' + + def test_convert_value_ordered_dict(self): + """ + Cases: + 引数に整列された辞書型のデータを指定した場合、JSONの文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value(OrderedDict( + [('テストデータキー', 'テストデータバリュー')] + )) + + # Expects + assert actual == '{"テストデータキー": "テストデータバリュー"}' diff --git a/ecs/crm-datafetch/tests/converter/test_converter.py b/ecs/crm-datafetch/tests/converter/test_converter.py index 0300381d..da80982b 100644 --- a/ecs/crm-datafetch/tests/converter/test_converter.py +++ b/ecs/crm-datafetch/tests/converter/test_converter.py @@ -71,7 +71,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 8), ('RowCause', 'テストのため2'), ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), - ('LastModifiedById', 2.234567E+6), + ('LastModifiedById', 2.23E+0), ('IsDeleted', True) ]), OrderedDict([ @@ -85,7 +85,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 12), ('RowCause', 'テストのため3'), ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), - ('LastModifiedById', 3.234567E+6), + ('LastModifiedById', 3.234567), ('IsDeleted', False) ]) ] @@ -100,9 +100,9 @@ class TestCSVStringConverter: # Expects expected_value = '''\ "Id","AccountId","UserOrGroupId","AccountAccessLevel","OpportunityAccessLevel","CaseAccessLevel","ContactAccessLevel","RowCause","LastModifiedDate","LastModifiedById","IsDeleted"\r\n\ - "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567","0"\r\n\ - "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2234567","1"\r\n\ - "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\ + "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567.0","0"\r\n\ + "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2.23","1"\r\n\ + "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3.234567","0"\r\n\ ''' # expected_valueのインデントが半角スペースと認識されてしまうため、`textwrap.dedent`にて補正 diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index b815e772..64530071 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -9,6 +9,8 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように 変更してしまった場合は各SOQLの取得日付とデータを修正してください """ +from typing import OrderedDict + import pytest from requests.exceptions import ConnectTimeout, ReadTimeout from src.config.objects import LastFetchDatetime, TargetObject @@ -33,8 +35,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-06T00:00:00.000Z + SystemModstamp > 2022-08-23T01:56:39.000Z AND + SystemModstamp <= 2022-08-24T00:00:00.000Z """ sut = SalesforceApiClient() @@ -63,7 +65,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual >= 0 - def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -79,8 +81,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + 'last_fetch_datetime_from': '2022-08-23T02:38:59.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -106,7 +108,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual == 1 - def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_to_lt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -122,8 +124,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:01.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -160,7 +162,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得件数が16になる + 取得件数が17になる """ execute_datetime = ExecuteDateTime() @@ -190,7 +192,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) - assert actual == 16 + assert actual == 17 def test_fetch_sf_data_one_record(self): """ @@ -217,8 +219,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-06T00:00:00.000Z + SystemModstamp > 2022-08-23T01:56:39.000Z AND + SystemModstamp <= 2022-08-24T00:00:00.000Z """ sut = SalesforceApiClient() @@ -284,7 +286,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_data(soql) assert len(actual) >= 0 - def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -300,8 +302,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + 'last_fetch_datetime_from': '2022-08-23T03:48:39.000Z', + 'last_fetch_datetime_to': '2022-08-23T03:48:40.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -353,7 +355,7 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect - def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_to_gt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -361,16 +363,16 @@ class TestSalesforceApiClient: - SystemModStampのToが指定日付未満のレコードは取得できないこと Arranges: - SalesforceのAccountオブジェクトに、レコードを作成する - - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） - - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） + - LastFetchDatetimeのFromがSystemModstampより小さくなるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstampより大きくなるように指定する（UTC指定） Expects: 取得できたオブジェクト1件が期待値どおりであること """ execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + 'last_fetch_datetime_from': '2022-08-23T03:42:24.000Z', + 'last_fetch_datetime_to': '2022-08-23T03:42:26.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -422,6 +424,69 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_by_soql_builder_address_item_check(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - できること + Arranges: + - SalesforceのAccountオブジェクトに、住所項目を持つレコードを作成する + - 住所項目を持つレコードだけが取れるよう日付を設定する + Expects: + 取得できたオブジェクト件数が1件になる + 住所項目(BillingAddress)が想定通りの値になっていること + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'BillingStreet', + 'BillingCity', + 'BillingState', + 'BillingPostalCode', + 'BillingCountry', + 'BillingLatitude', + 'BillingLongitude', + 'BillingGeocodeAccuracy', + 'BillingAddress', + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 1 + expect_address = OrderedDict([ + ("city", '〇〇区'), + ("country", "日本"), + ("geocodeAccuracy", None), + ("latitude", None), + ("longitude", None), + ("postalCode", '999-9999'), + ("state", '東京都'), + ("street", '△△-✗✗'), + ]) + + assert actual[0]['BillingAddress'] == expect_address + assert actual[0]['BillingCity'] == '〇〇区' + assert actual[0]['BillingCountry'] == '日本' + assert actual[0]['BillingGeocodeAccuracy'] is None + assert actual[0]['BillingLatitude'] is None + assert actual[0]['BillingLongitude'] is None + assert actual[0]['BillingPostalCode'] == '999-9999' + assert actual[0]['BillingState'] == '東京都' + assert actual[0]['BillingStreet'] == '△△-✗✗' + def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): """ Cases: @@ -433,7 +498,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得できたオブジェクト件数が16件になる + 取得できたオブジェクト件数が17件になる """ execute_datetime = ExecuteDateTime() @@ -448,6 +513,7 @@ class TestSalesforceApiClient: 'Name', 'SystemModstamp', 'LastModifiedDate', + 'BillingAddress', 'CustomItem1__c', 'CustomItem2__c', 'CustomItem3__c', @@ -463,7 +529,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_data(soql) - assert len(actual) == 16 + assert len(actual) == 17 # 内容の確認は別のケースで行っているため省略 def test_raise_create_instance_cause_auth_failed(self, monkeypatch): diff --git a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py index 9ad35d52..4e48c419 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py @@ -44,7 +44,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先1') ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -54,7 +55,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先2') ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -64,7 +66,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先3') ]), ] @@ -96,7 +99,7 @@ class TestBackupCrmDataProcess: actual = s3_client.get_object( Bucket=bucket_name, Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json') - assert actual['Body'].read().decode('utf-8') == json.dumps(response_json) + assert actual['Body'].read().decode('utf-8') == json.dumps(response_json, ensure_ascii=False) # ログの確認 assert generate_log_message_tuple( diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py index 49d960c0..bf439dac 100644 --- a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -34,7 +34,17 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', True) + ('IsDeleted', True), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -42,9 +52,19 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST002'), ('AccountNumber', 'test002'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567E+6), + ('LastModifiedById', 1.23E+0), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -52,9 +72,19 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST003'), ('AccountNumber', 'test003'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567E+6), + ('LastModifiedById', 1.234567), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), ] @@ -66,7 +96,8 @@ class TestConvertCrmCsvDataProcess: 'LastModifiedDate', 'LastModifiedById', 'SystemModstamp', - 'IsDeleted' + 'IsDeleted', + 'PersonMailingAddress' ] } @@ -79,10 +110,10 @@ class TestConvertCrmCsvDataProcess: # Assert expect_csv_string = """\ - "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ - "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ - "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ - "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted","PersonMailingAddress"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567.0","2022-06-01 09:00:00","1","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1.23","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1.234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ """ # 返り値の期待値チェック assert isinstance(actual_csv_string, str), 'CSV文字列が返却される' diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py index 66f5577e..6b17477f 100644 --- a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -217,7 +217,7 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - データ件数取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -225,7 +225,7 @@ class TestFetchCrmDataProcess: - データ件数取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -236,8 +236,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -271,15 +271,15 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - データ件数取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする + - データ件数取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データ件数取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -290,13 +290,13 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value - mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception] + mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception, exception] mock_client_inst.fetch_sf_data.return_value = common_expect # Act with pytest.raises(SalesforceAPIException) as e: @@ -304,16 +304,16 @@ class TestFetchCrmDataProcess: # Assert - # 取得は3回行われる - assert mock_counter_inst.describe.call_count == 3 - # 足し込みは2回のみ - assert mock_counter_inst.increment.call_count == 2 + # 取得は4回行われる + assert mock_counter_inst.describe.call_count == 4 + # 足し込みは3回のみ + assert mock_counter_inst.increment.call_count == 3 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 2 + assert called_log_counts == 3 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-01' @@ -333,7 +333,7 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を3に設定する + - レコード取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - レコード取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -341,7 +341,7 @@ class TestFetchCrmDataProcess: - データレコード取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -352,8 +352,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -387,15 +387,15 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を3に設定する + - レコード取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - レコード取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする + - レコード取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データレコード取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_INTERVAL', 1) @@ -406,30 +406,30 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value mock_client_inst.fetch_sf_count.return_value = 1 - mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception] + mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception, exception] # Act with pytest.raises(SalesforceAPIException) as e: fetch_crm_data_process(common_target_object, common_last_fetch_datetime) # Assert - # 取得は3回行われる - assert mock_counter_inst.describe.call_count == 3 - # 足し込みは2回のみ - assert mock_counter_inst.increment.call_count == 2 + # 取得は4回行われる + assert mock_counter_inst.describe.call_count == 4 + # 足し込みは3回のみ + assert mock_counter_inst.increment.call_count == 3 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 2 + assert called_log_counts == 3 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-02' diff --git a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py index 9471946f..8eb3d476 100644 --- a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py @@ -7,6 +7,9 @@ from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.prepare_data_fetch_process import prepare_data_fetch_process from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ +from src.system_var.environments import (CRM_CONFIG_BUCKET, + OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) from .test_utils.log_message import generate_log_message_tuple @@ -80,6 +83,8 @@ class TestPrepareDataFetchProcess: # ログの確認 assert generate_log_message_tuple(log_message='I-PRE-01 データ取得準備処理を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples + assert generate_log_message_tuple( + log_message=f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}]') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples def test_call_depended_modules(self): diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index e8c3b6e4..e2be6670 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -73,7 +73,6 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): object_info_list = get_object_config_list('object_info') for object_info in object_info_list: json_file = read_json(object_info) - json_file = to_upload_json(json_file) upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') object_info_files.append(json_file) last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') @@ -105,7 +104,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # Act logger = logging.getLogger() - logger.setLevel(logging.INFO) + logger.setLevel(logging.DEBUG) logger.info(f'##########################') logger.info(f'# 差分データ取得処理:実行開始 #') logger.info(f'##########################') @@ -118,7 +117,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # ループ前のログ確認 assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages - assert_prepare_process_log(log_messages, now) + assert_prepare_process_log(log_messages, now, path.basename(object_info_list[0])) assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages # オブジェクト情報を取得する(diff) object_info_list = object_info_files[0] @@ -207,9 +206,10 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): """ -def assert_prepare_process_log(log_messages, now): +def assert_prepare_process_log(log_messages, now, object_info_list_filename): assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages + assert f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{object_info_list_filename}]' in log_messages assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages @@ -295,21 +295,6 @@ def upload_json(json_file, s3_client, bucket, folder): s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str) -def to_upload_json(json_file): - """Userオブジェクトの取得できないプロパティを取り除く - TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す - """ - for object_info in json_file['objects']: - if object_info['object_name'] != 'User': - continue - columns: list = object_info['columns'] - columns.remove('LastPasswordChangeDate') - columns.remove('NumberOfFailedLogins') - columns.remove('UserPreferencesNativeEmailClient') - - return json_file - - def set_environment(monkeypatch): # 環境変数を設定(CRMの認証情報は別途設定しておくこと) monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) @@ -324,3 +309,4 @@ def set_environment(monkeypatch): monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) + monkeypatch.setattr('src.prepare_data_fetch_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index a5f8da67..dda641e2 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1645,13 +1645,11 @@ "DelegatedApproverId", "ManagerId", "LastLoginDate", - "LastPasswordChangeDate", "CreatedDate", "CreatedById", "LastModifiedDate", "LastModifiedById", "SystemModstamp", - "NumberOfFailedLogins", "OfflineTrialExpirationDate", "OfflinePdaTrialExpirationDate", "UserPermissionsMarketingUser", @@ -1725,7 +1723,6 @@ "UserPreferencesSRHOverrideActivities", "UserPreferencesNewLightningReportRunPageEnabled", "UserPreferencesReverseOpenActivitiesView", - "UserPreferencesNativeEmailClient", "UserPreferencesHideBrowseProductRedirectConfirmation", "UserPreferencesHideOnlineSalesAppWelcomeMat", "ContactId", diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index 580bf564..3ae4d809 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -1,13 +1,14 @@ LOG_LEVEL=INFO CRM_AUTH_TIMEOUT=100 CRM_GET_RECORD_COUNT_TIMEOUT=300 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=4 CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL=50 CRM_FETCH_RECORD_TIMEOUT=300 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=4 CRM_FETCH_RECORD_RETRY_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 -CONVERT_TZ='Asia/Tokyo' \ No newline at end of file +CONVERT_TZ=Asia/Tokyo +OBJECT_INFO_FILENAME=crm_object_list_diff.json \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt index d5d6c0dc..813f86ee 100644 --- a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt +++ b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -31 -Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c -id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c +32 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c,MSJ_Registration__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c,msj_registration__c src02.crm_event_attendee_vod__c org02.crm_event_attendee_vod__c diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt index a0d5c519..727d82e0 100644 --- a/s3/data/crm/settings/CRM_User.txt +++ b/s3/data/crm/settings/CRM_User.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -203 -Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,LastPasswordChangeDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,NumberOfFailedLogins,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesNativeEmailClient,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c -id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,last_password_change_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,number_of_failed_logins,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_native_email_client,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c +200 +Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c +id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c src02.crm_user org02.crm_user