From bae8447658a03ff8a0bd84948d493f332724bebb Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Fri, 2 Sep 2022 15:33:15 +0900 Subject: [PATCH] Revert "Merge branch 'develop-6crm' of nds-tyo.git.backlog.com:/NEWDWH2021/newsdwh2021 into develop-6crm" This reverts commit 820491adc06bb8bbd5c4bf0ec676176a6b5820eb, reversing changes made to 831de315e59bb89aa1bc4801abeacc4a5ad7a30f. --- ecs/crm-datafetch/Dockerfile | 2 +- ecs/crm-datafetch/src/aws/s3.py | 4 +- .../src/converter/convert_strategy.py | 21 ++- .../src/prepare_data_fetch_process.py | 6 +- .../src/system_var/environments.py | 8 +- .../tests/converter/test_convert_strategy.py | 145 ++++-------------- .../tests/converter/test_converter.py | 10 +- .../tests/salesforce/test_salesforce.py | 110 +++---------- .../tests/test_backup_crm_data_process.py | 11 +- .../test_convert_crm_csv_data_process.py | 51 ++---- .../tests/test_fetch_crm_data_process.py | 60 ++++---- .../tests/test_prepare_data_fetch_process.py | 5 - ecs/crm-datafetch/tests/test_walk_through.py | 24 ++- .../crm/object_info/crm_object_list_diff.json | 3 + s3/config/crm/task_settings/task_settings.env | 7 +- .../settings/CRM_Event_Attendee_vod__c.txt | 6 +- s3/data/crm/settings/CRM_User.txt | 6 +- 17 files changed, 149 insertions(+), 330 deletions(-) diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile index 2d6318e7..56087bf0 100644 --- a/ecs/crm-datafetch/Dockerfile +++ b/ecs/crm-datafetch/Dockerfile @@ -14,6 +14,6 @@ RUN \ pip uninstall -y pipenv virtualenv-clone virtualenv COPY main.py ./ -COPY src ./src +COPY src ./ CMD [ "python", "./main.py" ] diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index 7b63861f..d5bd03f0 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -87,7 +87,7 @@ class BackupBucket: def put_response_json(self, file_path: str, data: dict) -> None: object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) + self.__s3_resource.put_object(object_key, json.dumps(data)) return def put_csv(self, file_path: str, data: str) -> None: @@ -97,5 +97,5 @@ class BackupBucket: def put_result_json(self, file_path: str, data: dict) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) + self.__s3_resource.put_object(object_key, json.dumps(data)) return diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 7c123d39..22fca8fc 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,6 +1,4 @@ -import json import re -from collections import OrderedDict from datetime import datetime from dateutil.tz import gettz @@ -14,17 +12,20 @@ from src.system_var.environments import CONVERT_TZ class ConvertStrategyFactory: def __init__(self) -> None: self.__none_value_convert_strategy = NoneValueConvertStrategy() + self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__int_convert_strategy = IntConvertStrategy() self.__string_convert_strategy = StringConvertStrategy() - self.__dict_convert_strategy = DictConvertStrategy() def create(self, value): if value is None: convert_strategy = self.__none_value_convert_strategy + elif type(value) == float: + convert_strategy = self.__float_convert_strategy + elif type(value) == bool: convert_strategy = self.__boolean_convert_strategy @@ -34,8 +35,6 @@ class ConvertStrategyFactory: elif type(value) == int: convert_strategy = self.__int_convert_strategy - elif type(value) == dict or type(value) == OrderedDict: - convert_strategy = self.__dict_convert_strategy else: convert_strategy = self.__string_convert_strategy @@ -61,6 +60,12 @@ class DatetimeConvertStrategy: return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) +class FloatConvertStrategy: + def convert_value(self, convert_value: str) -> int: + """float型をint型に変換する処理""" + return int(convert_value) + + class IntConvertStrategy: def convert_value(self, convert_value: int): """int型を変換せずに返す処理""" @@ -73,9 +78,3 @@ class StringConvertStrategy: """string型を変換せずに返す処理""" # ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す return convert_value - - -class DictConvertStrategy: - def convert_value(self, convert_value: dict): - """dict型の項目を文字列に変換して返す処理""" - return json.dumps(convert_value, ensure_ascii=False) diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index f14e85b8..d546c7b8 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -34,9 +34,9 @@ def prepare_data_fetch_process(): try: # ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する - object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' - logger.info( - f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') + object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' + logger.debug( + f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') config_bucket = ConfigBucket() object_info_file_str = config_bucket.get_object_info_file() diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index c457386f..297cfb2b 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -9,8 +9,8 @@ LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) # CRMのレコード件数取得処理のタイムアウト秒数 CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) -# CRMのレコード件数取得処理の最大リトライ試行回数(処理の実施総回数) -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 4)) +# CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 @@ -19,8 +19,8 @@ CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_R CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50)) # CRMのレコード取得処理のタイムアウト秒数 CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300)) -# CRMのレコード取得処理の最大リトライ試行回数(処理の実施総回数) -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 4)) +# CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード取得処理のリトライ時の初回待ち秒数 CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最小待ち秒数 diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 129b3980..0e268fc4 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,9 +1,7 @@ -from collections import OrderedDict - from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, DatetimeConvertStrategy, - DictConvertStrategy, + FloatConvertStrategy, IntConvertStrategy, NoneValueConvertStrategy, StringConvertStrategy) @@ -31,7 +29,7 @@ class TestConvertStrategyFactory: def test_create_float(self): """ Cases: - 引数に指数表記を指定した場合、StringConvertStrategyインスタンスが返ってくること + 引数に指数表記を指定した場合、FloatConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -43,26 +41,7 @@ class TestConvertStrategyFactory: actual = sut.create(1.2345678E7) # Expects - # 変換しない - assert type(actual) == StringConvertStrategy - - def test_create_float_scale(self): - """ - Cases: - 引数に少数を指定した場合、StringConvertStrategyインスタンスが返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = ConvertStrategyFactory() - actual = sut.create(1.2345678) - - # Expects - # 変換しない - assert type(actual) == StringConvertStrategy + assert type(actual) == FloatConvertStrategy def test_create_bool_true(self): """ @@ -115,10 +94,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == DatetimeConvertStrategy - def test_create_str(self): + def test_create_other_str(self): """ Cases: - 引数にSalesforce日付型以外の文字列を指定した場合、StringConvertStrategyインスタンスが返ってくること + 引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -132,10 +111,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == StringConvertStrategy - def test_create_int(self): + def test_create_other_int(self): """ Cases: - 引数に整数を指定した場合、IntConvertStrategyインスタンスが返ってくること + 引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -149,40 +128,6 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == IntConvertStrategy - def test_create_dict(self): - """ - Cases: - 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = ConvertStrategyFactory() - actual = sut.create({'key': 'value'}) - - # Expects - assert type(actual) == DictConvertStrategy - - def test_create_ordered_dict_dict(self): - """ - Cases: - 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = ConvertStrategyFactory() - actual = sut.create(OrderedDict([('key', 'value')])) - - # Expects - assert type(actual) == DictConvertStrategy - class TestNoneValueConvertStrategy: @@ -261,6 +206,26 @@ class TestDatetimeConvertStrategy: assert actual == "2022-06-14 05:15:32" +class TestFloatConvertStrategy: + + def test_convert_value(self) -> int: + """ + Cases: + 引数に指数表記を指定した場合、整数で返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = FloatConvertStrategy() + actual = sut.convert_value(1.2345678E7) + + # Expects + assert actual == 12345678 + + class TestIntConvertStrategy: def test_convert_value(self): @@ -299,59 +264,3 @@ class TestStringConvertStrategy: # Expects assert actual == 'テストデータ' - - -class TestDictConvertStrategy: - - def test_convert_value_dict(self): - """ - Cases: - 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = DictConvertStrategy() - actual = sut.convert_value({'テストデータキー': 'テストデータバリュー'}) - - # Expects - assert actual == '{"テストデータキー": "テストデータバリュー"}' - - def test_convert_value_dict_in_line_break(self): - """ - Cases: - 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること(バリューに改行を含む) - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = DictConvertStrategy() - actual = sut.convert_value({'テストデータキー': 'テスト\nデータ\nバリュー'}) - - # Expects - assert actual == '{"テストデータキー": "テスト\\nデータ\\nバリュー"}' - - def test_convert_value_ordered_dict(self): - """ - Cases: - 引数に整列された辞書型のデータを指定した場合、JSONの文字列が返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = DictConvertStrategy() - actual = sut.convert_value(OrderedDict( - [('テストデータキー', 'テストデータバリュー')] - )) - - # Expects - assert actual == '{"テストデータキー": "テストデータバリュー"}' diff --git a/ecs/crm-datafetch/tests/converter/test_converter.py b/ecs/crm-datafetch/tests/converter/test_converter.py index da80982b..0300381d 100644 --- a/ecs/crm-datafetch/tests/converter/test_converter.py +++ b/ecs/crm-datafetch/tests/converter/test_converter.py @@ -71,7 +71,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 8), ('RowCause', 'テストのため2'), ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), - ('LastModifiedById', 2.23E+0), + ('LastModifiedById', 2.234567E+6), ('IsDeleted', True) ]), OrderedDict([ @@ -85,7 +85,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 12), ('RowCause', 'テストのため3'), ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), - ('LastModifiedById', 3.234567), + ('LastModifiedById', 3.234567E+6), ('IsDeleted', False) ]) ] @@ -100,9 +100,9 @@ class TestCSVStringConverter: # Expects expected_value = '''\ "Id","AccountId","UserOrGroupId","AccountAccessLevel","OpportunityAccessLevel","CaseAccessLevel","ContactAccessLevel","RowCause","LastModifiedDate","LastModifiedById","IsDeleted"\r\n\ - "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567.0","0"\r\n\ - "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2.23","1"\r\n\ - "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3.234567","0"\r\n\ + "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567","0"\r\n\ + "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2234567","1"\r\n\ + "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\ ''' # expected_valueのインデントが半角スペースと認識されてしまうため、`textwrap.dedent`にて補正 diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index 64530071..b815e772 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -9,8 +9,6 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように 変更してしまった場合は各SOQLの取得日付とデータを修正してください """ -from typing import OrderedDict - import pytest from requests.exceptions import ConnectTimeout, ReadTimeout from src.config.objects import LastFetchDatetime, TargetObject @@ -35,8 +33,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-23T01:56:39.000Z AND - SystemModstamp <= 2022-08-24T00:00:00.000Z + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z """ sut = SalesforceApiClient() @@ -65,7 +63,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual >= 0 - def test_fetch_sf_count_by_soql_builder_system_modstamp_to_ge(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -81,8 +79,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T02:38:59.000Z', - 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', + 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -108,7 +106,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual == 1 - def test_fetch_sf_count_by_soql_builder_system_modstamp_to_lt(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -124,8 +122,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', - 'last_fetch_datetime_to': '2022-08-23T02:39:01.000Z', + 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -162,7 +160,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得件数が17になる + 取得件数が16になる """ execute_datetime = ExecuteDateTime() @@ -192,7 +190,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) - assert actual == 17 + assert actual == 16 def test_fetch_sf_data_one_record(self): """ @@ -219,8 +217,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-23T01:56:39.000Z AND - SystemModstamp <= 2022-08-24T00:00:00.000Z + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z """ sut = SalesforceApiClient() @@ -286,7 +284,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_data(soql) assert len(actual) >= 0 - def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -302,8 +300,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T03:48:39.000Z', - 'last_fetch_datetime_to': '2022-08-23T03:48:40.000Z', + 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -355,7 +353,7 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect - def test_fetch_sf_data_by_soql_builder_system_modstamp_to_gt(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -363,16 +361,16 @@ class TestSalesforceApiClient: - SystemModStampのToが指定日付未満のレコードは取得できないこと Arranges: - SalesforceのAccountオブジェクトに、レコードを作成する - - LastFetchDatetimeのFromがSystemModstampより小さくなるように指定する（UTC指定） - - LastFetchDatetimeのToがSystemModstampより大きくなるように指定する（UTC指定） + - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） Expects: 取得できたオブジェクト1件が期待値どおりであること """ execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T03:42:24.000Z', - 'last_fetch_datetime_to': '2022-08-23T03:42:26.000Z', + 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -424,69 +422,6 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect - def test_fetch_sf_data_by_soql_builder_address_item_check(self): - """ - Cases: - - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること - - できること - Arranges: - - SalesforceのAccountオブジェクトに、住所項目を持つレコードを作成する - - 住所項目を持つレコードだけが取れるよう日付を設定する - Expects: - 取得できたオブジェクト件数が1件になる - 住所項目(BillingAddress)が想定通りの値になっていること - """ - - execute_datetime = ExecuteDateTime() - last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', - 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', - }, execute_datetime) - target_object = TargetObject({ - 'object_name': 'Account', - 'columns': [ - 'Id', - 'Name', - 'SystemModstamp', - 'LastModifiedDate', - 'BillingStreet', - 'BillingCity', - 'BillingState', - 'BillingPostalCode', - 'BillingCountry', - 'BillingLatitude', - 'BillingLongitude', - 'BillingGeocodeAccuracy', - 'BillingAddress', - ] - }, execute_datetime) - soql_builder = SOQLBuilder(target_object, last_fetch_datetime) - soql = soql_builder.create_fetch_soql() - sut = SalesforceApiClient() - - actual = sut.fetch_sf_data(soql) - assert len(actual) == 1 - expect_address = OrderedDict([ - ("city", '〇〇区'), - ("country", "日本"), - ("geocodeAccuracy", None), - ("latitude", None), - ("longitude", None), - ("postalCode", '999-9999'), - ("state", '東京都'), - ("street", '△△-✗✗'), - ]) - - assert actual[0]['BillingAddress'] == expect_address - assert actual[0]['BillingCity'] == '〇〇区' - assert actual[0]['BillingCountry'] == '日本' - assert actual[0]['BillingGeocodeAccuracy'] is None - assert actual[0]['BillingLatitude'] is None - assert actual[0]['BillingLongitude'] is None - assert actual[0]['BillingPostalCode'] == '999-9999' - assert actual[0]['BillingState'] == '東京都' - assert actual[0]['BillingStreet'] == '△△-✗✗' - def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): """ Cases: @@ -498,7 +433,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得できたオブジェクト件数が17件になる + 取得できたオブジェクト件数が16件になる """ execute_datetime = ExecuteDateTime() @@ -513,7 +448,6 @@ class TestSalesforceApiClient: 'Name', 'SystemModstamp', 'LastModifiedDate', - 'BillingAddress', 'CustomItem1__c', 'CustomItem2__c', 'CustomItem3__c', @@ -529,7 +463,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_data(soql) - assert len(actual) == 17 + assert len(actual) == 16 # 内容の確認は別のケースで行っているため省略 def test_raise_create_instance_cause_auth_failed(self, monkeypatch): diff --git a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py index 4e48c419..9ad35d52 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py @@ -44,8 +44,7 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('Name', 'テスト取引先1') + ('IsDeleted', False) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -55,8 +54,7 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('Name', 'テスト取引先2') + ('IsDeleted', False) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -66,8 +64,7 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('Name', 'テスト取引先3') + ('IsDeleted', False) ]), ] @@ -99,7 +96,7 @@ class TestBackupCrmDataProcess: actual = s3_client.get_object( Bucket=bucket_name, Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json') - assert actual['Body'].read().decode('utf-8') == json.dumps(response_json, ensure_ascii=False) + assert actual['Body'].read().decode('utf-8') == json.dumps(response_json) # ログの確認 assert generate_log_message_tuple( diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py index bf439dac..49d960c0 100644 --- a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -34,17 +34,7 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', True), - ('PersonMailingAddress', OrderedDict([ - ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 - ('PersonMailingCity', 'New york city'), - ('PersonMailingState', 'Ohaio'), - ('PersonMailingPostalCode', '999-9999'), - ('PersonMailingCountry', 'US'), - ('PersonMailingLatitude', 50.1234567), - ('PersonMailingLongitude', 103.1234567), - ('PersonMailingGeocodeAccuracy', 'Address'), - ])), + ('IsDeleted', True) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -52,19 +42,9 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST002'), ('AccountNumber', 'test002'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.23E+0), + ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('PersonMailingAddress', OrderedDict([ - ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 - ('PersonMailingCity', 'New york city'), - ('PersonMailingState', 'Ohaio'), - ('PersonMailingPostalCode', '999-9999'), - ('PersonMailingCountry', 'US'), - ('PersonMailingLatitude', 50.1234567), - ('PersonMailingLongitude', 103.1234567), - ('PersonMailingGeocodeAccuracy', 'Address'), - ])), + ('IsDeleted', False) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -72,19 +52,9 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST003'), ('AccountNumber', 'test003'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567), + ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('PersonMailingAddress', OrderedDict([ - ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 - ('PersonMailingCity', 'New york city'), - ('PersonMailingState', 'Ohaio'), - ('PersonMailingPostalCode', '999-9999'), - ('PersonMailingCountry', 'US'), - ('PersonMailingLatitude', 50.1234567), - ('PersonMailingLongitude', 103.1234567), - ('PersonMailingGeocodeAccuracy', 'Address'), - ])), + ('IsDeleted', False) ]), ] @@ -96,8 +66,7 @@ class TestConvertCrmCsvDataProcess: 'LastModifiedDate', 'LastModifiedById', 'SystemModstamp', - 'IsDeleted', - 'PersonMailingAddress' + 'IsDeleted' ] } @@ -110,10 +79,10 @@ class TestConvertCrmCsvDataProcess: # Assert expect_csv_string = """\ - "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted","PersonMailingAddress"\r\n\ - "TEST001","test001","2022-06-01 09:00:00","1234567.0","2022-06-01 09:00:00","1","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ - "TEST002","test002","2022-06-01 09:00:00","1.23","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ - "TEST003","test003","2022-06-01 09:00:00","1.234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ """ # 返り値の期待値チェック assert isinstance(actual_csv_string, str), 'CSV文字列が返却される' diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py index 6b17477f..66f5577e 100644 --- a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -217,7 +217,7 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を4に設定する + - データ件数取得処理の最大リトライ試行回数を3に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - データ件数取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -225,7 +225,7 @@ class TestFetchCrmDataProcess: - データ件数取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -236,8 +236,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3, 4] - mock_counter_inst.increment.side_effect = [2, 3, 4, 5] + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -271,15 +271,15 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を4に設定する + - データ件数取得処理の最大リトライ試行回数を3に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - データ件数取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする + - データ件数取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データ件数取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -290,13 +290,13 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3, 4] - mock_counter_inst.increment.side_effect = [2, 3, 4, 5] + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value - mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception, exception] + mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception] mock_client_inst.fetch_sf_data.return_value = common_expect # Act with pytest.raises(SalesforceAPIException) as e: @@ -304,16 +304,16 @@ class TestFetchCrmDataProcess: # Assert - # 取得は4回行われる - assert mock_counter_inst.describe.call_count == 4 - # 足し込みは3回のみ - assert mock_counter_inst.increment.call_count == 3 + # 取得は3回行われる + assert mock_counter_inst.describe.call_count == 3 + # 足し込みは2回のみ + assert mock_counter_inst.increment.call_count == 2 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 3 + assert called_log_counts == 2 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-01' @@ -333,7 +333,7 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を4に設定する + - レコード取得処理の最大リトライ試行回数を3に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - レコード取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -341,7 +341,7 @@ class TestFetchCrmDataProcess: - データレコード取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -352,8 +352,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3, 4] - mock_counter_inst.increment.side_effect = [2, 3, 4, 5] + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -387,15 +387,15 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を4に設定する + - レコード取得処理の最大リトライ試行回数を3に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - レコード取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする + - レコード取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データレコード取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 4) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 3) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_INTERVAL', 1) @@ -406,30 +406,30 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3, 4] - mock_counter_inst.increment.side_effect = [2, 3, 4, 5] + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value mock_client_inst.fetch_sf_count.return_value = 1 - mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception, exception] + mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception] # Act with pytest.raises(SalesforceAPIException) as e: fetch_crm_data_process(common_target_object, common_last_fetch_datetime) # Assert - # 取得は4回行われる - assert mock_counter_inst.describe.call_count == 4 - # 足し込みは3回のみ - assert mock_counter_inst.increment.call_count == 3 + # 取得は3回行われる + assert mock_counter_inst.describe.call_count == 3 + # 足し込みは2回のみ + assert mock_counter_inst.increment.call_count == 2 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 3 + assert called_log_counts == 2 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-02' diff --git a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py index 8eb3d476..9471946f 100644 --- a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py @@ -7,9 +7,6 @@ from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.prepare_data_fetch_process import prepare_data_fetch_process from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ -from src.system_var.environments import (CRM_CONFIG_BUCKET, - OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER) from .test_utils.log_message import generate_log_message_tuple @@ -83,8 +80,6 @@ class TestPrepareDataFetchProcess: # ログの確認 assert generate_log_message_tuple(log_message='I-PRE-01 データ取得準備処理を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples - assert generate_log_message_tuple( - log_message=f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}]') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples def test_call_depended_modules(self): diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index e2be6670..e8c3b6e4 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -73,6 +73,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): object_info_list = get_object_config_list('object_info') for object_info in object_info_list: json_file = read_json(object_info) + json_file = to_upload_json(json_file) upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') object_info_files.append(json_file) last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') @@ -104,7 +105,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # Act logger = logging.getLogger() - logger.setLevel(logging.DEBUG) + logger.setLevel(logging.INFO) logger.info(f'##########################') logger.info(f'# 差分データ取得処理:実行開始 #') logger.info(f'##########################') @@ -117,7 +118,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # ループ前のログ確認 assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages - assert_prepare_process_log(log_messages, now, path.basename(object_info_list[0])) + assert_prepare_process_log(log_messages, now) assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages # オブジェクト情報を取得する(diff) object_info_list = object_info_files[0] @@ -206,10 +207,9 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): """ -def assert_prepare_process_log(log_messages, now, object_info_list_filename): +def assert_prepare_process_log(log_messages, now): assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages - assert f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{object_info_list_filename}]' in log_messages assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages @@ -295,6 +295,21 @@ def upload_json(json_file, s3_client, bucket, folder): s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str) +def to_upload_json(json_file): + """Userオブジェクトの取得できないプロパティを取り除く + TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す + """ + for object_info in json_file['objects']: + if object_info['object_name'] != 'User': + continue + columns: list = object_info['columns'] + columns.remove('LastPasswordChangeDate') + columns.remove('NumberOfFailedLogins') + columns.remove('UserPreferencesNativeEmailClient') + + return json_file + + def set_environment(monkeypatch): # 環境変数を設定(CRMの認証情報は別途設定しておくこと) monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) @@ -309,4 +324,3 @@ def set_environment(monkeypatch): monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) - monkeypatch.setattr('src.prepare_data_fetch_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index dda641e2..a5f8da67 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1645,11 +1645,13 @@ "DelegatedApproverId", "ManagerId", "LastLoginDate", + "LastPasswordChangeDate", "CreatedDate", "CreatedById", "LastModifiedDate", "LastModifiedById", "SystemModstamp", + "NumberOfFailedLogins", "OfflineTrialExpirationDate", "OfflinePdaTrialExpirationDate", "UserPermissionsMarketingUser", @@ -1723,6 +1725,7 @@ "UserPreferencesSRHOverrideActivities", "UserPreferencesNewLightningReportRunPageEnabled", "UserPreferencesReverseOpenActivitiesView", + "UserPreferencesNativeEmailClient", "UserPreferencesHideBrowseProductRedirectConfirmation", "UserPreferencesHideOnlineSalesAppWelcomeMat", "ContactId", diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index 3ae4d809..580bf564 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -1,14 +1,13 @@ LOG_LEVEL=INFO CRM_AUTH_TIMEOUT=100 CRM_GET_RECORD_COUNT_TIMEOUT=300 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=4 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3 CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL=50 CRM_FETCH_RECORD_TIMEOUT=300 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=4 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 CRM_FETCH_RECORD_RETRY_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 -CONVERT_TZ=Asia/Tokyo -OBJECT_INFO_FILENAME=crm_object_list_diff.json \ No newline at end of file +CONVERT_TZ='Asia/Tokyo' \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt index 813f86ee..d5d6c0dc 100644 --- a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt +++ b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -32 -Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c,MSJ_Registration__c -id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c,msj_registration__c +31 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c src02.crm_event_attendee_vod__c org02.crm_event_attendee_vod__c diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt index 727d82e0..a0d5c519 100644 --- a/s3/data/crm/settings/CRM_User.txt +++ b/s3/data/crm/settings/CRM_User.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -200 -Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c -id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c +203 +Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,LastPasswordChangeDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,NumberOfFailedLogins,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesNativeEmailClient,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c +id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,last_password_change_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,number_of_failed_logins,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_native_email_client,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c src02.crm_user org02.crm_user