From 9af0d9dd5cf895c16bf8c8600b36101fa4622645 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 4 Aug 2022 16:30:56 +0900 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20SOQLBuilder=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/__init__.py | 0 .../tests/salesforce/test_soql_builder.py | 104 ++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 ecs/crm-datafetch/tests/salesforce/__init__.py create mode 100644 ecs/crm-datafetch/tests/salesforce/test_soql_builder.py diff --git a/ecs/crm-datafetch/tests/salesforce/__init__.py b/ecs/crm-datafetch/tests/salesforce/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py new file mode 100644 index 00000000..9a6f9a45 --- /dev/null +++ b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py @@ -0,0 +1,104 @@ + +from src.config.objects import ExecuteDateTime, LastFetchDatetime, TargetObject +from src.salesforce.soql_builder import SOQLBuilder + + +class TestSOQLBuilder: + + def test_create_count_soql(self): + test_target_object_json = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + } + + test_last_fetch_datetime_json = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2000-01-01T00:00:00.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(test_target_object_json, execute_datetime) + test_last_fetch_datetime = LastFetchDatetime(test_last_fetch_datetime_json, execute_datetime) + + sut = SOQLBuilder(target_object, test_last_fetch_datetime) + actual = sut.create_count_soql() + + expect = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + + assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') + + def test_create_fetch_soql(self): + test_target_object_json = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + } + + test_last_fetch_datetime_json = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2000-01-01T00:00:00.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(test_target_object_json, execute_datetime) + test_last_fetch_datetime = LastFetchDatetime(test_last_fetch_datetime_json, execute_datetime) + + sut = SOQLBuilder(target_object, test_last_fetch_datetime) + actual = sut.create_fetch_soql() + + expect = """SELECT + Id, + Name, + SystemModstamp, + LastModifiedDate, + CustomItem1__c, + CustomItem2__c, + CustomItem3__c, + CustomItem4__c, + CustomItem5__c, + CustomItem6__c, + CustomItem7__c, + CustomItem8__c + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + + print('actual', actual) + print('expect', expect) + + assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') From e3cd0dbb101a4ec1749e737bcfa61278704af63c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 08:14:09 +0900 Subject: [PATCH 2/9] =?UTF-8?q?feat:Salesforce=E9=96=A2=E9=80=A3=E3=81=AE?= =?UTF-8?q?=E6=9C=80=E4=BD=8E=E9=99=90=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/salesforce/salesforce_api.py | 8 +-- .../tests/salesforce/test_salesfoce.py | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 ecs/crm-datafetch/tests/salesforce/test_salesfoce.py diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index 4e4bb267..edd9aeb3 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -13,16 +13,16 @@ class SalesforceApiClient(): domain=CRM_AUTH_DOMAIN ) - def query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + def __query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): return self.__sf.query(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) - def query_all(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + def __query_all(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): return self.__sf.query_all(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) def fetch_sf_count(self, soql: str): - count_res = self.query(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) + count_res = self.__query(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) return count_res.get('records')[0].get('expr0') def fetch_sf_data(self, soql: str): - data_res = self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + data_res = self.__query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) return data_res.get('records') diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py new file mode 100644 index 00000000..413fae92 --- /dev/null +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -0,0 +1,52 @@ +import pytest +from src.salesforce.salesforce_api import SalesforceApiClient + + +class TestSalesforceApiClient: + + def test_fetch_sf_count(self): + + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-04T16:36:00.000Z + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual >= 1 + + def test_fetch_sf_data_one_record(self): + + soql = """SELECT + Id, + Name, + SystemModstamp, + LastModifiedDate, + CustomItem1__c, + CustomItem2__c, + CustomItem3__c, + CustomItem4__c, + CustomItem5__c, + CustomItem6__c, + CustomItem7__c, + CustomItem8__c + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-04T16:36:00.000Z + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + print(actual) + assert len(actual) >= 1 + + def test_raise_create_instance_cause_auth_failed(self, monkeypatch): + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_USER_NAME', 'invalid_username') + with pytest.raises(Exception): + SalesforceApiClient() From 8373ccc456fdf30c60c0a9df605e0af7cf82ea62 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 20:08:03 +0900 Subject: [PATCH 3/9] =?UTF-8?q?feat:=E3=82=B1=E3=83=BC=E3=82=B9=E3=81=AE?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=83=BB=E4=BF=AE=E6=AD=A3=E3=80=82=E3=83=89?= =?UTF-8?q?=E3=82=AD=E3=83=A5=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 51 ++++++++++++++++- .../tests/salesforce/test_soql_builder.py | 56 +++++++++++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 413fae92..4b3f8ae1 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -5,7 +5,14 @@ from src.salesforce.salesforce_api import SalesforceApiClient class TestSalesforceApiClient: def test_fetch_sf_count(self): - + """ + Cases: + Salesforceからオブジェクトの件数が取得できること + Arranges: + SalesforceのAccountオブジェクトに、レコードを作成する(手作業、コード上では行わない) + Expects: + 取得件数が1件以上になる + """ soql = """SELECT COUNT(Id) FROM @@ -20,7 +27,14 @@ class TestSalesforceApiClient: assert actual >= 1 def test_fetch_sf_data_one_record(self): - + """ + Cases: + Salesforceからオブジェクトが取得できること + Arranges: + SalesforceのAccountオブジェクトに、レコードを作成する(手作業、コード上では行わない) + Expects: + オブジェクトが取得でき、期待値と一致していること + """ soql = """SELECT Id, Name, @@ -42,11 +56,42 @@ class TestSalesforceApiClient: """ sut = SalesforceApiClient() + expect = { + 'Name': 'テスト取引先名1', + 'CustomItem1__c': 'テスト', + 'CustomItem2__c': 1.0, + 'CustomItem3__c': True, + 'CustomItem4__c': '01:15:00.000Z', + 'CustomItem5__c': '1;2;3;4', + 'CustomItem6__c': '改行ありの\r\nテスト\r\n項目です', + 'CustomItem7__c': '2022-08-04', + 'CustomItem8__c': '2022-08-04T03:00:00.000+0000', + } actual = sut.fetch_sf_data(soql) - print(actual) assert len(actual) >= 1 + # Id, SystemModstamp, LastModifiedDateは自動生成なので、キーの有無だけ確認する + # Attributesも + assert 'Id' in actual[0].keys() + assert 'SystemModstamp' in actual[0].keys() + assert 'LastModifiedDate' in actual[0].keys() + assert 'attributes' in actual[0].keys() + + del actual[0]['Id'] + del actual[0]['SystemModstamp'] + del actual[0]['LastModifiedDate'] + del actual[0]['attributes'] + + assert dict(actual[0]) == expect def test_raise_create_instance_cause_auth_failed(self, monkeypatch): + """ + Cases: + 存在しないユーザを指定した場合、エラーが発生すること + Arranges: + なし + Expects: + ユーザ認証でエラーが発生すること + """ monkeypatch.setattr('src.salesforce.salesforce_api.CRM_USER_NAME', 'invalid_username') with pytest.raises(Exception): SalesforceApiClient() diff --git a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py index 9a6f9a45..da42d486 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py +++ b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py @@ -6,6 +6,14 @@ from src.salesforce.soql_builder import SOQLBuilder class TestSOQLBuilder: def test_create_count_soql(self): + """ + Cases: + 件数取得のSOQLが生成できること + Arranges: + SOQL生成用のパラメータを用意する + Expects: + 期待値通りのSOQLが生成されること + """ test_target_object_json = { 'object_name': 'Account', 'columns': [ @@ -48,6 +56,14 @@ class TestSOQLBuilder: assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') def test_create_fetch_soql(self): + """ + Cases: + データ取得用のSOQLが生成できること + Arranges: + SOQL生成用のパラメータを用意する + Expects: + 期待値通りのSOQLが生成されること + """ test_target_object_json = { 'object_name': 'Account', 'columns': [ @@ -102,3 +118,43 @@ class TestSOQLBuilder: print('expect', expect) assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') + + def test_create_fetch_soql_no_columns(self): + """ + Cases: + データ取得用のSOQLがカラムがない状態で生成されること + Arranges: + SOQL生成用のパラメータを用意する + Expects: + 取得対象のカラムがないSOQLが生成されること + """ + test_target_object_json = { + 'object_name': 'Account', + 'columns': [] + } + + test_last_fetch_datetime_json = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2000-01-01T00:00:00.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(test_target_object_json, execute_datetime) + test_last_fetch_datetime = LastFetchDatetime(test_last_fetch_datetime_json, execute_datetime) + + sut = SOQLBuilder(target_object, test_last_fetch_datetime) + actual = sut.create_fetch_soql() + + # TargetObjectのバリデーションで、columnsが空の場合はエラーになるため、本来は発生しない + expect = """SELECT + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + + print('actual', actual) + print('expect', expect) + + assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') From c479c3bca316d2431e35ab68b671953f97c1dbd5 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 21:07:36 +0900 Subject: [PATCH 4/9] =?UTF-8?q?feat:=20SOQLBuilder=E3=81=8B=E3=82=89?= =?UTF-8?q?=E7=94=9F=E6=88=90=E3=81=97=E3=81=9FSOQL=E3=81=A7=E3=81=AE?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 244 +++++++++++++++++- 1 file changed, 242 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 4b3f8ae1..3b58b49d 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -1,5 +1,19 @@ +""" +！！！！注意！！！！ +当テストコードはSalesforceのレコードに依存しています。 +Accountオブジェクトの下記SFIDのレコードはいじらないようにしてください + - 0015i00000LNywwAAD(テスト取引先1) + - 0015i00000LOClSAAX(テスト取引先2) + - 0015i00000LOCmGAAX(テスト取引先3) + +変更してしまった場合は各SOQLの取得日付とデータを修正してください +""" + import pytest +from src.config.objects import LastFetchDatetime, TargetObject from src.salesforce.salesforce_api import SalesforceApiClient +from src.salesforce.soql_builder import SOQLBuilder +from src.util.execute_datetime import ExecuteDateTime class TestSalesforceApiClient: @@ -19,13 +33,99 @@ class TestSalesforceApiClient: Account WHERE SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-04T16:36:00.000Z + SystemModstamp <= 2022-08-06T00:00:00.000Z """ sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) assert actual >= 1 + def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること + - SystemModStampがFrom指定日付未満のレコードは取得できないこと + - SystemModStampのToが指定日付以上のレコードは取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromがSystemModstamp未満になるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstampピッタリになるように指定する（UTC指定） + Expects: + 取得件数が1になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_count_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual == 1 + + def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること + - SystemModStampのFromが指定日付のより大きいレコードは取得できること + - SystemModStampのToが指定日付未満のレコードは取得できないこと + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） + Expects: + 取得件数が1になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_count_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual == 1 + def test_fetch_sf_data_one_record(self): """ Cases: @@ -52,7 +152,7 @@ class TestSalesforceApiClient: Account WHERE SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-04T16:36:00.000Z + SystemModstamp <= 2022-08-06T00:00:00.000Z """ sut = SalesforceApiClient() @@ -69,6 +169,8 @@ class TestSalesforceApiClient: } actual = sut.fetch_sf_data(soql) assert len(actual) >= 1 + # 複数取れるが、アサーション対象は1つだけ + actual = [record for record in actual if record['Name'] == 'テスト取引先名1'] # Id, SystemModstamp, LastModifiedDateは自動生成なので、キーの有無だけ確認する # Attributesも assert 'Id' in actual[0].keys() @@ -83,6 +185,144 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - SystemModStampがFrom指定日付未満のレコードは取得できないこと + - SystemModStampのToが指定日付以上のレコードは取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromがSystemModstamp未満になるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstampピッタリになるように指定する（UTC指定） + Expects: + 取得できたオブジェクト1件が期待値どおりであること + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 1 + + # Id, SystemModstamp, LastModifiedDateは自動生成なので、キーの有無だけ確認する + # Attributesも + assert 'Id' in actual[0].keys() + assert 'SystemModstamp' in actual[0].keys() + assert 'LastModifiedDate' in actual[0].keys() + assert 'attributes' in actual[0].keys() + + del actual[0]['Id'] + del actual[0]['SystemModstamp'] + del actual[0]['LastModifiedDate'] + del actual[0]['attributes'] + + expect = { + 'Name': 'テスト取引先名3', + 'CustomItem1__c': 'テスト3', + 'CustomItem2__c': 3.0, + 'CustomItem3__c': True, + 'CustomItem4__c': '00:45:00.000Z', + 'CustomItem5__c': '2;3', + 'CustomItem6__c': 'かいぎょ', + 'CustomItem7__c': '2022-08-06', + 'CustomItem8__c': '2022-08-06T00:00:00.000+0000', + } + + assert dict(actual[0]) == expect + + def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - SystemModStampのFromが指定日付のより大きいレコードは取得できること + - SystemModStampのToが指定日付未満のレコードは取得できないこと + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） + Expects: + 取が1になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 1 + + # Id, SystemModstamp, LastModifiedDateは自動生成なので、キーの有無だけ確認する + # Attributesも + assert 'Id' in actual[0].keys() + assert 'SystemModstamp' in actual[0].keys() + assert 'LastModifiedDate' in actual[0].keys() + assert 'attributes' in actual[0].keys() + + del actual[0]['Id'] + del actual[0]['SystemModstamp'] + del actual[0]['LastModifiedDate'] + del actual[0]['attributes'] + + expect = { + 'Name': 'テスト取引先名2', + 'CustomItem1__c': 'テスト2', + 'CustomItem2__c': 2.0, + 'CustomItem3__c': False, + 'CustomItem4__c': '00:30:00.000Z', + 'CustomItem5__c': '1;4', + 'CustomItem6__c': '改行あり項目', + 'CustomItem7__c': '2022-08-05', + 'CustomItem8__c': '2022-08-04T23:30:00.000+0000', + } + + assert dict(actual[0]) == expect + def test_raise_create_instance_cause_auth_failed(self, monkeypatch): """ Cases: From 7a029f4bb7c26ee44f7a4776c87b443ac33854b3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 07:30:59 +0900 Subject: [PATCH 5/9] =?UTF-8?q?feat;=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=8F=8D=E6=98=A0=20=E5=89=8D?= =?UTF-8?q?=E7=AF=84=E5=9B=B2=E5=8F=96=E5=BE=97=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0=20https://nds-tyo.backlog.?= =?UTF-8?q?com/git/NEWDWH2021/newsdwh2021/pullRequests/59#comment-1313618?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 90 ++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 3b58b49d..623ead4f 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -10,6 +10,7 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように """ import pytest +from requests.exceptions import ConnectTimeout, ReadTimeout from src.config.objects import LastFetchDatetime, TargetObject from src.salesforce.salesforce_api import SalesforceApiClient from src.salesforce.soql_builder import SOQLBuilder @@ -126,6 +127,49 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual == 1 + def test_fetch_sf_count_by_soql_builder_system_modstamp_all_range(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること + - SystemModStampのFromが2000年1月1日以降のレコードが取得できること + - SystemModStampのToが2100年12月31日未満のレコードが取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromに2000年1月1日を指定する + - LastFetchDatetimeのToに2100年12月31日を指定する + Expects: + 取得件数が16になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_count_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual == 16 + def test_fetch_sf_data_one_record(self): """ Cases: @@ -265,7 +309,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） Expects: - 取が1になる + 取得できたオブジェクト1件が期待値どおりであること """ execute_datetime = ExecuteDateTime() @@ -323,6 +367,50 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - SystemModStampのFromが2000年1月1日以降のレコードが取得できること + - SystemModStampのToが2100年12月31日未満のレコードが取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromに2000年1月1日を指定する + - LastFetchDatetimeのToに2100年12月31日を指定する + Expects: + 取得できたオブジェクト件数が16件になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 16 + # 内容の確認は別のケースで行っているため省略 + def test_raise_create_instance_cause_auth_failed(self, monkeypatch): """ Cases: From 61d7275c2b3142f327c163c460f33375dc7a0266 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 07:43:08 +0900 Subject: [PATCH 6/9] =?UTF-8?q?feat:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=8F=8D=E6=98=A0=20=E3=82=BF?= =?UTF-8?q?=E3=82=A4=E3=83=A0=E3=82=A2=E3=82=A6=E3=83=88=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=83=BBSOQL=E4=B8=8D=E6=AD=A3=E3=81=AE=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=82=B1=E3=83=BC=E3=82=B9=E3=82=92=E8=BF=BD=E5=8A=A0=20https:?= =?UTF-8?q?//nds-https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh2021/pu?= =?UTF-8?q?llRequests/59#comment-1313618?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 118 +++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 623ead4f..aac7f109 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -416,10 +416,126 @@ class TestSalesforceApiClient: Cases: 存在しないユーザを指定した場合、エラーが発生すること Arranges: - なし + CRMのユーザ名を保持する環境変数に、存在しないユーザー名を指定する Expects: ユーザ認証でエラーが発生すること """ monkeypatch.setattr('src.salesforce.salesforce_api.CRM_USER_NAME', 'invalid_username') with pytest.raises(Exception): SalesforceApiClient() + + def test_raise_fetch_sf_count_auth_timeout(self, monkeypatch): + """ + Cases: + 認証タイムアウトが発生した場合、エラーが発生すること + Arranges: + 認証タイムアウト秒数を保持する環境変数に、0.0000000001を指定する + Expects: + コネクションタイムアウトエラーが発生すること + """ + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_AUTH_TIMEOUT', 0.0000000001) + sf = SalesforceApiClient() + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z + """ + with pytest.raises(ConnectTimeout): + sf.fetch_sf_count(soql) + + def test_raise_fetch_sf_count_read_timeout(self, monkeypatch): + """ + Cases: + 読み取りタイムアウトが発生した場合、エラーが発生すること + Arranges: + CRMの件数取得タイムアウト秒数を保持する環境変数に、0.0000000001を指定する + Expects: + 読み取りタイムアウトエラーが発生すること + """ + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_GET_RECORD_COUNT_TIMEOUT', 0.0000000001) + sf = SalesforceApiClient() + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z + """ + with pytest.raises(ReadTimeout): + sf.fetch_sf_count(soql) + + def test_raise_fetch_sf_count_invalid_soql(self, monkeypatch): + """ + Cases: + 不正なSOQLを指定した場合に、エラーが発生すること + Arranges: + 不正なSOQLを作成する + Expects: + エラーが発生すること + """ + sf = SalesforceApiClient() + soql = "SELECT" + with pytest.raises(Exception): + sf.fetch_sf_count(soql) + + def test_raise_fetch_sf_data_auth_timeout(self, monkeypatch): + """ + Cases: + 認証タイムアウトが発生した場合、エラーが発生すること + Arranges: + 認証タイムアウト秒数を保持する環境変数に、0.0000000001を指定する + Expects: + コネクションタイムアウトエラーが発生すること + """ + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_AUTH_TIMEOUT', 0.0000000001) + sf = SalesforceApiClient() + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z + """ + with pytest.raises(ConnectTimeout): + sf.fetch_sf_data(soql) + + def test_raise_fetch_sf_data_read_timeout(self, monkeypatch): + """ + Cases: + 読み取りタイムアウトが発生した場合、エラーが発生すること + Arranges: + CRMのデータ取得タイムアウト秒数を保持する環境変数に、0.0000000001を指定する + Expects: + 読み取りタイムアウトエラーが発生すること + """ + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_FETCH_RECORD_TIMEOUT', 0.0000000001) + sf = SalesforceApiClient() + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z + """ + with pytest.raises(ReadTimeout): + sf.fetch_sf_data(soql) + + def test_raise_fetch_sf_data_invalid_soql(self, monkeypatch): + """ + Cases: + 不正なSOQLを指定した場合に、エラーが発生すること + Arranges: + 不正なSOQLを作成する + Expects: + エラーが発生すること + """ + sf = SalesforceApiClient() + soql = "SELECT" + with pytest.raises(Exception): + sf.fetch_sf_count(soql) From 44b7256f3cef43e5ce79874d47cb811cf7176ec3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 11:08:05 +0900 Subject: [PATCH 7/9] =?UTF-8?q?feat:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C=20https://nds-tyo?= =?UTF-8?q?.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/59#comment?= =?UTF-8?q?-1319332=20https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh20?= =?UTF-8?q?21/pullRequests/59#comment-1319342?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index aac7f109..612c3679 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -41,6 +41,28 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual >= 1 + def test_fetch_sf_count_zero_record(self): + """ + Cases: + 取得範囲外の場合、Salesforceからオブジェクトの件数が取得できないこと + Arranges: + SalesforceのAccountオブジェクトに、レコードを作成する(手作業、コード上では行わない) + Expects: + 取得件数が0件になる + """ + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual >= 0 + def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): """ Cases: @@ -229,6 +251,28 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_zero_record(self): + """ + Cases: + 取得範囲外の場合、Salesforceからオブジェクトが取得できないこと + Arranges: + SalesforceのAccountオブジェクトに、レコードを作成する(手作業、コード上では行わない) + Expects: + 取得件数が0件になる + """ + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) >= 0 + def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): """ Cases: @@ -538,4 +582,4 @@ class TestSalesforceApiClient: sf = SalesforceApiClient() soql = "SELECT" with pytest.raises(Exception): - sf.fetch_sf_count(soql) + sf.fetch_sf_data(soql) From 2f230e2f15088fdda96fb3bba45e3b2a2f2a7a21 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 08:36:43 +0900 Subject: [PATCH 8/9] =?UTF-8?q?fix:=20=E3=83=87=E3=83=BC=E3=82=BF=E5=8F=96?= =?UTF-8?q?=E5=BE=97=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=81=AESOQL?= =?UTF-8?q?=E6=96=87=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 612c3679..b815e772 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -261,7 +261,18 @@ class TestSalesforceApiClient: 取得件数が0件になる """ soql = """SELECT - COUNT(Id) + Id, + Name, + SystemModstamp, + LastModifiedDate, + CustomItem1__c, + CustomItem2__c, + CustomItem3__c, + CustomItem4__c, + CustomItem5__c, + CustomItem6__c, + CustomItem7__c, + CustomItem8__c FROM Account WHERE From f4b03c0c4541fccbe906737998d0a16122b0381f Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 10:59:47 +0900 Subject: [PATCH 9/9] =?UTF-8?q?refactor:=20textwrap.dedent=E3=82=92?= =?UTF-8?q?=E4=BD=BF=E3=81=A3=E3=81=A6=E4=BD=99=E5=88=86=E3=81=AA=E5=8D=8A?= =?UTF-8?q?=E8=A7=92=E3=82=B9=E3=83=9A=E3=83=BC=E3=82=B9=E3=82=92=E6=B6=88?= =?UTF-8?q?=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/salesforce/soql_builder.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ecs/crm-datafetch/src/salesforce/soql_builder.py b/ecs/crm-datafetch/src/salesforce/soql_builder.py index a84744a8..42d4ecea 100644 --- a/ecs/crm-datafetch/src/salesforce/soql_builder.py +++ b/ecs/crm-datafetch/src/salesforce/soql_builder.py @@ -1,12 +1,15 @@ -from src.config.objects import TargetObject, LastFetchDatetime +import textwrap + +from src.config.objects import LastFetchDatetime, TargetObject class SOQLBuilder: def __init__(self, target_object: TargetObject, last_fetch_datetime: LastFetchDatetime) -> None: - self.__SELECT_SOQL = """SELECT {column_or_expression} FROM {object_name} - WHERE {datetime_column} > {last_fetch_datetime_from} - AND {datetime_column} <= {last_fetch_datetime_to} - """ + self.__SELECT_SOQL = textwrap.dedent("""\ + SELECT {column_or_expression} FROM {object_name} + WHERE {datetime_column} > {last_fetch_datetime_from} + AND {datetime_column} <= {last_fetch_datetime_to} + """) self.__target_object = target_object self.__last_fetch_datetime = last_fetch_datetime