From 1151c015ca85aa462846ec46c78ac6cc5f1d3e8b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20=E3=83=89=E3=82=AD=E3=83=A5=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=81=8B?= =?UTF-8?q?=E3=82=89=E3=83=AC=E3=83=9D=E3=83=BC=E3=83=88=E3=82=92=E7=94=9F?= =?UTF-8?q?=E6=88=90=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + .../.vscode/python.code-snippets | 17 ++++ ecs/crm-datafetch/Pipfile | 2 + ecs/crm-datafetch/Pipfile.lock | 32 +++++-- ecs/crm-datafetch/tests/aws/test_s3.py | 89 +++++++++++++++++-- ecs/crm-datafetch/tests/conftest.py | 41 +++++++++ ecs/crm-datafetch/tests/docstring_parser.py | 32 +++++++ 7 files changed, 203 insertions(+), 13 deletions(-) create mode 100644 ecs/crm-datafetch/.vscode/python.code-snippets create mode 100644 ecs/crm-datafetch/tests/docstring_parser.py diff --git a/.gitignore b/.gitignore index 88a052ff..16a48619 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* __pycache__/ .env **/.vscode/settings.json + +# python test .coverage +.report/ \ No newline at end of file diff --git a/ecs/crm-datafetch/.vscode/python.code-snippets b/ecs/crm-datafetch/.vscode/python.code-snippets new file mode 100644 index 00000000..9f01c623 --- /dev/null +++ b/ecs/crm-datafetch/.vscode/python.code-snippets @@ -0,0 +1,17 @@ +{ + "Generate Test docstring": { + "scope": "python", + "prefix": "\"\"\"\"\"\"", + "body": [ + "\"\"\"", + "Tests:", + " $1", + "Arranges:", + " $2", + "Expects:", + " $3", + "\"\"\"" + ], + "description": "Test docstring (User Snipets)" + } +} diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 3d7988e7..990beb4b 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -6,6 +6,7 @@ name = "pypi" [scripts] test = "pytest tests/" "test:cov" = "pytest --cov=src tests/" +"test:report" = "pytest --cov=src --html=.report/test_result.html tests/" [packages] boto3 = "*" @@ -17,6 +18,7 @@ autopep8 = "*" flake8 = "*" pytest = "*" pytest-cov = "*" +pytest-html = "*" moto = "*" [requires] diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index 7c13f0c9..cfd16ed2 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f1433a55f486f24bb14d6447713a0cdeeb704542a695103debd9514face495cc" + "sha256": "7006de596d6123ecd56760b584ab75430fa6bcfc0ecd3fdf49f08368ff53477d" }, "pipfile-spec": 6, "requires": { @@ -59,7 +59,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2022.6.15" }, "cffi": { @@ -136,7 +136,7 @@ "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2.1.0" }, "cryptography": { @@ -164,7 +164,7 @@ "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==37.0.4" }, "idna": { @@ -363,7 +363,7 @@ "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==4.1.0" } }, @@ -405,7 +405,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2022.6.15" }, "cffi": { @@ -482,7 +482,7 @@ "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2.1.0" }, "coverage": { @@ -560,7 +560,7 @@ "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==37.0.4" }, "flake8": { @@ -735,6 +735,22 @@ "index": "pypi", "version": "==3.0.0" }, + "pytest-html": { + "hashes": [ + "sha256:3ee1cf319c913d19fe53aeb0bc400e7b0bc2dbeb477553733db1dad12eb75ee3", + "sha256:b7f82f123936a3f4d2950bc993c2c1ca09ce262c9ae12f9ac763a2401380b455" + ], + "index": "pypi", + "version": "==3.1.1" + }, + "pytest-metadata": { + "hashes": [ + "sha256:39261ee0086f17649b180baf2a8633e1922a4c4b6fcc28a2de7d8127a82541bf", + "sha256:fcd2f416f15be295943527b3c8ba16a44ae5a7141939c90c3dc5ce9d167cf2a5" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.0.2" + }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index db97825a..b6016b35 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -16,27 +16,71 @@ def s3_test(s3_client, bucket_name): class TestS3Resource: def test_get_object(self, s3_test, s3_client, bucket_name): + """ + Cases: + S3からオブジェクトが取得できるか + Arranges: + - S3をモック化する + - 期待値となるファイルを配置する + Expects: + オブジェクトが取得でき、期待値と正しいこと + """ + # Arrange s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + + # ActAssert s3_resource = S3Resource(bucket_name) actual = s3_resource.get_object('hogehoge/test.txt') + + # Assert assert actual == 'aaaaaaaaaaaaaaa' def test_put_object(self, s3_test, s3_client, bucket_name): + """ + Cases: + S3にオブジェクトをPUTできるか + Arranges: + - S3をモック化する + Expects: + オブジェクトがPUTできること + """ s3_resource = S3Resource(bucket_name) + s3_resource.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') actual = s3_client.get_object(Bucket=bucket_name, Key='hogehoge/test.txt') + assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' def test_copy(self, s3_test, s3_client, bucket_name): + """ + Cases: + S3内のオブジェクトを別バケットにコピーできるか + Arranges: + - S3をモック化する + - 期待値となるファイルをコピー元バケットに配置する + Expects: + - コピーされたファイルが存在する + - コピーされたファイルの内容が期待値と一致する + """ for_copy_bucket = 'for-copy-bucket' s3_client.create_bucket(Bucket=for_copy_bucket) s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + s3_resource = S3Resource(bucket_name) s3_resource.copy(bucket_name, 'hogehoge/test.txt', for_copy_bucket, 'test.txt') + actual = s3_client.get_object(Bucket=for_copy_bucket, Key='test.txt') assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' def test_init_raise_no_provide_bucket_name(self): + """ + Cases: + バケット名を指定しない場合、例外となること + Arranges: + + Expects: + 例外が発生すること + """ with pytest.raises(Exception) as e: S3Resource() assert e.value.args[0] == "__init__() missing 1 required positional argument: 'bucket_name'" @@ -45,36 +89,71 @@ class TestS3Resource: class TestConfigBucket: def test_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + オブジェクト情報ファイルが取得できること + Arranges: + オブジェクト情報ファイルを配置する + Expects: + オブジェクト情報ファイルが文字列として取得でき、期待値と一致する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm') monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json') s3_client.put_object(Bucket=bucket_name, Key=f'crm/objects.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() - print('*' * 50, str(config_bucket), '*' * 50) actual = config_bucket.get_object_info_file() - print(actual) + assert actual == 'aaaaaaaaaaaaaaa' def test_get_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + オブジェクト最終更新日時ファイルが取得できること + Arranges: + オブジェクト最終更新日時ファイルを配置する + Expects: + オブジェクト最終更新日時ファイルが文字列として取得でき、期待値と一致する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') s3_client.put_object(Bucket=bucket_name, Key=f'crm/Object.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() - print('*' * 50, str(config_bucket), '*' * 50) actual = config_bucket.get_last_fetch_datetime_file('Object.json') - print(actual) + assert actual == 'aaaaaaaaaaaaaaa' def test_put_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + オブジェクト最終更新日時ファイルをPUTできること + Arranges: + + Expects: + オブジェクト最終更新日時ファイルが存在する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') + config_bucket = ConfigBucket() - print('*' * 50, str(config_bucket), '*' * 50) config_bucket.put_last_fetch_datetime_file('Object.json', 'aaaaaaaaaaaaaaa') + actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/Object.json') assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' def test_config_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること + Arranges: + + Expects: + 環境変数の設定ファイル配置バケット名と一致する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + config_bucket = ConfigBucket() + assert str(config_bucket) == bucket_name diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py index 1120ced6..e46746bb 100644 --- a/ecs/crm-datafetch/tests/conftest.py +++ b/ecs/crm-datafetch/tests/conftest.py @@ -1,8 +1,12 @@ import os +from datetime import datetime import boto3 import pytest from moto import mock_s3 +from py.xml import html # type: ignore + +from . import docstring_parser @pytest.fixture @@ -19,3 +23,40 @@ def s3_client(aws_credentials): with mock_s3(): conn = boto3.client("s3", region_name="us-east-1") yield conn + + +# 以下、レポート出力用の設定 + +def pytest_html_report_title(report): + # レポートタイトル + report.title = "CRMデータ連携 CRMデータ取得機能 単体機能テスト結果報告書" + + +# # def pytest_configure(config): +# # config._metadata["結果確認者"] = "" # Version情報を追加 + + +def pytest_html_results_table_header(cells): + del cells[2:] + cells.insert(3, html.th("Cases")) + cells.insert(4, html.th("Arranges")) + cells.insert(5, html.th("Expects")) + cells.append(html.th("Time", class_="sortable time", col="time")) + + +def pytest_html_results_table_row(report, cells): + del cells[2:] + cells.insert(3, html.td(html.pre(report.cases))) # 「テスト内容」をレポートに出力 + cells.insert(4, html.td(html.pre(report.arranges))) # 「期待結果」をレポートに出力 + cells.insert(5, html.td(html.pre(report.expects))) # 「期待結果」をレポートに出力 + cells.append(html.td(datetime.now(), class_="col-time")) # ついでに「時間」もレポートに出力 + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + docstring = docstring_parser.parse(str(item.function.__doc__)) + report.cases = docstring.get("Cases", '') # 「テスト内容」を`report`に追加 + report.arranges = docstring.get("Arranges", '') # 「準備作業」を`report`に追加 + report.expects = docstring.get("Expects", '') # 「期待結果」を`report`に追加 diff --git a/ecs/crm-datafetch/tests/docstring_parser.py b/ecs/crm-datafetch/tests/docstring_parser.py new file mode 100644 index 00000000..32ac1579 --- /dev/null +++ b/ecs/crm-datafetch/tests/docstring_parser.py @@ -0,0 +1,32 @@ +import re +from itertools import takewhile + +_section_rgx = re.compile(r"^\s*[a-zA-Z]+:\s*$") +_lspace_rgx = re.compile(r"^\s*") + + +def _parse_section(lines: list) -> list: + matches = map(lambda x: _section_rgx.match(x), lines) + indexes = [i for i, x in enumerate(matches) if x is not None] + return list(map(lambda x: (x, lines[x].strip()[: -1]), indexes)) + + +def _count_lspace(s: str) -> int: + rgx = _lspace_rgx.match(s) + if rgx is not None: + return rgx.end() + return 0 + + +def _parse_content(index: int, lines: list) -> str: + lspace = _count_lspace(lines[index]) + i = index + 1 + contents = takewhile(lambda x: _count_lspace(x) > lspace, lines[i:]) + return "\n".join(map(lambda x: x.strip(), contents)) + + +def parse(docstring: str) -> dict: + """🚧sloppy docstring parser🚧""" + lines = docstring.splitlines() + sections = _parse_section(lines) + return dict(map(lambda x: (x[1], _parse_content(x[0], lines)), sections))