diff --git a/.gitignore b/.gitignore
index 0e397fc3..16a48619 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,7 @@ lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/*
__pycache__/
.env
**/.vscode/settings.json
+
+# python test
+.coverage
+.report/
\ No newline at end of file
diff --git a/ecs/crm-datafetch/.env.example b/ecs/crm-datafetch/.env.example
new file mode 100644
index 00000000..4f105876
--- /dev/null
+++ b/ecs/crm-datafetch/.env.example
@@ -0,0 +1,17 @@
+CRM_AUTH_DOMAIN=test
+CRM_USER_NAME=test
+CRM_USER_PASSWORD=test
+CRM_USER_SECURITY_TOKEN=test
+CRM_CONFIG_BUCKET=test
+CRM_BACKUP_BUCKET=test
+IMPORT_DATA_BUCKET=test
+OBJECT_INFO_FOLDER=test
+OBJECT_INFO_FILENAME=test
+PROCESS_RESULT_FOLDER=test
+PROCESS_RESULT_FILENAME=test
+LAST_FETCH_DATE_FOLDER=test
+CRM_IMPORT_DATA_FOLDER=test
+LAST_FETCH_DATE_BACKUP_FOLDER=test
+RESPONSE_JSON_BACKUP_FOLDER=test
+CRM_IMPORT_DATA_BACKUP_FOLDER=test
+LOG_LEVEL=INFO
\ No newline at end of file
diff --git a/ecs/crm-datafetch/.vscode/python.code-snippets b/ecs/crm-datafetch/.vscode/python.code-snippets
new file mode 100644
index 00000000..3b658c8d
--- /dev/null
+++ b/ecs/crm-datafetch/.vscode/python.code-snippets
@@ -0,0 +1,17 @@
+{
+ "Generate Test docstring": {
+ "scope": "python",
+ "prefix": "\"\"\"\"\"\"",
+ "body": [
+ "\"\"\"",
+ "Cases:",
+ " $1",
+ "Arranges:",
+ " $2",
+ "Expects:",
+ " $3",
+ "\"\"\""
+ ],
+ "description": "Test docstring (User Snippets)"
+ }
+}
diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile
index 33501d98..82796598 100644
--- a/ecs/crm-datafetch/Pipfile
+++ b/ecs/crm-datafetch/Pipfile
@@ -3,6 +3,11 @@ url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
+[scripts]
+test = "pytest tests/"
+"test:cov" = "pytest --cov=src --cov-branch --cov-report=term-missing tests/"
+"test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/test_result.html tests/"
+
[packages]
boto3 = "*"
simple-salesforce = "*"
@@ -11,6 +16,10 @@ tenacity = "*"
[dev-packages]
autopep8 = "*"
flake8 = "*"
+pytest = "*"
+pytest-cov = "*"
+pytest-html = "*"
+moto = "*"
[requires]
python_version = "3.8"
diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock
index c8c60540..cfd16ed2 100644
--- a/ecs/crm-datafetch/Pipfile.lock
+++ b/ecs/crm-datafetch/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "ec1d83143aff859500979be73f67196dcfe2298ad3553a7d81ad0605a277d672"
+ "sha256": "7006de596d6123ecd56760b584ab75430fa6bcfc0ecd3fdf49f08368ff53477d"
},
"pipfile-spec": 6,
"requires": {
@@ -18,11 +18,11 @@
"default": {
"attrs": {
"hashes": [
- "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4",
- "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"
+ "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6",
+ "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"
],
- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
- "version": "==21.4.0"
+ "markers": "python_version >= '3.5'",
+ "version": "==22.1.0"
},
"authlib": {
"hashes": [
@@ -33,19 +33,19 @@
},
"boto3": {
"hashes": [
- "sha256:5c775dcb12ca5d6be3f5aa3c49d77783faa64eb30fd3f4af93ff116bb42f9ffb",
- "sha256:5d9bcc355cf6edd7f3849fedac4252e12a0aa2b436cdbc0d4371b16a0f852a30"
+ "sha256:aec404d06690a0ca806592efc6bbe30a9ac88fa2ad73b6d907f7794a8b3b1459",
+ "sha256:df3d6ef02304bd7c3711090936c092d5db735dda109decac1e236c3ef7fdb7af"
],
"index": "pypi",
- "version": "==1.24.34"
+ "version": "==1.24.42"
},
"botocore": {
"hashes": [
- "sha256:0d824a5315f5f5c3bea53c14107a69695ef43190edf647f1281bac8f172ca77c",
- "sha256:9c695d47f1f1212f3e306e51f7bacdf67e58055194ddcf7d8296660b124cf135"
+ "sha256:38a180a6666c5a9b069a75ec3cf374ff2a64c3e90c9f24a916858bcdeb04456d",
+ "sha256:f8e6c2f69a9d577fb9c69e4e74f49f4315a48decee0e7dc88b6e470772110884"
],
"markers": "python_version >= '3.7'",
- "version": "==1.27.34"
+ "version": "==1.27.42"
},
"cached-property": {
"hashes": [
@@ -352,11 +352,11 @@
},
"urllib3": {
"hashes": [
- "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec",
- "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6"
+ "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc",
+ "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
- "version": "==1.26.10"
+ "version": "==1.26.11"
},
"zeep": {
"hashes": [
@@ -368,6 +368,14 @@
}
},
"develop": {
+ "attrs": {
+ "hashes": [
+ "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6",
+ "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c"
+ ],
+ "markers": "python_version >= '3.5'",
+ "version": "==22.1.0"
+ },
"autopep8": {
"hashes": [
"sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979",
@@ -376,36 +384,419 @@
"index": "pypi",
"version": "==1.6.0"
},
- "flake8": {
+ "boto3": {
"hashes": [
- "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d",
- "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"
+ "sha256:aec404d06690a0ca806592efc6bbe30a9ac88fa2ad73b6d907f7794a8b3b1459",
+ "sha256:df3d6ef02304bd7c3711090936c092d5db735dda109decac1e236c3ef7fdb7af"
],
"index": "pypi",
- "version": "==4.0.1"
+ "version": "==1.24.42"
+ },
+ "botocore": {
+ "hashes": [
+ "sha256:38a180a6666c5a9b069a75ec3cf374ff2a64c3e90c9f24a916858bcdeb04456d",
+ "sha256:f8e6c2f69a9d577fb9c69e4e74f49f4315a48decee0e7dc88b6e470772110884"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==1.27.42"
+ },
+ "certifi": {
+ "hashes": [
+ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d",
+ "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==2022.6.15"
+ },
+ "cffi": {
+ "hashes": [
+ "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5",
+ "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef",
+ "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104",
+ "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426",
+ "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405",
+ "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375",
+ "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a",
+ "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e",
+ "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc",
+ "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf",
+ "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185",
+ "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497",
+ "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3",
+ "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35",
+ "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c",
+ "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83",
+ "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21",
+ "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca",
+ "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984",
+ "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac",
+ "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd",
+ "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee",
+ "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a",
+ "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2",
+ "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192",
+ "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7",
+ "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585",
+ "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f",
+ "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e",
+ "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27",
+ "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b",
+ "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e",
+ "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e",
+ "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d",
+ "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c",
+ "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415",
+ "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82",
+ "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02",
+ "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314",
+ "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325",
+ "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c",
+ "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3",
+ "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914",
+ "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045",
+ "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d",
+ "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9",
+ "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5",
+ "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2",
+ "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c",
+ "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3",
+ "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2",
+ "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8",
+ "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d",
+ "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d",
+ "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9",
+ "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162",
+ "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76",
+ "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4",
+ "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e",
+ "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9",
+ "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6",
+ "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b",
+ "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01",
+ "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"
+ ],
+ "version": "==1.15.1"
+ },
+ "charset-normalizer": {
+ "hashes": [
+ "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5",
+ "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==2.1.0"
+ },
+ "coverage": {
+ "extras": [
+ "toml"
+ ],
+ "hashes": [
+ "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32",
+ "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7",
+ "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996",
+ "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55",
+ "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46",
+ "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de",
+ "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039",
+ "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee",
+ "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1",
+ "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f",
+ "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63",
+ "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083",
+ "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe",
+ "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0",
+ "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6",
+ "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe",
+ "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933",
+ "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0",
+ "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c",
+ "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07",
+ "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8",
+ "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b",
+ "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e",
+ "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120",
+ "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f",
+ "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e",
+ "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd",
+ "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f",
+ "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386",
+ "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8",
+ "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae",
+ "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc",
+ "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783",
+ "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d",
+ "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c",
+ "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97",
+ "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978",
+ "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf",
+ "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29",
+ "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39",
+ "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==6.4.2"
+ },
+ "cryptography": {
+ "hashes": [
+ "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59",
+ "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596",
+ "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3",
+ "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5",
+ "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab",
+ "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884",
+ "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82",
+ "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b",
+ "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441",
+ "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa",
+ "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d",
+ "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b",
+ "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a",
+ "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6",
+ "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157",
+ "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280",
+ "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282",
+ "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67",
+ "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8",
+ "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046",
+ "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327",
+ "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==37.0.4"
+ },
+ "flake8": {
+ "hashes": [
+ "sha256:44e3ecd719bba1cb2ae65d1b54212cc9df4f5db15ac271f8856e5e6c2eebefed",
+ "sha256:9c51d3d1426379fd444d3b79eabbeb887849368bd053039066439523d8486961"
+ ],
+ "index": "pypi",
+ "version": "==5.0.1"
+ },
+ "idna": {
+ "hashes": [
+ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
+ "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
+ ],
+ "markers": "python_version >= '3.5'",
+ "version": "==3.3"
+ },
+ "iniconfig": {
+ "hashes": [
+ "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+ "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+ ],
+ "version": "==1.1.1"
+ },
+ "jinja2": {
+ "hashes": [
+ "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
+ "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==3.1.2"
+ },
+ "jmespath": {
+ "hashes": [
+ "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
+ "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==1.0.1"
+ },
+ "markupsafe": {
+ "hashes": [
+ "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003",
+ "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88",
+ "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5",
+ "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7",
+ "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a",
+ "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603",
+ "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1",
+ "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135",
+ "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247",
+ "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6",
+ "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601",
+ "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77",
+ "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02",
+ "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e",
+ "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63",
+ "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f",
+ "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980",
+ "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b",
+ "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812",
+ "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff",
+ "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96",
+ "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1",
+ "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925",
+ "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a",
+ "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6",
+ "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e",
+ "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f",
+ "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4",
+ "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f",
+ "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3",
+ "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c",
+ "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a",
+ "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417",
+ "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a",
+ "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a",
+ "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37",
+ "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452",
+ "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933",
+ "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a",
+ "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==2.1.1"
},
"mccabe": {
"hashes": [
- "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
- "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
+ "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325",
+ "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"
],
- "version": "==0.6.1"
+ "markers": "python_version >= '3.6'",
+ "version": "==0.7.0"
+ },
+ "moto": {
+ "hashes": [
+ "sha256:8bb8e267d9b948509d4739d81d995615a193d2c459f5c0a979aaeb0d3bd4b381",
+ "sha256:cbe8ad8a949f519771e5d25b670738604757fb67cd474d75d14c20677582e81f"
+ ],
+ "index": "pypi",
+ "version": "==3.1.16"
+ },
+ "packaging": {
+ "hashes": [
+ "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
+ "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==21.3"
+ },
+ "pluggy": {
+ "hashes": [
+ "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159",
+ "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==1.0.0"
+ },
+ "py": {
+ "hashes": [
+ "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719",
+ "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+ "version": "==1.11.0"
},
"pycodestyle": {
"hashes": [
- "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20",
- "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f"
+ "sha256:289cdc0969d589d90752582bef6dff57c5fbc6949ee8b013ad6d6449a8ae9437",
+ "sha256:beaba44501f89d785be791c9462553f06958a221d166c64e1f107320f839acc2"
],
- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
- "version": "==2.8.0"
+ "markers": "python_version >= '3.6'",
+ "version": "==2.9.0"
+ },
+ "pycparser": {
+ "hashes": [
+ "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
+ "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"
+ ],
+ "version": "==2.21"
},
"pyflakes": {
"hashes": [
- "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c",
- "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e"
+ "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2",
+ "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"
+ ],
+ "markers": "python_version >= '3.6'",
+ "version": "==2.5.0"
+ },
+ "pyparsing": {
+ "hashes": [
+ "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
+ "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
+ ],
+ "markers": "python_full_version >= '3.6.8'",
+ "version": "==3.0.9"
+ },
+ "pytest": {
+ "hashes": [
+ "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c",
+ "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"
+ ],
+ "index": "pypi",
+ "version": "==7.1.2"
+ },
+ "pytest-cov": {
+ "hashes": [
+ "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6",
+ "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"
+ ],
+ "index": "pypi",
+ "version": "==3.0.0"
+ },
+ "pytest-html": {
+ "hashes": [
+ "sha256:3ee1cf319c913d19fe53aeb0bc400e7b0bc2dbeb477553733db1dad12eb75ee3",
+ "sha256:b7f82f123936a3f4d2950bc993c2c1ca09ce262c9ae12f9ac763a2401380b455"
+ ],
+ "index": "pypi",
+ "version": "==3.1.1"
+ },
+ "pytest-metadata": {
+ "hashes": [
+ "sha256:39261ee0086f17649b180baf2a8633e1922a4c4b6fcc28a2de7d8127a82541bf",
+ "sha256:fcd2f416f15be295943527b3c8ba16a44ae5a7141939c90c3dc5ce9d167cf2a5"
+ ],
+ "markers": "python_version >= '3.7' and python_version < '4'",
+ "version": "==2.0.2"
+ },
+ "python-dateutil": {
+ "hashes": [
+ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+ "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==2.4.0"
+ "version": "==2.8.2"
+ },
+ "pytz": {
+ "hashes": [
+ "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7",
+ "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"
+ ],
+ "version": "==2022.1"
+ },
+ "requests": {
+ "hashes": [
+ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
+ "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
+ ],
+ "markers": "python_version >= '3.7' and python_version < '4'",
+ "version": "==2.28.1"
+ },
+ "responses": {
+ "hashes": [
+ "sha256:2dcc863ba63963c0c3d9ee3fa9507cbe36b7d7b0fccb4f0bdfd9e96c539b1487",
+ "sha256:b82502eb5f09a0289d8e209e7bad71ef3978334f56d09b444253d5ad67bf5253"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==0.21.0"
+ },
+ "s3transfer": {
+ "hashes": [
+ "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd",
+ "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==0.6.0"
+ },
+ "six": {
+ "hashes": [
+ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+ "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==1.16.0"
},
"toml": {
"hashes": [
@@ -414,6 +805,38 @@
],
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.10.2"
+ },
+ "tomli": {
+ "hashes": [
+ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+ "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==2.0.1"
+ },
+ "urllib3": {
+ "hashes": [
+ "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc",
+ "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
+ "version": "==1.26.11"
+ },
+ "werkzeug": {
+ "hashes": [
+ "sha256:4d7013ef96fd197d1cdeb03e066c6c5a491ccb44758a5b2b91137319383e5a5a",
+ "sha256:7e1db6a5ba6b9a8be061e47e900456355b8714c0f238b0313f53afce1a55a79a"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==2.2.1"
+ },
+ "xmltodict": {
+ "hashes": [
+ "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56",
+ "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"
+ ],
+ "markers": "python_version >= '3.4'",
+ "version": "==0.13.0"
}
}
}
diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md
index 5a23e966..243e33e5 100644
--- a/ecs/crm-datafetch/README.md
+++ b/ecs/crm-datafetch/README.md
@@ -67,3 +67,124 @@
- 環境変数が必要な場合、直接設定するか、上記JSONの`"envFile"`に設定されたパスに`.env`ファイルを作成し、環境変数を入力する
- キーボードの「F5」キーを押して起動する
- デバッグモードで実行されるため、適当なところにブレークポイントを置いてデバッグすることができる
+
+## ファイル/フォルダ構成
+
+`[〇〇処理]モジュール`と記載されているファイルは、設計書に記載のシートと一致したPythonファイルです
+
+```text
+.
+├── Dockerfile -- Dokcerイメージを作成するためのファイル
+├── Pipfile -- Pipenv(Pythonの仮想環境管理モジュール)で、依存関係を管理するためのファイル
+├── Pipfile.lock -- Pipenvでインストールされた依存関係のバージョン固定ファイル
+├── README.md -- README
+├── main.py -- CRMデータ取得処理のエントリーポイント
+├── src/ -- プロダクトコード置き場
+│ ├── aws/ -- AWSのリソース操作関連のモジュール置き場
+│ ├── backup_crm_csv_data_process.py -- [CSVバックアップ処理]モジュール
+│ ├── backup_crm_data_process.py -- [CRM電文データバックアップ処理]モジュール
+│ ├── check_object_info_process.py -- [オブジェクト情報形式チェック処理]モジュール
+│ ├── config/ -- 設定ファイル関連のモジュール置き場
+│ ├── controller.py -- [コントロール処理]モジュール
+│ ├── convert_crm_csv_data_process.py -- [CSV変換処理]モジュール
+│ ├── converter/ -- CSV変換処理で実際に変換を行うモジュール置き場
+│ ├── copy_crm_csv_data_process.py -- [CSVアップロード処理]モジュール
+│ ├── error/ -- 処理エラー発生時カスタム例外モジュール置き場
+│ ├── fetch_crm_data_process.py -- [CRMデータ取得処理]モジュール
+│ ├── parser/ -- JSON設定ファイル読み込み処理モジュール置き場
+│ ├── prepare_data_fetch_process.py -- [データ取得準備処理]モジュール
+│ ├── salesforce/ -- SalesforceのAPIリクエストモジュール置き場
+│ ├── set_datetime_period_process.py -- [データ取得期間設定処理]モジュール
+│ ├── system_var/ -- 環境変数と定数ファイル置き場
+│ ├── upload_last_fetch_datetime_process.py -- [前回取得日時ファイル更新処理]モジュール
+│ ├── upload_result_data_process.py -- [取得処理実施結果アップロード処理]モジュール
+│ └── util/ -- ユーティリティモジュール置き場
+│ ├── counter_object.py -- リトライ判定のためのカウントアップクラス
+│ ├── dict_checker.py -- 辞書型値オブジェクトの設定値チェック用クラス
+│ ├── execute_datetime.py -- 取得処理開始年月日時分秒の管理クラス
+│ └── logger.py -- ログ管理クラス
+│
+└── tests/ -- テストコード置き場
+ ├── aws -- AWS操作モジュールのテスト
+ ├── ... -- src配下のモジュール構成と同じ階層にテストコードを追加していく
+ ├── conftest.py -- pytestのフィクスチャやフックを管理するファイル
+ └── docstring_parser.py -- pytest-htmlのレポート出力用のヘルパー
+```
+
+## 単体テストについて
+
+### 前提
+
+- Pytestを使用する
+ -
+- カバレッジも取得したいため、pytest-covも使う
+ -
+- レポートを出力するため、pytest-htmlを使う
+ -
+- S3をモック化したいため、motoをつかう
+ -
+- CRMはテスト用の環境を使いたいため、newdwh_opeのアドレスでDeveloper組織を登録する
+
+### テスト環境構築
+
+- Pipenvの仮想環境下で、以下のコマンドを実行する
+
+```sh
+pipenv install --dev
+```
+
+- `.env.example`をコピーし、同じ階層に`.env`を作成する
+- `.env`の以下に示す環境変数の値をDeveloper組織のものに書き換える
+ - CRM_AUTH_DOMAIN
+ - CRM_USER_NAME
+ - CRM_USER_PASSWORD
+ - CRM_USER_SECURITY_TOKEN
+- 以下のコマンドを実行して単体テストを起動する
+
+```sh
+pipenv run test:cov
+```
+
+#### 拡張機能：Python Test Explorer UIを導入する場合
+
+- VSCodeの拡張機能メニューから、「Python Test Explorer for Visual Studio Code」をインストール
+- コマンドパレットから「Python Configure Tests」を選択、「Pytest」を選択
+- テストメニュー(フラスコのマーク)から、テストを実行することができるようになる
+
+#### 各コマンドの説明
+
+- `pipenv run test`
+ - pytestを使用してテストを実行する
+ - `tests`フォルダに配置されているテストモジュールを対象に、単体テストを実行する
+- `pipenv run test:cov`
+ - pytestのテスト終了時にカバレッジを収集する
+ - 標準出力とカバレッジファイル(`.coverage`)に出力される
+- `pipenv run test:report`
+ - pytestのテスト終了時にテスト結果をHTMLで出力する
+ - `.report/test_result.html`が出力される
+
+## 単体テストの追加方法
+
+- `tests`フォルダ内に、`src`フォルダの構成と同じようにフォルダを作り、`test_<テスト対象のモジュール名.py>`というファイルを作成する
+ - 例：`src/aws/s3.py`をテストする場合は`tests/aws/test_s3.py`というファイル名にする
+- テスト関数はクラスにまとめて、テストスイートとする
+- テスト関数にはドキュメントコメントを付け、テスト観点・準備作業・期待値を記載すること
+ - 上記が出力されるスニペットを用意してある。
+ - `""""""`と入力し、「Test docstring (User Snippets)」を選択し、ドキュメントコメントを挿入できる
+
+```python
+
+from src.aws.s3 import S3Resource
+class TestS3Resource:
+ def test_get_object(self, s3_test, s3_):
+ """
+ Cases:
+ S3からオブジェクトが取得できるか
+ Arranges:
+ - S3をモック化する
+ - 期待値となるファイルを配置する
+ Expects:
+ オブジェクトが取得でき、期待値と正しいこと
+ """
+ # more code...
+```
diff --git a/ecs/crm-datafetch/tests/__init__.py b/ecs/crm-datafetch/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/ecs/crm-datafetch/tests/aws/__init__.py b/ecs/crm-datafetch/tests/aws/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py
new file mode 100644
index 00000000..1daab1e7
--- /dev/null
+++ b/ecs/crm-datafetch/tests/aws/test_s3.py
@@ -0,0 +1,300 @@
+import pytest
+from src.aws.s3 import BackupBucket, ConfigBucket, DataBucket, S3Resource
+
+
+@pytest.fixture
+def bucket_name():
+ return 'test-bucket'
+
+
+@pytest.fixture
+def s3_test(s3_client, bucket_name):
+ s3_client.create_bucket(Bucket=bucket_name)
+ yield
+
+
+class TestS3Resource:
+
+ def test_get_object(self, s3_test, s3_client, bucket_name):
+ """
+ Cases:
+ - S3からオブジェクトが取得できるか
+ Arranges:
+ - S3をモック化する
+ - 期待値となるファイルを配置する
+ Expects:
+ - オブジェクトが取得でき、期待値と正しいこと
+ """
+ # Arrange
+ s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa')
+
+ # Act
+ sut = S3Resource(bucket_name)
+ actual = sut.get_object('hogehoge/test.txt')
+
+ # Assert
+ assert actual == 'aaaaaaaaaaaaaaa'
+
+ def test_put_object(self, s3_test, s3_client, bucket_name):
+ """
+ Cases:
+ - S3にオブジェクトをPUTできるか
+ Arranges:
+ - S3をモック化する
+ Expects:
+ - PUTされたファイルが存在する
+ - PUTされたファイルの内容が期待値と一致する
+ """
+ sut = S3Resource(bucket_name)
+
+ sut.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa')
+ actual = s3_client.get_object(Bucket=bucket_name, Key='hogehoge/test.txt')
+
+ assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa'
+
+ def test_copy(self, s3_test, s3_client, bucket_name):
+ """
+ Cases:
+ - S3内のオブジェクトを別バケットにコピーできるか
+ Arranges:
+ - S3をモック化する
+ - コピー先バケットを作成する
+ - 期待値となるファイルをコピー元バケットに配置する
+ Expects:
+ - コピーされたファイルが存在する
+ - コピーされたファイルの内容が期待値と一致する
+ """
+ for_copy_bucket = 'for-copy-bucket'
+ s3_client.create_bucket(Bucket=for_copy_bucket)
+ s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa')
+
+ sut = S3Resource(bucket_name)
+ sut.copy(bucket_name, 'hogehoge/test.txt', for_copy_bucket, 'test.txt')
+
+ actual = s3_client.get_object(Bucket=for_copy_bucket, Key='test.txt')
+ assert actual['Body'].read() == b'aaaaaaaaaaaaaaa'
+
+ def test_init_raise_no_provide_bucket_name(self):
+ """
+ Cases:
+ - バケット名を指定しない場合、例外となること
+ Arranges:
+
+ Expects:
+ - インスタンス生成時に例外が発生すること
+ """
+ with pytest.raises(Exception) as e:
+ S3Resource()
+ assert e.value.args[0] == "__init__() missing 1 required positional argument: 'bucket_name'"
+
+
+class TestConfigBucket:
+
+ def test_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - オブジェクト情報ファイルが取得できること
+ Arranges:
+ - オブジェクト情報ファイルを配置する
+ Expects:
+ - オブジェクト情報ファイルが文字列として取得でき、期待値と一致する
+ """
+ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name)
+ monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm')
+ monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json')
+ s3_client.put_object(Bucket=bucket_name, Key=f'crm/objects.json', Body=b'aaaaaaaaaaaaaaa')
+
+ sut = ConfigBucket()
+ actual = sut.get_object_info_file()
+
+ assert actual == 'aaaaaaaaaaaaaaa'
+
+ def test_get_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - オブジェクト最終更新日時ファイルが取得できること
+ Arranges:
+ - S3をモック化する
+ - オブジェクト最終更新日時ファイルを配置する
+ Expects:
+ - オブジェクト最終更新日時ファイルが文字列として取得でき、期待値と一致する
+ """
+ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name)
+ monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm')
+ s3_client.put_object(Bucket=bucket_name, Key=f'crm/Object.json', Body=b'aaaaaaaaaaaaaaa')
+
+ sut = ConfigBucket()
+ actual = sut.get_last_fetch_datetime_file('Object.json')
+
+ assert actual == 'aaaaaaaaaaaaaaa'
+
+ def test_put_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - オブジェクト最終更新日時ファイルをPUTできること
+ Arranges:
+ - S3をモック化する
+ - 環境変数をテスト用の値に置き換える
+ Expects:
+ - オブジェクト最終更新日時ファイルが存在する
+ """
+ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name)
+ monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm')
+
+ sut = ConfigBucket()
+ sut.put_last_fetch_datetime_file('Object.json', 'aaaaaaaaaaaaaaa')
+
+ actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/Object.json')
+ assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa'
+
+ def test_config_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること
+ Arranges:
+ - 環境変数をテスト用の値に置き換える
+ Expects:
+ - 環境変数の設定ファイル配置バケット名と一致する
+ """
+ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name)
+
+ sut = ConfigBucket()
+
+ assert str(sut) == bucket_name
+
+
+class TestDataBucket:
+
+ def test_put_csv(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - CSVファイルをPUTできること
+ Arranges:
+ - S3をモック化する
+ - 環境変数をテスト用の値に置き換える
+ Expects:
+ - PUTしたファイルが存在する
+ """
+ monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name)
+ monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target')
+
+ sut = DataBucket()
+ sut.put_csv('test.csv', 'test,test,test')
+
+ actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/target/test.csv')
+ assert actual['Body'].read().decode('utf-8') == 'test,test,test'
+
+ def test_put_csv_from(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - 他のバケットからCSVファイルをコピーできること
+ Arranges:
+ - S3をモック化する
+ - コピー先バケットを作成する
+ - 期待値となるファイルコピー元バケットに配置する
+ - 環境変数をテスト用の値に置き換える
+ Expects:
+ - コピーしたファイルが存在する
+ """
+ for_copy_bucket = 'for-copy-bucket'
+ s3_client.create_bucket(Bucket=for_copy_bucket)
+ s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.csv', Body=b'test,test,test')
+
+ monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', for_copy_bucket)
+ monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target')
+
+ sut = DataBucket()
+ sut.put_csv_from(bucket_name, 'hogehoge/test.csv')
+ actual = s3_client.get_object(Bucket=for_copy_bucket, Key=f'crm/target/test.csv')
+
+ assert actual['Body'].read().decode('utf-8') == 'test,test,test'
+
+ def test_data_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - データ登録バケットを文字列化したとき、バケット名が取得できること
+ Arranges:
+ - 環境変数をテスト用の値に置き換える
+ Expects:
+ - 環境変数のデータ登録バケット名と一致する
+ """
+ monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name)
+
+ sut = DataBucket()
+
+ assert str(sut) == bucket_name
+
+
+class TestBackupBucket:
+
+ def test_put_csv(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - CSVファイルをPUTできること
+ Arranges:
+ - S3をモック化する
+ - 環境変数をテスト用の値に置き換える
+ Expects:
+ - PUTしたファイルが存在する
+ """
+ monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name)
+ monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', 'data_import')
+
+ sut = BackupBucket()
+ sut.put_csv('test.csv', 'test,test,test')
+
+ actual = s3_client.get_object(Bucket=bucket_name, Key=f'data_import/test.csv')
+ assert actual['Body'].read().decode('utf-8') == 'test,test,test'
+
+ def test_put_response_json(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - JSONファイルをPUTできること
+ Arranges:
+ - S3をモック化する
+ - 環境変数をテスト用の値に置き換える
+ Expects:
+ - PUTしたファイルが存在する
+ """
+ monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name)
+ monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', 'response_json')
+
+ sut = BackupBucket()
+ sut.put_response_json('test.json', {"test": "test"})
+
+ actual = s3_client.get_object(Bucket=bucket_name, Key=f'response_json/test.json')
+ assert actual['Body'].read().decode('utf-8') == '{"test": "test"}'
+
+ def test_put_result_json(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - JSONファイルをPUTできること
+ Arranges:
+ - S3をモック化する
+ - 環境変数をテスト用の値に置き換える
+ Expects:
+ - PUTしたファイルが存在する
+ """
+ monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name)
+ monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', 'data_import')
+
+ sut = BackupBucket()
+ sut.put_result_json('result.json', {"test": "test"})
+
+ actual = s3_client.get_object(Bucket=bucket_name, Key=f'data_import/result.json')
+ assert actual['Body'].read().decode('utf-8') == '{"test": "test"}'
+
+ def test_backup_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch):
+ """
+ Cases:
+ - CRMデータバックアップバケットを文字列化したとき、バケット名が取得できること
+ Arranges:
+ - 環境変数をテスト用の値に置き換える
+ Expects:
+ - 環境変数のCRMデータバックアップバケット名と一致する
+ """
+ monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name)
+
+ sut = BackupBucket()
+
+ assert str(sut) == bucket_name
diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py
new file mode 100644
index 00000000..55ade88c
--- /dev/null
+++ b/ecs/crm-datafetch/tests/conftest.py
@@ -0,0 +1,59 @@
+"""pytestでフィクスチャやフック(テスト実行前後に差し込む処理)を管理するモジュール"""
+import os
+from datetime import datetime
+
+import boto3
+import pytest
+from moto import mock_s3
+from py.xml import html # type: ignore
+
+from . import docstring_parser
+
+
+@pytest.fixture
+def aws_credentials():
+ """Mocked AWS Credentials for moto."""
+ os.environ["AWS_ACCESS_KEY_ID"] = "testing"
+ os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
+ os.environ["AWS_SECURITY_TOKEN"] = "testing"
+ os.environ["AWS_SESSION_TOKEN"] = "testing"
+
+
+@pytest.fixture
+def s3_client(aws_credentials):
+ with mock_s3():
+ conn = boto3.client("s3", region_name="us-east-1")
+ yield conn
+
+
+# 以下、レポート出力用の設定
+
+def pytest_html_report_title(report):
+ # レポートタイトル
+ report.title = "CRMデータ連携 CRMデータ取得機能 単体機能テスト結果報告書"
+
+
+def pytest_html_results_table_header(cells):
+ del cells[2:]
+ cells.insert(3, html.th("Cases"))
+ cells.insert(4, html.th("Arranges"))
+ cells.insert(5, html.th("Expects"))
+ cells.append(html.th("Time", class_="sortable time", col="time"))
+
+
+def pytest_html_results_table_row(report, cells):
+ del cells[2:]
+ cells.insert(3, html.td(html.pre(report.cases))) # 「テスト内容」をレポートに出力
+ cells.insert(4, html.td(html.pre(report.arranges))) # 「期待結果」をレポートに出力
+ cells.insert(5, html.td(html.pre(report.expects))) # 「期待結果」をレポートに出力
+ cells.append(html.td(datetime.now(), class_="col-time")) # ついでに「時間」もレポートに出力
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+ outcome = yield
+ report = outcome.get_result()
+ docstring = docstring_parser.parse(str(item.function.__doc__))
+ report.cases = docstring.get("Cases", '') # 「テスト内容」を`report`に追加
+ report.arranges = docstring.get("Arranges", '') # 「準備作業」を`report`に追加
+ report.expects = docstring.get("Expects", '') # 「期待結果」を`report`に追加
diff --git a/ecs/crm-datafetch/tests/docstring_parser.py b/ecs/crm-datafetch/tests/docstring_parser.py
new file mode 100644
index 00000000..040c9b5d
--- /dev/null
+++ b/ecs/crm-datafetch/tests/docstring_parser.py
@@ -0,0 +1,45 @@
+"""pytest-htmlでレポート出力するため、各テスト関数のドキュメントコメントのセクションを抜き出して、辞書にする
+Examples:
+ <セクション名>:となっている部分が対象になる
+ \"\"\"
+ Cases:
+ テストケース
+ Arranges:
+ 準備作業
+ Expects:
+ 期待値
+ \"\"\"
+"""
+
+import re
+from itertools import takewhile
+
+_section_rgx = re.compile(r"^\s*[a-zA-Z]+:\s*$")
+_lspace_rgx = re.compile(r"^\s*")
+
+
+def _parse_section(lines: list) -> list:
+ matches = map(lambda x: _section_rgx.match(x), lines)
+ indexes = [i for i, x in enumerate(matches) if x is not None]
+ return list(map(lambda x: (x, lines[x].strip()[: -1]), indexes))
+
+
+def _count_lspace(s: str) -> int:
+ rgx = _lspace_rgx.match(s)
+ if rgx is not None:
+ return rgx.end()
+ return 0
+
+
+def _parse_content(index: int, lines: list) -> str:
+ lspace = _count_lspace(lines[index])
+ i = index + 1
+ contents = takewhile(lambda x: _count_lspace(x) > lspace, lines[i:])
+ return "\n".join(map(lambda x: x.strip(), contents))
+
+
+def parse(docstring: str) -> dict:
+ """🚧sloppy docstring parser🚧"""
+ lines = docstring.splitlines()
+ sections = _parse_section(lines)
+ return dict(map(lambda x: (x[1], _parse_content(x[0], lines)), sections))