From 54d5961614b7e35093bb21ec3e6ba8f7e688951a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 27 Jun 2024 17:31:34 +0900 Subject: [PATCH 01/42] =?UTF-8?q?feat:=20=E6=96=B0=E8=A6=8F=E4=BD=9C?= =?UTF-8?q?=E6=88=90=E3=80=82=E3=82=B3=E3=83=BC=E3=83=89=E3=83=99=E3=83=BC?= =?UTF-8?q?=E3=82=B9=E3=81=AF=E5=AE=9F=E8=A3=85=E6=A4=9C=E8=A8=BC=E6=99=82?= =?UTF-8?q?=E3=81=AE=E3=82=82=E3=81=AE=E3=82=92=E5=88=A9=E7=94=A8=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/transfer-medpass-data/Dockerfile | 19 ++ lambda/transfer-medpass-data/Pipfile | 16 + lambda/transfer-medpass-data/Pipfile.lock | 292 ++++++++++++++++++ .../src/transfer-medpass-data.py | 204 ++++++++++++ 4 files changed, 531 insertions(+) create mode 100644 lambda/transfer-medpass-data/Dockerfile create mode 100644 lambda/transfer-medpass-data/Pipfile create mode 100644 lambda/transfer-medpass-data/Pipfile.lock create mode 100644 lambda/transfer-medpass-data/src/transfer-medpass-data.py diff --git a/lambda/transfer-medpass-data/Dockerfile b/lambda/transfer-medpass-data/Dockerfile new file mode 100644 index 00000000..b33f6e2f --- /dev/null +++ b/lambda/transfer-medpass-data/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.9 + +ENV TZ="Asia/Tokyo" + +WORKDIR /function +COPY Pipfile Pipfile.lock ${WORKDIR} +RUN \ + apt update -y && \ + # パッケージのセキュリティアップデートのみを適用するコマンド + apt install -y unattended-upgrades && \ + unattended-upgrades && \ + pip install --upgrade pip wheel setuptools && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv +COPY src ./ + +ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] +CMD [ "transfer-medpass-data.handler" ] diff --git a/lambda/transfer-medpass-data/Pipfile b/lambda/transfer-medpass-data/Pipfile new file mode 100644 index 00000000..f54fe4aa --- /dev/null +++ b/lambda/transfer-medpass-data/Pipfile @@ -0,0 +1,16 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +awslambdaric = "*" +boto3 = "*" +pyzipper = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.9" diff --git a/lambda/transfer-medpass-data/Pipfile.lock b/lambda/transfer-medpass-data/Pipfile.lock new file mode 100644 index 00000000..1cf927ca --- /dev/null +++ b/lambda/transfer-medpass-data/Pipfile.lock @@ -0,0 +1,292 @@ +{ + "_meta": { + "hash": { + "sha256": "aedbb7748f654696e0433a6c30f5bee011d6cf0acff5d4cc6fe8b4a6d7c617aa" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "awslambdaric": { + "hashes": [ + "sha256:04d4d4b63cef0e9aad3adea95a5cf3f81634bccde625f104ff025f764a3e8874", + "sha256:0a9922690eb7722417cbac55b913ab070c88444ab83290293debf29d5d9ca371", + "sha256:0e55a815caef1258ed7a9b0adad54e5f1b2ca63966d21342dc5da5b55dd471ef", + "sha256:2566a9db2c613fddddc22aedb45e74dbcd5c1a044da6992424680be719db80c9", + "sha256:34a3e8d27b84ccc9535086487eed50425c0db7a016bf91cf24d8d6cc853faec3", + "sha256:403109390f1a9856f8f6d5f0d5e7f2f83bdec99a62d024e433bff50eda2ed373", + "sha256:4dbd3e2262d080327a90e1ac9d087d7616e8b5be6da83be9fd3ca1b5ae575173", + "sha256:506f004ca08a2f43a898b65dad9d6dd2b2a4910ebe43dcc6682dc5ca0267c24a", + "sha256:662b95b6079b563d3e2ef4fda7bcb20e360901eae7a646425612ef2f4e5d6bdf", + "sha256:6de8d5ea0fc8cb780e060fc06a84328fda3ed329faf97606db59287814490ade", + "sha256:6f298068e2791b0ec5c420ef27cdf8aaba23486dfa5917b6c3067e30c97dde4f", + "sha256:7699c822197f98cc22c262af2368c99d69b43a73113099be52545b195c5e0064", + "sha256:7b75376d62a294e37d96507a68290dd94ce83f5b1a597c42d29242545e1ee126", + "sha256:98b16b4752f16b6a0562cfc1db2e6ce63b2a41ffa97c27e784c3ae5e8c2dc91d", + "sha256:bf57203ebeb4d718b34cd168f6b5ef497bfd2481adf848c5252faf38dccd7bf1", + "sha256:bf824aeff2cde789db1167c377573f1dbf2bf01e75eeb31651de00786ef09ed0", + "sha256:d3438bf0be646d61f8a60fb668c0c3b103963611462f35bf2db11ef50242f54f", + "sha256:d530ab882984188436f3869a2fc172086ab6b43a5f7502a86b241347a5441de3", + "sha256:e4be962e90035f362511eac65ad5c0242da15f73cfae27716de42b47f9d0a2a8", + "sha256:e5145fa63560901d7dbe26c5ee0dd5977f3783ab799fa04f50c50c207be78305", + "sha256:eb1a13c130a8a2ffc1c127a2bd581f45b9f10ec32a3892e41b5b1bdc9788b92d", + "sha256:ed798ab90f5230fe11079b269c3748bd875bc218c2ad4528ae2b4a04a0cd92b5", + "sha256:f55cf42cea67661ccea604b7774ba8cb1e467dadaf3d4c4b7f4e029faf43a46b" + ], + "index": "pypi", + "version": "==2.0.11" + }, + "boto3": { + "hashes": [ + "sha256:684cba753d64978a486e8ea9645d53de0d4e3b4a3ab1495b26bd04b9541cea2d", + "sha256:db7bbb1c6059e99b74dcf634e497b04addcac4c527ae2b2696e47c39eccc6c50" + ], + "index": "pypi", + "version": "==1.34.92" + }, + "botocore": { + "hashes": [ + "sha256:4211a22a1f6c6935e70cbb84c2cd93b29f9723eaf5036d59748dd104f389a681", + "sha256:d1ca4886271f184445ec737cd2e752498648cca383887c5a37b2e01c8ab94039" + ], + "markers": "python_version >= '3.8'", + "version": "==1.34.92" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "pycryptodomex": { + "hashes": [ + "sha256:0daad007b685db36d977f9de73f61f8da2a7104e20aca3effd30752fd56f73e1", + "sha256:108e5f1c1cd70ffce0b68739c75734437c919d2eaec8e85bffc2c8b4d2794305", + "sha256:19764605feea0df966445d46533729b645033f134baeb3ea26ad518c9fdf212c", + "sha256:1be97461c439a6af4fe1cf8bf6ca5936d3db252737d2f379cc6b2e394e12a458", + "sha256:25cd61e846aaab76d5791d006497134602a9e451e954833018161befc3b5b9ed", + "sha256:2a47bcc478741b71273b917232f521fd5704ab4b25d301669879e7273d3586cc", + "sha256:59af01efb011b0e8b686ba7758d59cf4a8263f9ad35911bfe3f416cee4f5c08c", + "sha256:5dcac11031a71348faaed1f403a0debd56bf5404232284cf8c761ff918886ebc", + "sha256:62a5ec91388984909bb5398ea49ee61b68ecb579123694bffa172c3b0a107079", + "sha256:645bd4ca6f543685d643dadf6a856cc382b654cc923460e3a10a49c1b3832aeb", + "sha256:653b29b0819605fe0898829c8ad6400a6ccde096146730c2da54eede9b7b8baa", + "sha256:69138068268127cd605e03438312d8f271135a33140e2742b417d027a0539427", + "sha256:6e186342cfcc3aafaad565cbd496060e5a614b441cacc3995ef0091115c1f6c5", + "sha256:76bd15bb65c14900d98835fcd10f59e5e0435077431d3a394b60b15864fddd64", + "sha256:7805830e0c56d88f4d491fa5ac640dfc894c5ec570d1ece6ed1546e9df2e98d6", + "sha256:7a710b79baddd65b806402e14766c721aee8fb83381769c27920f26476276c1e", + "sha256:7a7a8f33a1f1fb762ede6cc9cbab8f2a9ba13b196bfaf7bc6f0b39d2ba315a43", + "sha256:82ee7696ed8eb9a82c7037f32ba9b7c59e51dda6f105b39f043b6ef293989cb3", + "sha256:88afd7a3af7ddddd42c2deda43d53d3dfc016c11327d0915f90ca34ebda91499", + "sha256:8af1a451ff9e123d0d8bd5d5e60f8e3315c3a64f3cdd6bc853e26090e195cdc8", + "sha256:8ee606964553c1a0bc74057dd8782a37d1c2bc0f01b83193b6f8bb14523b877b", + "sha256:91852d4480a4537d169c29a9d104dda44094c78f1f5b67bca76c29a91042b623", + "sha256:9c682436c359b5ada67e882fec34689726a09c461efd75b6ea77b2403d5665b7", + "sha256:bc3ee1b4d97081260d92ae813a83de4d2653206967c4a0a017580f8b9548ddbc", + "sha256:bca649483d5ed251d06daf25957f802e44e6bb6df2e8f218ae71968ff8f8edc4", + "sha256:c39778fd0548d78917b61f03c1fa8bfda6cfcf98c767decf360945fe6f97461e", + "sha256:cbe71b6712429650e3883dc81286edb94c328ffcd24849accac0a4dbcc76958a", + "sha256:d00fe8596e1cc46b44bf3907354e9377aa030ec4cd04afbbf6e899fc1e2a7781", + "sha256:d3584623e68a5064a04748fb6d76117a21a7cb5eaba20608a41c7d0c61721794", + "sha256:e48217c7901edd95f9f097feaa0388da215ed14ce2ece803d3f300b4e694abea", + "sha256:f2e497413560e03421484189a6b65e33fe800d3bd75590e6d78d4dfdb7accf3b", + "sha256:ff5c9a67f8a4fba4aed887216e32cbc48f2a6fb2673bb10a99e43be463e15913" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==3.20.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.9.0.post0" + }, + "pyzipper": { + "hashes": [ + "sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc", + "sha256:6d097f465bfa47796b1494e12ea65d1478107d38e13bc56f6e58eedc4f6c1a87" + ], + "index": "pypi", + "version": "==0.3.6" + }, + "s3transfer": { + "hashes": [ + "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19", + "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d" + ], + "markers": "python_version >= '3.8'", + "version": "==0.10.1" + }, + "simplejson": { + "hashes": [ + "sha256:01f426ee9e3a2d205aa4c22c3da996b51f2de75c4199ef703258a28b304dea8c", + "sha256:03de1ec4ad734f28ca49b0a758b997d752be0d089ed30360157c4e8811999c8f", + "sha256:041dd69026284d10f035cefb4a75026d2cfcef31f31e62585eeb2b7776e7e047", + "sha256:05a668d4a93816fb8a644e90e7987aa3beeb9d2112ca50a474d41e6acb5bb88a", + "sha256:094275b1b8f003afce1167c8a674cd1ee2fd48c566632dac5d149901d5012ff8", + "sha256:097e48686e49026836ef384c7c10ca670acc023cb16a976a689c2eb6c1852df4", + "sha256:0cc9a47bf8cde85c99db5f4a919bb756e62427ade0f2e875a6ec89ae8492d486", + "sha256:0cdb5069870f7d26a34e5adc30672d0a7b26e652720530a023bb3a8d8a42e37f", + "sha256:0dcc54e7cfbd9674ec4ca181e26eaa5b038446601faeaa6c83d146ddef2f2652", + "sha256:16fbebfc38ad4285c256d2430797fd669b0437d090e985c6d443521d4303b133", + "sha256:1844d7782652f859d9648531778582d4842d80cfff8d334eb23bb8da0d22a1b0", + "sha256:2362c66d2c633925d90f2f177f05e0570d320d986130d34dff9ad6edbf7be8ac", + "sha256:29a86bc9c8a913a4e0ffab85c563a7505cdf4bd13fba05342f8314facc0b7586", + "sha256:2a6e5c0e0817fb20dbb880c83caebbd4ef39f1901f6f8e53b73a3c74de4e5172", + "sha256:2d1b47f768e1f4c1c8a9457effabed735939401e85c0ddcdf68444c88a9242e6", + "sha256:30e381471158290ccb79bd31e7bbda4c8f2cf7e1a5f6b557c1b97d6036ccd05b", + "sha256:313dfd911723dc3022fed7050a7b315d5d0681cd56eee08e44e2cbd39fd9ad81", + "sha256:32de1672f91a789cc9e1c36c406b2d75457a242d64e9e73a70b9b814ef00095e", + "sha256:340b7d085b4a5063aacb8664b1250e4a7426c16e1cc80705c548a229153af147", + "sha256:34d95ad8e27754f0d91917600d6ea273e05c82a71021f168c45be48637d9502f", + "sha256:3d549efc7e8f9a180c59462b124991b690ff25c235d5cf495c3246c66a7679cd", + "sha256:3dbfaa79b1c0efdb768392a19110f1aff793f3e8d43f57e292f46734b8affb45", + "sha256:44058bea97429cfa0d6fb1d8eb0736a77022f34a326d5bc64fd6fed8d9304571", + "sha256:46b8cc86204b51eddcf157cbaf3c44a20f24393030442af0909eeb961186cb67", + "sha256:47509775a5c41ec2a6cd17c9c00fc14965cad8e6670059663872ba5e39332f57", + "sha256:4b5df4ee48403885046c6f4fd8adc84c4ac0adec69482f22a17bd4ba52876341", + "sha256:544e5607142d66a469ecf78a3154ec0f915834dc3b8cfdb2677a78ca58319ad6", + "sha256:56d36f47bc7c7684504f0f18feb161a0b1162546b3622e45aa6155f8285180ac", + "sha256:5b217201efc007166e24e9a282007cc208a2d059350a7c5bd0b0303460ad3019", + "sha256:5c4f59dd358c3a99efa46d62dc1583be3a1c37171f5240c4cbdc2d5838870902", + "sha256:5d45ed9452a42064805143480397b586ea2ea322f4b8b69034c51181e7f38342", + "sha256:6197cfebe659ac802a686b5408494115a7062b45cdf37679c4d6a9d4f39649b7", + "sha256:65de5876e34780b43f92d9d2539de16ecc56d16f56e56e59b34adfa1cebe064f", + "sha256:676e8c182f8079851f12ae1cee2fcebe04def2da2a5703a9d747ab125af47732", + "sha256:682b202f56d9d9e1bb22eaca3e37321002223fd5ddef7189b9233e3c14079917", + "sha256:695da62e494e4689ab78fae173a78390a175b6a5ccc4292277ce0f8dba3945d5", + "sha256:6d65ea4582b47d77e9094c22eb0aeded0ebd96c1df86e988870b40c6514c6e21", + "sha256:706a7fc81ceeb321a1040d008b134056012188f95a5c31ad94fb03153b35cc84", + "sha256:7339bd6203351555c1e728acd601ba95ebce0f6041ebdb386e025f00af3f1769", + "sha256:7701a289d45fdfeb37f1d15cf638801cea439df667a613379443772a86e82936", + "sha256:7f27a079cb009ba569983061a50a9270b7e1d35f81e4eeaf0e26f8924027e550", + "sha256:827ddc3b3603f7d0421b054388da6face7871d800c4b3bbedeedc8778e4085ea", + "sha256:8ccc982197982cdda19e3e5ba4ef7f6ad6bed3eb39bb423bfbf7fa2cd29488ab", + "sha256:8f381747c2edebe3c750a571e55103bfcc33b2707a9b91ae033ab9ba718d976a", + "sha256:93ba80fbf959b5852554f23201a5f4b30885930c303546ffa883859a435ea3cf", + "sha256:9a50a9da1cf93e35f26c4ddee162abf3184a340339ec2d4001c34607b87e71b4", + "sha256:a1163bfe5d043c20adeb5c4c8e89dd1dd39b375c8ca6f1c1e35ec537ad7a12e7", + "sha256:a2285609b4edbf9957440642493788ebef6583042b3fb96217c2e71f29bc6d80", + "sha256:a255d30cda6334ba780eb40a56e8134efd3453948b995d3966e45212e34bf018", + "sha256:a3bba99178f1b25878752a8bc6da2f93fbae754ebd4914d2ac4b869b9fb24102", + "sha256:a56005332d70b8d02d476d4a85818b27b01e51dac1a21d5c1a1d8a5df2efb4a6", + "sha256:a89d7fe994b115f0a792e6673f387af3db812a1760d594abad51e0ea11d3e470", + "sha256:a8ac155e3fd3b54a63040df024e57e62c130b15a2fc66eff3c2a946f42beed52", + "sha256:ab5941e1fd509fc151258477ef4b663fe14c94f8faf3581827bf4b02080fd4ba", + "sha256:ab64f087c5863ac621b42e227e5a43bd9b28de581afe7be12ad96562b9be8203", + "sha256:ad37f25fd8dfbed80815c3281b82a165be2a74e663856b9a50099d18789987bc", + "sha256:b17026f3f349a6e87818cd3531e3bbb5cc78a6f4b2b6718f574a8e0512d71e08", + "sha256:b43d3c2e204d709af955bdb904ae127fe137363ace87fbf7dc8fe6017f7f8449", + "sha256:b482d1fdd8f860e743c7de8cd6dfe54fb9fe8cd6ccba29e2966912ac89e17b2f", + "sha256:b6c6cfc492710d8f0303705fa1ff7bb3d6a145f523384e45a6f3b13ada37021f", + "sha256:b9893852c559998f667e6434d2c2474518d4cdfd1b9cec8e57b3c9d577ba55c1", + "sha256:bd694c465cc61fa8e599355e535b6eb561279834d9883aeef08d0e86c44c300c", + "sha256:c0444423129df448788edc66a129bc7560ad7d6a661d74f0900959c0b44349a1", + "sha256:c37b092d29741096c4723f48924a80b1d3de62ca1de254ce88178fa083dd520c", + "sha256:ced906b172bfad62736a27cfafcb6e24bc9938533b0529ff8150f7926fe35b54", + "sha256:d0d3b9f7cee233368d92c89746dde74313abafaa3ec1f0c06a3f4f164dc27bcc", + "sha256:d5f67bffa6fc68e391b2250e1feb43d534ded64a7b918eb89cf7e3e679759d94", + "sha256:d9f0dfde448611f4f818da05f9b544a78f29355dc39151b0dad8e7c65c513e4f", + "sha256:da6dc0cb00ef1e1a8daf285074ca8b2bb89591170c42ceab0c37bcdb9adc802c", + "sha256:dc74a9ef4d61e18ee6f1886b6ef1fe285b1f432885288afacfb7402f7d469448", + "sha256:deb71e6166e4f1264174d78b5b88abd52b14c6649e6eabaf9cf93cb1c7362850", + "sha256:e042ae053e05fe193514d51d6b0f0243729961901e9a75f8b596bfaf69522c52", + "sha256:e2f87a483c4ab0bb2a9adc9ca09173e7f7cf3696e4fa67bd45a6b33181e57921", + "sha256:e2fa1ee5ca34ab2ecfbe3f7a7e952a1ecaebb5b4818f002b5b146324912ac3d5", + "sha256:e7d3f7cd57ce0c6a5bb8133f8ed5c3d1be0473a88b7d91a300626298f12d0999", + "sha256:edb334cab35dcd90eb563fdacb085f10e5dd0b1acb57fa43f8933308b42a8f88", + "sha256:efae49d0148ec68b6e012f1b9e19bd530f4dced378ba919e3e906ae2b829cc31", + "sha256:f1085cadec0f7e76377951d7a87744628c90ac6cc634fc97eecce0c4d41ec563", + "sha256:f15f56b3119fb71fa57eb4613bcd87eb7df6c2f3547de7d341853d3e50cef97e", + "sha256:f1b425a857ce52e651739314e4118fc68bd702ef983148b8fd5cb6f68bb6a020", + "sha256:f31e126204ec38f92dee119af87cf881044ef7dea6f7477ef774ed3d84199c24", + "sha256:f5e0a03e533313eee9437ccc6c4eab47369f17bc919b57df4a20ccd8bc85d8fd", + "sha256:f85d87986ca375b8305b5c4f166783b8db383a6469e8b99b8dba22878388f234", + "sha256:fa6fe8fa94a831886ee164ac03514f361e1387a62a1b9da32fde5c0c1f27fa8d", + "sha256:fb0f8b35c11fd8e4b924f974d331b20fa54555282451db7f2a3b24bd2d33cc11" + ], + "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.18.4" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "urllib3": { + "hashes": [ + "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07", + "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0" + ], + "markers": "python_version < '3.10'", + "version": "==1.26.18" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:1fa8964e4618929488f4ec36795c7ff12924a68b8bf01366c094fc52f770b6e7", + "sha256:2bb76888c5edbcafe6aabab3c47ba534f5a2c2d245c2eddced4a30c4b4946357" + ], + "index": "pypi", + "version": "==2.1.0" + }, + "flake8": { + "hashes": [ + "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132", + "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3" + ], + "index": "pypi", + "version": "==7.0.0" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f", + "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67" + ], + "markers": "python_version >= '3.8'", + "version": "==2.11.1" + }, + "pyflakes": { + "hashes": [ + "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f", + "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a" + ], + "markers": "python_version >= '3.8'", + "version": "==3.2.0" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + } + } +} diff --git a/lambda/transfer-medpass-data/src/transfer-medpass-data.py b/lambda/transfer-medpass-data/src/transfer-medpass-data.py new file mode 100644 index 00000000..efb7e42c --- /dev/null +++ b/lambda/transfer-medpass-data/src/transfer-medpass-data.py @@ -0,0 +1,204 @@ +import datetime +import logging +import os +from zoneinfo import ZoneInfo + +import boto3 +import pyzipper +from pyzipper.zipfile import BadZipFile + +# 環境変数 +# mbj-newdwh2021-staging-data +DATA_IMPORT_BUCKET = os.environ["DATA_IMPORT_BUCKET"] +# mbj-newdwh2021-staging-backup-medpass +MEDPASS_BACKUP_BUCKET = os.environ["MEDPASS_BACKUP_BUCKET"] +MEDPASS_TARGET_FOLDER = os.environ["MEDPASS_TARGET_FOLDER"] # medpass/target +# /staging/s3/medpassZipExtractPassword +MEDPASS_ZIP_PASSWORD_PARAMETER_STORE_KEY = os.environ["MEDPASS_ZIP_PASSWORD_PARAMETER_STORE_KEY"] +LOG_LEVEL = os.environ["LOG_LEVEL"] +TZ = os.environ["TZ"] + +# 定数 +# 多重起動抑制用のコントロールファイルの拡張子 +EXCLUSIVE_CONTROL_FILE_EXT = '.doing' +# tmpフォルダパス +PATH_TMP = '/tmp' +# 拡張子 +ZIP_FILE_EXT = 'zip' +CSV_FILE_EXT = 'csv' + +# S3クライアント +s3_client = boto3.client('s3') +# SystemsManagerクライアント +ssm_client = boto3.client('ssm') + +# logger設定 +logger = logging.getLogger() + + +def log_datetime_convert_tz(*arg): + """ログに出力するタイムスタンプのロケールを変更する（JST指定）""" + return datetime.datetime.now(ZoneInfo(TZ)).timetuple() + + +formatter = logging.Formatter( + '[%(levelname)s]\t%(asctime)s\t%(message)s\n', + '%Y-%m-%d %H:%M:%S' +) +formatter.converter = log_datetime_convert_tz +for handler in logger.handlers: + handler.setFormatter(formatter) + +level = logging.getLevelName(LOG_LEVEL) +if not isinstance(level, int): + level = logging.INFO +logger.setLevel(level) + + +def extract_zip_with_password(zip_filepath: str, extract_to_folder: str, password: str): + """ + 暗号化ZIPを解凍する。 + + :param zip_filepath: ZIPファイルが保管されているフォルダパス + :param extract_to_folder: ZIPファイルの解凍先フォルダ + :param password: ZIPパスワード + """ + # ZIPを解凍 + try: + with pyzipper.AESZipFile(zip_filepath) as z: + # ZIP内のファイルは1つのみ + inner_filename = z.filelist[0].filename + z.extractall(path=extract_to_folder, pwd=password.encode()) + except BadZipFile as e: + pass + except Exception as e: + pass + + logger.info(f'解凍先フォルダ: {os.listdir("/tmp")}') + logger.info(f'解凍ファイルパス: {os.path.join(extract_to_folder, inner_filename)}') + return os.path.join(extract_to_folder, inner_filename) + + +def get_s3_event_parameter(event: dict): + s3_event = event["Records"][0]["s3"] + event_bucket_name = s3_event["bucket"]["name"] + event_object_key = s3_event["object"]["key"] + event_file_name = os.path.basename(event_object_key) + event_folder_name = os.path.dirname(event_object_key).split('/')[0] + + return event_bucket_name, event_object_key, event_file_name, event_folder_name + + +def get_ssm_params(parameter_key: str, with_decryption: bool = True): + response = ssm_client.get_parameter( + Name=parameter_key, WithDecryption=with_decryption) + parameter_value = response['Parameter']['Value'] + return parameter_value + + +def delete_doing_file(event_bucket_name: str, event_object_key: str): + # ⑨ メモリに保持したバケット名/フォルダ名内の「受信データファイル名.doing」ファイルを削除する + s3_client.delete_object( + Bucket=event_bucket_name, Key=f'{event_object_key}{EXCLUSIVE_CONTROL_FILE_EXT}') + + +def handler(event, context): + try: + # ① 処理開始ログを出力する + logger.info('I-01-01 処理開始　medパスデータ解凍・復号化・転送処理') + + # ② 処理開始時に受け取ったイベント情報をログに出力する + # バケット名・フォルダ名・受信データファイル名をメモリに保持 + ( + event_bucket_name, + event_object_key, + event_file_name, + event_folder_name + ) = get_s3_event_parameter(event) + logger.info(f'I-02-01 受信バケット：{event_bucket_name}') + logger.info(f'I-02-01 フォルダ名：{event_folder_name}') + logger.info(f'I-02-01 ファイル名：{event_file_name}') + # ③ S3イベントによるLambdaの重複発火防止の為、メモリに保持したバケット名/フォルダ名内に、「受信データファイル名.doing」ファイルが存在するかチェックする + try: + s3_client.head_object( + Bucket=event_bucket_name, Key=f'{event_object_key}{EXCLUSIVE_CONTROL_FILE_EXT}') + logger.error( + f'E-01-01 {event_bucket_name}/{event_object_key}は現在処理中です。処理を終了します。') + return + except Exception: + # .doingファイルが見つからなかった場合は、処理を続行する + # メモリに保持したバケット名/フォルダ名内に、「受信データファイル名.doing」ファイルを作成する + logger.info('I-03-01 medパスデータの解凍・復号化・転送を開始します') + s3_client.put_object( + Bucket=event_bucket_name, Key=f'{event_object_key}{EXCLUSIVE_CONTROL_FILE_EXT}', Body=b'') + + # ④ S3から暗号化ZIPファイルを読み込む + try: + logger.info( + f'I-04-01 暗号化ZIPファイル読込　読込元：{event_bucket_name}/{event_object_key}') + s3_client.download_file( + event_bucket_name, event_object_key, os.path.join(PATH_TMP, event_file_name)) + logger.info('I-04-02 暗号化ZIPファイルをダウンロードしました') + except Exception as e: + logger.exception(f'E-04-01 暗号化ZIPファイルのダウンロードに失敗しました　エラー内容：{e}') + raise e + + # ⑤ ZIP解凍パスワードをSSM パラメータストアから取得する + try: + logger.info( + f'I-05-01 ZIP解凍パスワードを読み込み　読込元：{MEDPASS_ZIP_PASSWORD_PARAMETER_STORE_KEY}') + zip_password = get_ssm_params( + MEDPASS_ZIP_PASSWORD_PARAMETER_STORE_KEY) + except Exception as e: + logger.exception(f'E-05-01 ZIP解凍パスワードの読み込みに失敗しました　エラー内容：{e}') + raise e + + # ⑥ ZIPファイルを解凍してローカルに保存 + try: + logger.info(f'I-06-01 ZIP解凍開始') + extracted_zip_file_path = extract_zip_with_password( + os.path.join(PATH_TMP, event_file_name), PATH_TMP, zip_password) + except Exception as e: + logger.exception(f'E-06-01 ZIPの解凍に失敗しました　エラー内容：{e}') + raise e + logger.info(f'I-06-02 ZIP解凍成功') + # ⑦ バックアップS3バケットにコピー + copy_source = {'Bucket': event_bucket_name, 'Key': event_object_key} + execute_date_yyyymm = datetime.date.today().strftime('%Y/%m/%d') + s3_client.copy_object( + Bucket=MEDPASS_BACKUP_BUCKET, + Key=f'{execute_date_yyyymm}/{event_file_name}', + CopySource=copy_source + ) + logger.info( + f'I-07-01 medパス受信データのバックアップ完了：{MEDPASS_BACKUP_BUCKET}/{execute_date_yyyymm}/{event_file_name}') + + # ⑧ データ登録S3バケットにアップロード + csv_file_name = extracted_zip_file_path.split('/')[-1] + logger.info(csv_file_name) + s3_client.upload_file( + extracted_zip_file_path, + Bucket=DATA_IMPORT_BUCKET, + Key=f'{MEDPASS_TARGET_FOLDER}/{csv_file_name}' + ) + + # アップロード後、元のバケットからは削除する + s3_client.delete_object(Bucket=event_bucket_name, Key=event_object_key) + logger.info( + f'I-08-01 Encise受信データの転送完了：{DATA_IMPORT_BUCKET}/{MEDPASS_TARGET_FOLDER}/{csv_file_name}') + + # ⑨ メモリに保持したバケット名/フォルダ名内の「受信データファイル名.doing」ファイルを削除する + delete_doing_file(event_bucket_name, event_object_key) + + logger.info('I-08-09 処理終了　Medpass受信データ転送処理') + + except Exception as e: + logger.exception(f'想定外のエラーが発生しました。処理を終了します。　例外内容：{e}') + ( + event_bucket_name, + event_object_key, + _, + _ + ) = get_s3_event_parameter(event) + delete_doing_file(event_bucket_name, event_object_key) + raise e From 479fe869bf4e50c1ea1c865b45643e2cdfe4861c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Jul 2024 09:43:00 +0900 Subject: [PATCH 02/42] =?UTF-8?q?feat:=20python3.12=E3=81=AB=E5=AF=BE?= =?UTF-8?q?=E5=BF=9C=E3=80=82Dockerfile=E3=82=92=E6=9C=80=E9=81=A9?= =?UTF-8?q?=E5=8C=96=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/transfer-medpass-data/Dockerfile | 26 +- lambda/transfer-medpass-data/Pipfile | 2 +- lambda/transfer-medpass-data/Pipfile.lock | 279 +++++++++--------- .../{src/transfer-medpass-data.py => main.py} | 0 4 files changed, 160 insertions(+), 147 deletions(-) rename lambda/transfer-medpass-data/{src/transfer-medpass-data.py => main.py} (100%) diff --git a/lambda/transfer-medpass-data/Dockerfile b/lambda/transfer-medpass-data/Dockerfile index b33f6e2f..b00422cf 100644 --- a/lambda/transfer-medpass-data/Dockerfile +++ b/lambda/transfer-medpass-data/Dockerfile @@ -1,19 +1,19 @@ -FROM python:3.9 +# AWS公式のDockerイメージを利用 +FROM public.ecr.aws/lambda/python:3.12 -ENV TZ="Asia/Tokyo" +# pythonの標準出力をバッファリングしないフラグ +ENV PYTHONUNBUFFERED=1 +# pythonのバイトコードを生成しないフラグ +ENV PYTHONDONTWRITEBYTECODE=1 -WORKDIR /function -COPY Pipfile Pipfile.lock ${WORKDIR} -RUN \ - apt update -y && \ - # パッケージのセキュリティアップデートのみを適用するコマンド - apt install -y unattended-upgrades && \ - unattended-upgrades && \ - pip install --upgrade pip wheel setuptools && \ +# 必要なファイルをイメージにコピー +COPY Pipfile Pipfile.lock main.py ./ + +# ライブラリインストール +RUN pip install --upgrade pip wheel setuptools && \ pip install pipenv --no-cache-dir && \ pipenv install --system --deploy && \ pip uninstall -y pipenv virtualenv-clone virtualenv -COPY src ./ -ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] -CMD [ "transfer-medpass-data.handler" ] + # lambdaハンドラを起動 +CMD [ "main.handler" ] diff --git a/lambda/transfer-medpass-data/Pipfile b/lambda/transfer-medpass-data/Pipfile index f54fe4aa..8bec0696 100644 --- a/lambda/transfer-medpass-data/Pipfile +++ b/lambda/transfer-medpass-data/Pipfile @@ -13,4 +13,4 @@ autopep8 = "*" flake8 = "*" [requires] -python_version = "3.9" +python_version = "3.12" diff --git a/lambda/transfer-medpass-data/Pipfile.lock b/lambda/transfer-medpass-data/Pipfile.lock index 1cf927ca..b4a34c21 100644 --- a/lambda/transfer-medpass-data/Pipfile.lock +++ b/lambda/transfer-medpass-data/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "aedbb7748f654696e0433a6c30f5bee011d6cf0acff5d4cc6fe8b4a6d7c617aa" + "sha256": "233bd2442299b9e1bbedd7037f98893225b53953c1b68e0258e93ade1c1ed159" }, "pipfile-spec": 6, "requires": { - "python_version": "3.9" + "python_version": "3.12" }, "sources": [ { @@ -18,48 +18,48 @@ "default": { "awslambdaric": { "hashes": [ - "sha256:04d4d4b63cef0e9aad3adea95a5cf3f81634bccde625f104ff025f764a3e8874", - "sha256:0a9922690eb7722417cbac55b913ab070c88444ab83290293debf29d5d9ca371", - "sha256:0e55a815caef1258ed7a9b0adad54e5f1b2ca63966d21342dc5da5b55dd471ef", - "sha256:2566a9db2c613fddddc22aedb45e74dbcd5c1a044da6992424680be719db80c9", - "sha256:34a3e8d27b84ccc9535086487eed50425c0db7a016bf91cf24d8d6cc853faec3", - "sha256:403109390f1a9856f8f6d5f0d5e7f2f83bdec99a62d024e433bff50eda2ed373", - "sha256:4dbd3e2262d080327a90e1ac9d087d7616e8b5be6da83be9fd3ca1b5ae575173", - "sha256:506f004ca08a2f43a898b65dad9d6dd2b2a4910ebe43dcc6682dc5ca0267c24a", - "sha256:662b95b6079b563d3e2ef4fda7bcb20e360901eae7a646425612ef2f4e5d6bdf", - "sha256:6de8d5ea0fc8cb780e060fc06a84328fda3ed329faf97606db59287814490ade", - "sha256:6f298068e2791b0ec5c420ef27cdf8aaba23486dfa5917b6c3067e30c97dde4f", - "sha256:7699c822197f98cc22c262af2368c99d69b43a73113099be52545b195c5e0064", - "sha256:7b75376d62a294e37d96507a68290dd94ce83f5b1a597c42d29242545e1ee126", - "sha256:98b16b4752f16b6a0562cfc1db2e6ce63b2a41ffa97c27e784c3ae5e8c2dc91d", - "sha256:bf57203ebeb4d718b34cd168f6b5ef497bfd2481adf848c5252faf38dccd7bf1", - "sha256:bf824aeff2cde789db1167c377573f1dbf2bf01e75eeb31651de00786ef09ed0", - "sha256:d3438bf0be646d61f8a60fb668c0c3b103963611462f35bf2db11ef50242f54f", - "sha256:d530ab882984188436f3869a2fc172086ab6b43a5f7502a86b241347a5441de3", - "sha256:e4be962e90035f362511eac65ad5c0242da15f73cfae27716de42b47f9d0a2a8", - "sha256:e5145fa63560901d7dbe26c5ee0dd5977f3783ab799fa04f50c50c207be78305", - "sha256:eb1a13c130a8a2ffc1c127a2bd581f45b9f10ec32a3892e41b5b1bdc9788b92d", - "sha256:ed798ab90f5230fe11079b269c3748bd875bc218c2ad4528ae2b4a04a0cd92b5", - "sha256:f55cf42cea67661ccea604b7774ba8cb1e467dadaf3d4c4b7f4e029faf43a46b" + "sha256:099f032df56f3a0685df2a2dd8f08246b68e44bd5b04cca0e53638c53d036acc", + "sha256:1224fc8f989f08eb8f10ff902068ea393c1a7fe0e28b683983c66d62a39c86c7", + "sha256:16bc3147fed0c079d108f21ca9ca5b5348ac49044b7bf9f806a36d9a7f410fa9", + "sha256:2087484fa39c2417d447788aed73423fdee5918c2107589aaafc8de9ec8df66d", + "sha256:26cc35cdecb5b5eeb1d20684ce5079dbd8e5c86e44016f47c7fcce1d4fcaeeb7", + "sha256:41b68501a2338ae982306bed4309529323fc6dd6e048e33b05091d8dc48ecbcd", + "sha256:42f68fa404d59d65aead9c290cff82e6272737565f4621c3b1455075d9a72779", + "sha256:4becf3da0366e2797db3c922ad4d2fbe817bc87f86f3c94ca37ebc9519993659", + "sha256:5bee5faf7c4d330f585ad94fc41e5a3728054f3d17316c6ca6810ad8c8cd4fe2", + "sha256:6b5952aa4f697da4ca2308e461adaec98408a2d4548752963fa8cb2ca348f642", + "sha256:76d30202dec0839422ec16900d1b4e43b612f5feb8c8649add14d3f00968083b", + "sha256:8182f0e6b9ec34458423eb271ab27b288b3adf7befd321e7b5d50477307588d5", + "sha256:8b5bfa5f22977cd74e8a66e0f6badfb5f3c537aeeb21d5f9471a85df096970d3", + "sha256:91144b664b8f9d936ed7c1e2f8cceb9904e91888cdb5e2b04865fd98f85f0789", + "sha256:afd86820e985a01e7e245327c34f50ee3608efcac55b3f55ffe61315b8419bd2", + "sha256:c496f833002da3185270af7239f42018ca0dedc6d0df1ca476098c714d46aa52", + "sha256:cb5f71727ab1a04e1f76232a4ae8fa46e30fac89fd9f81cdd2b7272a49361940", + "sha256:d54c2fc5537299c982a8cde59e8099693b02f75936adc24edb2a5125b5ec672e", + "sha256:db094789549a7e8c3d312edb7ce4ce644a48282984e1d109c2eb53293c2ab3e3", + "sha256:df4aca7b4e77c37ca96179d990668468632aff27ee63b4a143abed60bb49adfd", + "sha256:ec3ba835613e7ea0ff49e09c5cd64dd8560e2a51b0b3df789f2bdb707ba8593d", + "sha256:f02b84322a83302f2a1cae5a2624ec7da96d976a29de8643f43bd80895a5efda", + "sha256:f228bdfd3cfb5c4ac348a7743da0c1be54e85e05206f97d0c57186ac6806e6ae" ], "index": "pypi", - "version": "==2.0.11" + "version": "==2.0.12" }, "boto3": { "hashes": [ - "sha256:684cba753d64978a486e8ea9645d53de0d4e3b4a3ab1495b26bd04b9541cea2d", - "sha256:db7bbb1c6059e99b74dcf634e497b04addcac4c527ae2b2696e47c39eccc6c50" + "sha256:342782c02ff077aae118c9c61179eed95c585831fba666baacc5588ff04aa6e1", + "sha256:f6d6e5b0c9ab022a75373fa16c01f0cd54bc1bb64ef3b6ac64ac7cedd56cbe9c" ], "index": "pypi", - "version": "==1.34.92" + "version": "==1.34.134" }, "botocore": { "hashes": [ - "sha256:4211a22a1f6c6935e70cbb84c2cd93b29f9723eaf5036d59748dd104f389a681", - "sha256:d1ca4886271f184445ec737cd2e752498648cca383887c5a37b2e01c8ab94039" + "sha256:45219e00639755f92569b29f8f279d5dde721494791412c1f7026a3779e8d9f4", + "sha256:e29c299599426ed16dd2d4c1e20eef784f96b15e1850ebbc59a3250959285b95" ], "markers": "python_version >= '3.8'", - "version": "==1.34.92" + "version": "==1.34.134" }, "jmespath": { "hashes": [ @@ -125,102 +125,115 @@ }, "s3transfer": { "hashes": [ - "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19", - "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d" + "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6", + "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69" ], "markers": "python_version >= '3.8'", - "version": "==0.10.1" + "version": "==0.10.2" }, "simplejson": { "hashes": [ - "sha256:01f426ee9e3a2d205aa4c22c3da996b51f2de75c4199ef703258a28b304dea8c", - "sha256:03de1ec4ad734f28ca49b0a758b997d752be0d089ed30360157c4e8811999c8f", - "sha256:041dd69026284d10f035cefb4a75026d2cfcef31f31e62585eeb2b7776e7e047", - "sha256:05a668d4a93816fb8a644e90e7987aa3beeb9d2112ca50a474d41e6acb5bb88a", - "sha256:094275b1b8f003afce1167c8a674cd1ee2fd48c566632dac5d149901d5012ff8", - "sha256:097e48686e49026836ef384c7c10ca670acc023cb16a976a689c2eb6c1852df4", - "sha256:0cc9a47bf8cde85c99db5f4a919bb756e62427ade0f2e875a6ec89ae8492d486", - "sha256:0cdb5069870f7d26a34e5adc30672d0a7b26e652720530a023bb3a8d8a42e37f", - "sha256:0dcc54e7cfbd9674ec4ca181e26eaa5b038446601faeaa6c83d146ddef2f2652", - "sha256:16fbebfc38ad4285c256d2430797fd669b0437d090e985c6d443521d4303b133", - "sha256:1844d7782652f859d9648531778582d4842d80cfff8d334eb23bb8da0d22a1b0", - "sha256:2362c66d2c633925d90f2f177f05e0570d320d986130d34dff9ad6edbf7be8ac", - "sha256:29a86bc9c8a913a4e0ffab85c563a7505cdf4bd13fba05342f8314facc0b7586", - "sha256:2a6e5c0e0817fb20dbb880c83caebbd4ef39f1901f6f8e53b73a3c74de4e5172", - "sha256:2d1b47f768e1f4c1c8a9457effabed735939401e85c0ddcdf68444c88a9242e6", - "sha256:30e381471158290ccb79bd31e7bbda4c8f2cf7e1a5f6b557c1b97d6036ccd05b", - "sha256:313dfd911723dc3022fed7050a7b315d5d0681cd56eee08e44e2cbd39fd9ad81", - "sha256:32de1672f91a789cc9e1c36c406b2d75457a242d64e9e73a70b9b814ef00095e", - "sha256:340b7d085b4a5063aacb8664b1250e4a7426c16e1cc80705c548a229153af147", - "sha256:34d95ad8e27754f0d91917600d6ea273e05c82a71021f168c45be48637d9502f", - "sha256:3d549efc7e8f9a180c59462b124991b690ff25c235d5cf495c3246c66a7679cd", - "sha256:3dbfaa79b1c0efdb768392a19110f1aff793f3e8d43f57e292f46734b8affb45", - "sha256:44058bea97429cfa0d6fb1d8eb0736a77022f34a326d5bc64fd6fed8d9304571", - "sha256:46b8cc86204b51eddcf157cbaf3c44a20f24393030442af0909eeb961186cb67", - "sha256:47509775a5c41ec2a6cd17c9c00fc14965cad8e6670059663872ba5e39332f57", - "sha256:4b5df4ee48403885046c6f4fd8adc84c4ac0adec69482f22a17bd4ba52876341", - "sha256:544e5607142d66a469ecf78a3154ec0f915834dc3b8cfdb2677a78ca58319ad6", - "sha256:56d36f47bc7c7684504f0f18feb161a0b1162546b3622e45aa6155f8285180ac", - "sha256:5b217201efc007166e24e9a282007cc208a2d059350a7c5bd0b0303460ad3019", - "sha256:5c4f59dd358c3a99efa46d62dc1583be3a1c37171f5240c4cbdc2d5838870902", - "sha256:5d45ed9452a42064805143480397b586ea2ea322f4b8b69034c51181e7f38342", - "sha256:6197cfebe659ac802a686b5408494115a7062b45cdf37679c4d6a9d4f39649b7", - "sha256:65de5876e34780b43f92d9d2539de16ecc56d16f56e56e59b34adfa1cebe064f", - "sha256:676e8c182f8079851f12ae1cee2fcebe04def2da2a5703a9d747ab125af47732", - "sha256:682b202f56d9d9e1bb22eaca3e37321002223fd5ddef7189b9233e3c14079917", - "sha256:695da62e494e4689ab78fae173a78390a175b6a5ccc4292277ce0f8dba3945d5", - "sha256:6d65ea4582b47d77e9094c22eb0aeded0ebd96c1df86e988870b40c6514c6e21", - "sha256:706a7fc81ceeb321a1040d008b134056012188f95a5c31ad94fb03153b35cc84", - "sha256:7339bd6203351555c1e728acd601ba95ebce0f6041ebdb386e025f00af3f1769", - "sha256:7701a289d45fdfeb37f1d15cf638801cea439df667a613379443772a86e82936", - "sha256:7f27a079cb009ba569983061a50a9270b7e1d35f81e4eeaf0e26f8924027e550", - "sha256:827ddc3b3603f7d0421b054388da6face7871d800c4b3bbedeedc8778e4085ea", - "sha256:8ccc982197982cdda19e3e5ba4ef7f6ad6bed3eb39bb423bfbf7fa2cd29488ab", - "sha256:8f381747c2edebe3c750a571e55103bfcc33b2707a9b91ae033ab9ba718d976a", - "sha256:93ba80fbf959b5852554f23201a5f4b30885930c303546ffa883859a435ea3cf", - "sha256:9a50a9da1cf93e35f26c4ddee162abf3184a340339ec2d4001c34607b87e71b4", - "sha256:a1163bfe5d043c20adeb5c4c8e89dd1dd39b375c8ca6f1c1e35ec537ad7a12e7", - "sha256:a2285609b4edbf9957440642493788ebef6583042b3fb96217c2e71f29bc6d80", - "sha256:a255d30cda6334ba780eb40a56e8134efd3453948b995d3966e45212e34bf018", - "sha256:a3bba99178f1b25878752a8bc6da2f93fbae754ebd4914d2ac4b869b9fb24102", - "sha256:a56005332d70b8d02d476d4a85818b27b01e51dac1a21d5c1a1d8a5df2efb4a6", - "sha256:a89d7fe994b115f0a792e6673f387af3db812a1760d594abad51e0ea11d3e470", - "sha256:a8ac155e3fd3b54a63040df024e57e62c130b15a2fc66eff3c2a946f42beed52", - "sha256:ab5941e1fd509fc151258477ef4b663fe14c94f8faf3581827bf4b02080fd4ba", - "sha256:ab64f087c5863ac621b42e227e5a43bd9b28de581afe7be12ad96562b9be8203", - "sha256:ad37f25fd8dfbed80815c3281b82a165be2a74e663856b9a50099d18789987bc", - "sha256:b17026f3f349a6e87818cd3531e3bbb5cc78a6f4b2b6718f574a8e0512d71e08", - "sha256:b43d3c2e204d709af955bdb904ae127fe137363ace87fbf7dc8fe6017f7f8449", - "sha256:b482d1fdd8f860e743c7de8cd6dfe54fb9fe8cd6ccba29e2966912ac89e17b2f", - "sha256:b6c6cfc492710d8f0303705fa1ff7bb3d6a145f523384e45a6f3b13ada37021f", - "sha256:b9893852c559998f667e6434d2c2474518d4cdfd1b9cec8e57b3c9d577ba55c1", - "sha256:bd694c465cc61fa8e599355e535b6eb561279834d9883aeef08d0e86c44c300c", - "sha256:c0444423129df448788edc66a129bc7560ad7d6a661d74f0900959c0b44349a1", - "sha256:c37b092d29741096c4723f48924a80b1d3de62ca1de254ce88178fa083dd520c", - "sha256:ced906b172bfad62736a27cfafcb6e24bc9938533b0529ff8150f7926fe35b54", - "sha256:d0d3b9f7cee233368d92c89746dde74313abafaa3ec1f0c06a3f4f164dc27bcc", - "sha256:d5f67bffa6fc68e391b2250e1feb43d534ded64a7b918eb89cf7e3e679759d94", - "sha256:d9f0dfde448611f4f818da05f9b544a78f29355dc39151b0dad8e7c65c513e4f", - "sha256:da6dc0cb00ef1e1a8daf285074ca8b2bb89591170c42ceab0c37bcdb9adc802c", - "sha256:dc74a9ef4d61e18ee6f1886b6ef1fe285b1f432885288afacfb7402f7d469448", - "sha256:deb71e6166e4f1264174d78b5b88abd52b14c6649e6eabaf9cf93cb1c7362850", - "sha256:e042ae053e05fe193514d51d6b0f0243729961901e9a75f8b596bfaf69522c52", - "sha256:e2f87a483c4ab0bb2a9adc9ca09173e7f7cf3696e4fa67bd45a6b33181e57921", - "sha256:e2fa1ee5ca34ab2ecfbe3f7a7e952a1ecaebb5b4818f002b5b146324912ac3d5", - "sha256:e7d3f7cd57ce0c6a5bb8133f8ed5c3d1be0473a88b7d91a300626298f12d0999", - "sha256:edb334cab35dcd90eb563fdacb085f10e5dd0b1acb57fa43f8933308b42a8f88", - "sha256:efae49d0148ec68b6e012f1b9e19bd530f4dced378ba919e3e906ae2b829cc31", - "sha256:f1085cadec0f7e76377951d7a87744628c90ac6cc634fc97eecce0c4d41ec563", - "sha256:f15f56b3119fb71fa57eb4613bcd87eb7df6c2f3547de7d341853d3e50cef97e", - "sha256:f1b425a857ce52e651739314e4118fc68bd702ef983148b8fd5cb6f68bb6a020", - "sha256:f31e126204ec38f92dee119af87cf881044ef7dea6f7477ef774ed3d84199c24", - "sha256:f5e0a03e533313eee9437ccc6c4eab47369f17bc919b57df4a20ccd8bc85d8fd", - "sha256:f85d87986ca375b8305b5c4f166783b8db383a6469e8b99b8dba22878388f234", - "sha256:fa6fe8fa94a831886ee164ac03514f361e1387a62a1b9da32fde5c0c1f27fa8d", - "sha256:fb0f8b35c11fd8e4b924f974d331b20fa54555282451db7f2a3b24bd2d33cc11" + "sha256:0405984f3ec1d3f8777c4adc33eac7ab7a3e629f3b1c05fdded63acc7cf01137", + "sha256:0436a70d8eb42bea4fe1a1c32d371d9bb3b62c637969cb33970ad624d5a3336a", + "sha256:061e81ea2d62671fa9dea2c2bfbc1eec2617ae7651e366c7b4a2baf0a8c72cae", + "sha256:064300a4ea17d1cd9ea1706aa0590dcb3be81112aac30233823ee494f02cb78a", + "sha256:08889f2f597ae965284d7b52a5c3928653a9406d88c93e3161180f0abc2433ba", + "sha256:0a48679310e1dd5c9f03481799311a65d343748fe86850b7fb41df4e2c00c087", + "sha256:0b0a3eb6dd39cce23801a50c01a0976971498da49bc8a0590ce311492b82c44b", + "sha256:0d2d5119b1d7a1ed286b8af37357116072fc96700bce3bec5bb81b2e7057ab41", + "sha256:0d551dc931638e2102b8549836a1632e6e7cf620af3d093a7456aa642bff601d", + "sha256:1018bd0d70ce85f165185d2227c71e3b1e446186f9fa9f971b69eee223e1e3cd", + "sha256:11c39fbc4280d7420684494373b7c5904fa72a2b48ef543a56c2d412999c9e5d", + "sha256:11cc3afd8160d44582543838b7e4f9aa5e97865322844b75d51bf4e0e413bb3e", + "sha256:1537b3dd62d8aae644f3518c407aa8469e3fd0f179cdf86c5992792713ed717a", + "sha256:16ca9c90da4b1f50f089e14485db8c20cbfff2d55424062791a7392b5a9b3ff9", + "sha256:176a1b524a3bd3314ed47029a86d02d5a95cc0bee15bd3063a1e1ec62b947de6", + "sha256:18955c1da6fc39d957adfa346f75226246b6569e096ac9e40f67d102278c3bcb", + "sha256:1bb5b50dc6dd671eb46a605a3e2eb98deb4a9af787a08fcdddabe5d824bb9664", + "sha256:1c768e7584c45094dca4b334af361e43b0aaa4844c04945ac7d43379eeda9bc2", + "sha256:1dd4f692304854352c3e396e9b5f0a9c9e666868dd0bdc784e2ac4c93092d87b", + "sha256:25785d038281cd106c0d91a68b9930049b6464288cea59ba95b35ee37c2d23a5", + "sha256:287e39ba24e141b046812c880f4619d0ca9e617235d74abc27267194fc0c7835", + "sha256:2c1467d939932901a97ba4f979e8f2642415fcf02ea12f53a4e3206c9c03bc17", + "sha256:2c433a412e96afb9a3ce36fa96c8e61a757af53e9c9192c97392f72871e18e69", + "sha256:2d022b14d7758bfb98405672953fe5c202ea8a9ccf9f6713c5bd0718eba286fd", + "sha256:2f98d918f7f3aaf4b91f2b08c0c92b1774aea113334f7cde4fe40e777114dbe6", + "sha256:2fc697be37585eded0c8581c4788fcfac0e3f84ca635b73a5bf360e28c8ea1a2", + "sha256:3194cd0d2c959062b94094c0a9f8780ffd38417a5322450a0db0ca1a23e7fbd2", + "sha256:332c848f02d71a649272b3f1feccacb7e4f7e6de4a2e6dc70a32645326f3d428", + "sha256:346820ae96aa90c7d52653539a57766f10f33dd4be609206c001432b59ddf89f", + "sha256:3471e95110dcaf901db16063b2e40fb394f8a9e99b3fe9ee3acc6f6ef72183a2", + "sha256:3848427b65e31bea2c11f521b6fc7a3145d6e501a1038529da2391aff5970f2f", + "sha256:39b6d79f5cbfa3eb63a869639cfacf7c41d753c64f7801efc72692c1b2637ac7", + "sha256:3e74355cb47e0cd399ead3477e29e2f50e1540952c22fb3504dda0184fc9819f", + "sha256:3f39bb1f6e620f3e158c8b2eaf1b3e3e54408baca96a02fe891794705e788637", + "sha256:40847f617287a38623507d08cbcb75d51cf9d4f9551dd6321df40215128325a3", + "sha256:4280e460e51f86ad76dc456acdbfa9513bdf329556ffc8c49e0200878ca57816", + "sha256:445a96543948c011a3a47c8e0f9d61e9785df2544ea5be5ab3bc2be4bd8a2565", + "sha256:4969d974d9db826a2c07671273e6b27bc48e940738d768fa8f33b577f0978378", + "sha256:49aaf4546f6023c44d7e7136be84a03a4237f0b2b5fb2b17c3e3770a758fc1a0", + "sha256:49e0e3faf3070abdf71a5c80a97c1afc059b4f45a5aa62de0c2ca0444b51669b", + "sha256:49f9da0d6cd17b600a178439d7d2d57c5ef01f816b1e0e875e8e8b3b42db2693", + "sha256:4a8c3cc4f9dfc33220246760358c8265dad6e1104f25f0077bbca692d616d358", + "sha256:4d36081c0b1c12ea0ed62c202046dca11438bee48dd5240b7c8de8da62c620e9", + "sha256:4edcd0bf70087b244ba77038db23cd98a1ace2f91b4a3ecef22036314d77ac23", + "sha256:554313db34d63eac3b3f42986aa9efddd1a481169c12b7be1e7512edebff8eaf", + "sha256:5675e9d8eeef0aa06093c1ff898413ade042d73dc920a03e8cea2fb68f62445a", + "sha256:60848ab779195b72382841fc3fa4f71698a98d9589b0a081a9399904487b5832", + "sha256:66e5dc13bfb17cd6ee764fc96ccafd6e405daa846a42baab81f4c60e15650414", + "sha256:6779105d2fcb7fcf794a6a2a233787f6bbd4731227333a072d8513b252ed374f", + "sha256:6ad331349b0b9ca6da86064a3599c425c7a21cd41616e175ddba0866da32df48", + "sha256:6f0a0b41dd05eefab547576bed0cf066595f3b20b083956b1405a6f17d1be6ad", + "sha256:73a8a4653f2e809049999d63530180d7b5a344b23a793502413ad1ecea9a0290", + "sha256:778331444917108fa8441f59af45886270d33ce8a23bfc4f9b192c0b2ecef1b3", + "sha256:7cb98be113911cb0ad09e5523d0e2a926c09a465c9abb0784c9269efe4f95917", + "sha256:7d74beca677623481810c7052926365d5f07393c72cbf62d6cce29991b676402", + "sha256:7f2398361508c560d0bf1773af19e9fe644e218f2a814a02210ac2c97ad70db0", + "sha256:8434dcdd347459f9fd9c526117c01fe7ca7b016b6008dddc3c13471098f4f0dc", + "sha256:8a390e56a7963e3946ff2049ee1eb218380e87c8a0e7608f7f8790ba19390867", + "sha256:92c4a4a2b1f4846cd4364855cbac83efc48ff5a7d7c06ba014c792dd96483f6f", + "sha256:9300aee2a8b5992d0f4293d88deb59c218989833e3396c824b69ba330d04a589", + "sha256:9453419ea2ab9b21d925d0fd7e3a132a178a191881fab4169b6f96e118cc25bb", + "sha256:9652e59c022e62a5b58a6f9948b104e5bb96d3b06940c6482588176f40f4914b", + "sha256:972a7833d4a1fcf7a711c939e315721a88b988553fc770a5b6a5a64bd6ebeba3", + "sha256:9c1a4393242e321e344213a90a1e3bf35d2f624aa8b8f6174d43e3c6b0e8f6eb", + "sha256:9e038c615b3906df4c3be8db16b3e24821d26c55177638ea47b3f8f73615111c", + "sha256:9e4c166f743bb42c5fcc60760fb1c3623e8fda94f6619534217b083e08644b46", + "sha256:9eb117db8d7ed733a7317c4215c35993b815bf6aeab67523f1f11e108c040672", + "sha256:9eb442a2442ce417801c912df68e1f6ccfcd41577ae7274953ab3ad24ef7d82c", + "sha256:a3cd18e03b0ee54ea4319cdcce48357719ea487b53f92a469ba8ca8e39df285e", + "sha256:a8617625369d2d03766413bff9e64310feafc9fc4f0ad2b902136f1a5cd8c6b0", + "sha256:a970a2e6d5281d56cacf3dc82081c95c1f4da5a559e52469287457811db6a79b", + "sha256:aad7405c033d32c751d98d3a65801e2797ae77fac284a539f6c3a3e13005edc4", + "sha256:adcb3332979cbc941b8fff07181f06d2b608625edc0a4d8bc3ffc0be414ad0c4", + "sha256:af9c7e6669c4d0ad7362f79cb2ab6784d71147503e62b57e3d95c4a0f222c01c", + "sha256:b01fda3e95d07a6148702a641e5e293b6da7863f8bc9b967f62db9461330562c", + "sha256:b8d940fd28eb34a7084877747a60873956893e377f15a32ad445fe66c972c3b8", + "sha256:bccb3e88ec26ffa90f72229f983d3a5d1155e41a1171190fa723d4135523585b", + "sha256:bcedf4cae0d47839fee7de344f96b5694ca53c786f28b5f773d4f0b265a159eb", + "sha256:be893258d5b68dd3a8cba8deb35dc6411db844a9d35268a8d3793b9d9a256f80", + "sha256:c0521e0f07cb56415fdb3aae0bbd8701eb31a9dfef47bb57206075a0584ab2a2", + "sha256:c594642d6b13d225e10df5c16ee15b3398e21a35ecd6aee824f107a625690374", + "sha256:c87c22bd6a987aca976e3d3e23806d17f65426191db36d40da4ae16a6a494cbc", + "sha256:c9ac1c2678abf9270e7228133e5b77c6c3c930ad33a3c1dfbdd76ff2c33b7b50", + "sha256:d0e5ffc763678d48ecc8da836f2ae2dd1b6eb2d27a48671066f91694e575173c", + "sha256:d0f402e787e6e7ee7876c8b05e2fe6464820d9f35ba3f172e95b5f8b699f6c7f", + "sha256:d222a9ed082cd9f38b58923775152003765016342a12f08f8c123bf893461f28", + "sha256:d94245caa3c61f760c4ce4953cfa76e7739b6f2cbfc94cc46fff6c050c2390c5", + "sha256:de9a2792612ec6def556d1dc621fd6b2073aff015d64fba9f3e53349ad292734", + "sha256:e2f5a398b5e77bb01b23d92872255e1bcb3c0c719a3be40b8df146570fe7781a", + "sha256:e8dd53a8706b15bc0e34f00e6150fbefb35d2fd9235d095b4f83b3c5ed4fa11d", + "sha256:e9eb3cff1b7d71aa50c89a0536f469cb8d6dcdd585d8f14fb8500d822f3bdee4", + "sha256:ed628c1431100b0b65387419551e822987396bee3c088a15d68446d92f554e0c", + "sha256:ef7938a78447174e2616be223f496ddccdbf7854f7bf2ce716dbccd958cc7d13", + "sha256:f1c70249b15e4ce1a7d5340c97670a95f305ca79f376887759b43bb33288c973", + "sha256:f3c7363a8cb8c5238878ec96c5eb0fc5ca2cb11fc0c7d2379863d342c6ee367a", + "sha256:fbbcc6b0639aa09b9649f36f1bcb347b19403fe44109948392fbb5ea69e48c3e", + "sha256:febffa5b1eda6622d44b245b0685aff6fb555ce0ed734e2d7b1c3acd018a2cff", + "sha256:ff836cd4041e16003549449cc0a5e372f6b6f871eb89007ab0ee18fb2800fded" ], "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.18.4" + "version": "==3.19.2" }, "six": { "hashes": [ @@ -232,29 +245,29 @@ }, "urllib3": { "hashes": [ - "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07", - "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0" + "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3", + "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429" ], "markers": "python_version < '3.10'", - "version": "==1.26.18" + "version": "==1.26.19" } }, "develop": { "autopep8": { "hashes": [ - "sha256:1fa8964e4618929488f4ec36795c7ff12924a68b8bf01366c094fc52f770b6e7", - "sha256:2bb76888c5edbcafe6aabab3c47ba534f5a2c2d245c2eddced4a30c4b4946357" + "sha256:8d6c87eba648fdcfc83e29b788910b8643171c395d9c4bcf115ece035b9c9dda", + "sha256:a203fe0fcad7939987422140ab17a930f684763bf7335bdb6709991dd7ef6c2d" ], "index": "pypi", - "version": "==2.1.0" + "version": "==2.3.1" }, "flake8": { "hashes": [ - "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132", - "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3" + "sha256:2e416edcc62471a64cea09353f4e7bdba32aeb079b6e360554c659a122b1bc6a", + "sha256:48a07b626b55236e0fb4784ee69a465fbf59d79eec1f5b4785c3d3bc57d17aa5" ], "index": "pypi", - "version": "==7.0.0" + "version": "==7.1.0" }, "mccabe": { "hashes": [ @@ -266,11 +279,11 @@ }, "pycodestyle": { "hashes": [ - "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f", - "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67" + "sha256:442f950141b4f43df752dd303511ffded3a04c2b6fb7f65980574f0c31e6e79c", + "sha256:949a39f6b86c3e1515ba1787c2022131d165a8ad271b11370a8819aa070269e4" ], "markers": "python_version >= '3.8'", - "version": "==2.11.1" + "version": "==2.12.0" }, "pyflakes": { "hashes": [ diff --git a/lambda/transfer-medpass-data/src/transfer-medpass-data.py b/lambda/transfer-medpass-data/main.py similarity index 100% rename from lambda/transfer-medpass-data/src/transfer-medpass-data.py rename to lambda/transfer-medpass-data/main.py From 8397998bb3fd9c938f0c71e2f5a4bd6a52b1539b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Jul 2024 11:27:16 +0900 Subject: [PATCH 03/42] =?UTF-8?q?feat:=20=E3=82=A4=E3=83=B3=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=83=BC=E3=83=AB=E3=81=99=E3=82=8B=E3=83=A9=E3=82=A4?= =?UTF-8?q?=E3=83=96=E3=83=A9=E3=83=AA=E3=82=92=E6=9C=80=E9=81=A9=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/transfer-medpass-data/Pipfile | 1 - lambda/transfer-medpass-data/Pipfile.lock | 147 ++-------------------- 2 files changed, 7 insertions(+), 141 deletions(-) diff --git a/lambda/transfer-medpass-data/Pipfile b/lambda/transfer-medpass-data/Pipfile index 8bec0696..4f1166b7 100644 --- a/lambda/transfer-medpass-data/Pipfile +++ b/lambda/transfer-medpass-data/Pipfile @@ -4,7 +4,6 @@ verify_ssl = true name = "pypi" [packages] -awslambdaric = "*" boto3 = "*" pyzipper = "*" diff --git a/lambda/transfer-medpass-data/Pipfile.lock b/lambda/transfer-medpass-data/Pipfile.lock index b4a34c21..1bae414a 100644 --- a/lambda/transfer-medpass-data/Pipfile.lock +++ b/lambda/transfer-medpass-data/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "233bd2442299b9e1bbedd7037f98893225b53953c1b68e0258e93ade1c1ed159" + "sha256": "d8b79fd5be60005b43448511c67536c114e5fd73722a17e77a5e60a9283aea25" }, "pipfile-spec": 6, "requires": { @@ -16,50 +16,21 @@ ] }, "default": { - "awslambdaric": { - "hashes": [ - "sha256:099f032df56f3a0685df2a2dd8f08246b68e44bd5b04cca0e53638c53d036acc", - "sha256:1224fc8f989f08eb8f10ff902068ea393c1a7fe0e28b683983c66d62a39c86c7", - "sha256:16bc3147fed0c079d108f21ca9ca5b5348ac49044b7bf9f806a36d9a7f410fa9", - "sha256:2087484fa39c2417d447788aed73423fdee5918c2107589aaafc8de9ec8df66d", - "sha256:26cc35cdecb5b5eeb1d20684ce5079dbd8e5c86e44016f47c7fcce1d4fcaeeb7", - "sha256:41b68501a2338ae982306bed4309529323fc6dd6e048e33b05091d8dc48ecbcd", - "sha256:42f68fa404d59d65aead9c290cff82e6272737565f4621c3b1455075d9a72779", - "sha256:4becf3da0366e2797db3c922ad4d2fbe817bc87f86f3c94ca37ebc9519993659", - "sha256:5bee5faf7c4d330f585ad94fc41e5a3728054f3d17316c6ca6810ad8c8cd4fe2", - "sha256:6b5952aa4f697da4ca2308e461adaec98408a2d4548752963fa8cb2ca348f642", - "sha256:76d30202dec0839422ec16900d1b4e43b612f5feb8c8649add14d3f00968083b", - "sha256:8182f0e6b9ec34458423eb271ab27b288b3adf7befd321e7b5d50477307588d5", - "sha256:8b5bfa5f22977cd74e8a66e0f6badfb5f3c537aeeb21d5f9471a85df096970d3", - "sha256:91144b664b8f9d936ed7c1e2f8cceb9904e91888cdb5e2b04865fd98f85f0789", - "sha256:afd86820e985a01e7e245327c34f50ee3608efcac55b3f55ffe61315b8419bd2", - "sha256:c496f833002da3185270af7239f42018ca0dedc6d0df1ca476098c714d46aa52", - "sha256:cb5f71727ab1a04e1f76232a4ae8fa46e30fac89fd9f81cdd2b7272a49361940", - "sha256:d54c2fc5537299c982a8cde59e8099693b02f75936adc24edb2a5125b5ec672e", - "sha256:db094789549a7e8c3d312edb7ce4ce644a48282984e1d109c2eb53293c2ab3e3", - "sha256:df4aca7b4e77c37ca96179d990668468632aff27ee63b4a143abed60bb49adfd", - "sha256:ec3ba835613e7ea0ff49e09c5cd64dd8560e2a51b0b3df789f2bdb707ba8593d", - "sha256:f02b84322a83302f2a1cae5a2624ec7da96d976a29de8643f43bd80895a5efda", - "sha256:f228bdfd3cfb5c4ac348a7743da0c1be54e85e05206f97d0c57186ac6806e6ae" - ], - "index": "pypi", - "version": "==2.0.12" - }, "boto3": { "hashes": [ - "sha256:342782c02ff077aae118c9c61179eed95c585831fba666baacc5588ff04aa6e1", - "sha256:f6d6e5b0c9ab022a75373fa16c01f0cd54bc1bb64ef3b6ac64ac7cedd56cbe9c" + "sha256:0314e6598f59ee0f34eb4e6d1a0f69fa65c146d2b88a6e837a527a9956ec2731", + "sha256:d41037e2c680ab8d6c61a0a4ee6bf1fdd9e857f43996672830a95d62d6f6fa79" ], "index": "pypi", - "version": "==1.34.134" + "version": "==1.34.136" }, "botocore": { "hashes": [ - "sha256:45219e00639755f92569b29f8f279d5dde721494791412c1f7026a3779e8d9f4", - "sha256:e29c299599426ed16dd2d4c1e20eef784f96b15e1850ebbc59a3250959285b95" + "sha256:7f7135178692b39143c8f152a618d2a3b71065a317569a7102d2306d4946f42f", + "sha256:c63fe9032091fb9e9477706a3ebfa4d0c109b807907051d892ed574f9b573e61" ], "markers": "python_version >= '3.8'", - "version": "==1.34.134" + "version": "==1.34.136" }, "jmespath": { "hashes": [ @@ -131,110 +102,6 @@ "markers": "python_version >= '3.8'", "version": "==0.10.2" }, - "simplejson": { - "hashes": [ - "sha256:0405984f3ec1d3f8777c4adc33eac7ab7a3e629f3b1c05fdded63acc7cf01137", - "sha256:0436a70d8eb42bea4fe1a1c32d371d9bb3b62c637969cb33970ad624d5a3336a", - "sha256:061e81ea2d62671fa9dea2c2bfbc1eec2617ae7651e366c7b4a2baf0a8c72cae", - "sha256:064300a4ea17d1cd9ea1706aa0590dcb3be81112aac30233823ee494f02cb78a", - "sha256:08889f2f597ae965284d7b52a5c3928653a9406d88c93e3161180f0abc2433ba", - "sha256:0a48679310e1dd5c9f03481799311a65d343748fe86850b7fb41df4e2c00c087", - "sha256:0b0a3eb6dd39cce23801a50c01a0976971498da49bc8a0590ce311492b82c44b", - "sha256:0d2d5119b1d7a1ed286b8af37357116072fc96700bce3bec5bb81b2e7057ab41", - "sha256:0d551dc931638e2102b8549836a1632e6e7cf620af3d093a7456aa642bff601d", - "sha256:1018bd0d70ce85f165185d2227c71e3b1e446186f9fa9f971b69eee223e1e3cd", - "sha256:11c39fbc4280d7420684494373b7c5904fa72a2b48ef543a56c2d412999c9e5d", - "sha256:11cc3afd8160d44582543838b7e4f9aa5e97865322844b75d51bf4e0e413bb3e", - "sha256:1537b3dd62d8aae644f3518c407aa8469e3fd0f179cdf86c5992792713ed717a", - "sha256:16ca9c90da4b1f50f089e14485db8c20cbfff2d55424062791a7392b5a9b3ff9", - "sha256:176a1b524a3bd3314ed47029a86d02d5a95cc0bee15bd3063a1e1ec62b947de6", - "sha256:18955c1da6fc39d957adfa346f75226246b6569e096ac9e40f67d102278c3bcb", - "sha256:1bb5b50dc6dd671eb46a605a3e2eb98deb4a9af787a08fcdddabe5d824bb9664", - "sha256:1c768e7584c45094dca4b334af361e43b0aaa4844c04945ac7d43379eeda9bc2", - "sha256:1dd4f692304854352c3e396e9b5f0a9c9e666868dd0bdc784e2ac4c93092d87b", - "sha256:25785d038281cd106c0d91a68b9930049b6464288cea59ba95b35ee37c2d23a5", - "sha256:287e39ba24e141b046812c880f4619d0ca9e617235d74abc27267194fc0c7835", - "sha256:2c1467d939932901a97ba4f979e8f2642415fcf02ea12f53a4e3206c9c03bc17", - "sha256:2c433a412e96afb9a3ce36fa96c8e61a757af53e9c9192c97392f72871e18e69", - "sha256:2d022b14d7758bfb98405672953fe5c202ea8a9ccf9f6713c5bd0718eba286fd", - "sha256:2f98d918f7f3aaf4b91f2b08c0c92b1774aea113334f7cde4fe40e777114dbe6", - "sha256:2fc697be37585eded0c8581c4788fcfac0e3f84ca635b73a5bf360e28c8ea1a2", - "sha256:3194cd0d2c959062b94094c0a9f8780ffd38417a5322450a0db0ca1a23e7fbd2", - "sha256:332c848f02d71a649272b3f1feccacb7e4f7e6de4a2e6dc70a32645326f3d428", - "sha256:346820ae96aa90c7d52653539a57766f10f33dd4be609206c001432b59ddf89f", - "sha256:3471e95110dcaf901db16063b2e40fb394f8a9e99b3fe9ee3acc6f6ef72183a2", - "sha256:3848427b65e31bea2c11f521b6fc7a3145d6e501a1038529da2391aff5970f2f", - "sha256:39b6d79f5cbfa3eb63a869639cfacf7c41d753c64f7801efc72692c1b2637ac7", - "sha256:3e74355cb47e0cd399ead3477e29e2f50e1540952c22fb3504dda0184fc9819f", - "sha256:3f39bb1f6e620f3e158c8b2eaf1b3e3e54408baca96a02fe891794705e788637", - "sha256:40847f617287a38623507d08cbcb75d51cf9d4f9551dd6321df40215128325a3", - "sha256:4280e460e51f86ad76dc456acdbfa9513bdf329556ffc8c49e0200878ca57816", - "sha256:445a96543948c011a3a47c8e0f9d61e9785df2544ea5be5ab3bc2be4bd8a2565", - "sha256:4969d974d9db826a2c07671273e6b27bc48e940738d768fa8f33b577f0978378", - "sha256:49aaf4546f6023c44d7e7136be84a03a4237f0b2b5fb2b17c3e3770a758fc1a0", - "sha256:49e0e3faf3070abdf71a5c80a97c1afc059b4f45a5aa62de0c2ca0444b51669b", - "sha256:49f9da0d6cd17b600a178439d7d2d57c5ef01f816b1e0e875e8e8b3b42db2693", - "sha256:4a8c3cc4f9dfc33220246760358c8265dad6e1104f25f0077bbca692d616d358", - "sha256:4d36081c0b1c12ea0ed62c202046dca11438bee48dd5240b7c8de8da62c620e9", - "sha256:4edcd0bf70087b244ba77038db23cd98a1ace2f91b4a3ecef22036314d77ac23", - "sha256:554313db34d63eac3b3f42986aa9efddd1a481169c12b7be1e7512edebff8eaf", - "sha256:5675e9d8eeef0aa06093c1ff898413ade042d73dc920a03e8cea2fb68f62445a", - "sha256:60848ab779195b72382841fc3fa4f71698a98d9589b0a081a9399904487b5832", - "sha256:66e5dc13bfb17cd6ee764fc96ccafd6e405daa846a42baab81f4c60e15650414", - "sha256:6779105d2fcb7fcf794a6a2a233787f6bbd4731227333a072d8513b252ed374f", - "sha256:6ad331349b0b9ca6da86064a3599c425c7a21cd41616e175ddba0866da32df48", - "sha256:6f0a0b41dd05eefab547576bed0cf066595f3b20b083956b1405a6f17d1be6ad", - "sha256:73a8a4653f2e809049999d63530180d7b5a344b23a793502413ad1ecea9a0290", - "sha256:778331444917108fa8441f59af45886270d33ce8a23bfc4f9b192c0b2ecef1b3", - "sha256:7cb98be113911cb0ad09e5523d0e2a926c09a465c9abb0784c9269efe4f95917", - "sha256:7d74beca677623481810c7052926365d5f07393c72cbf62d6cce29991b676402", - "sha256:7f2398361508c560d0bf1773af19e9fe644e218f2a814a02210ac2c97ad70db0", - "sha256:8434dcdd347459f9fd9c526117c01fe7ca7b016b6008dddc3c13471098f4f0dc", - "sha256:8a390e56a7963e3946ff2049ee1eb218380e87c8a0e7608f7f8790ba19390867", - "sha256:92c4a4a2b1f4846cd4364855cbac83efc48ff5a7d7c06ba014c792dd96483f6f", - "sha256:9300aee2a8b5992d0f4293d88deb59c218989833e3396c824b69ba330d04a589", - "sha256:9453419ea2ab9b21d925d0fd7e3a132a178a191881fab4169b6f96e118cc25bb", - "sha256:9652e59c022e62a5b58a6f9948b104e5bb96d3b06940c6482588176f40f4914b", - "sha256:972a7833d4a1fcf7a711c939e315721a88b988553fc770a5b6a5a64bd6ebeba3", - "sha256:9c1a4393242e321e344213a90a1e3bf35d2f624aa8b8f6174d43e3c6b0e8f6eb", - "sha256:9e038c615b3906df4c3be8db16b3e24821d26c55177638ea47b3f8f73615111c", - "sha256:9e4c166f743bb42c5fcc60760fb1c3623e8fda94f6619534217b083e08644b46", - "sha256:9eb117db8d7ed733a7317c4215c35993b815bf6aeab67523f1f11e108c040672", - "sha256:9eb442a2442ce417801c912df68e1f6ccfcd41577ae7274953ab3ad24ef7d82c", - "sha256:a3cd18e03b0ee54ea4319cdcce48357719ea487b53f92a469ba8ca8e39df285e", - "sha256:a8617625369d2d03766413bff9e64310feafc9fc4f0ad2b902136f1a5cd8c6b0", - "sha256:a970a2e6d5281d56cacf3dc82081c95c1f4da5a559e52469287457811db6a79b", - "sha256:aad7405c033d32c751d98d3a65801e2797ae77fac284a539f6c3a3e13005edc4", - "sha256:adcb3332979cbc941b8fff07181f06d2b608625edc0a4d8bc3ffc0be414ad0c4", - "sha256:af9c7e6669c4d0ad7362f79cb2ab6784d71147503e62b57e3d95c4a0f222c01c", - "sha256:b01fda3e95d07a6148702a641e5e293b6da7863f8bc9b967f62db9461330562c", - "sha256:b8d940fd28eb34a7084877747a60873956893e377f15a32ad445fe66c972c3b8", - "sha256:bccb3e88ec26ffa90f72229f983d3a5d1155e41a1171190fa723d4135523585b", - "sha256:bcedf4cae0d47839fee7de344f96b5694ca53c786f28b5f773d4f0b265a159eb", - "sha256:be893258d5b68dd3a8cba8deb35dc6411db844a9d35268a8d3793b9d9a256f80", - "sha256:c0521e0f07cb56415fdb3aae0bbd8701eb31a9dfef47bb57206075a0584ab2a2", - "sha256:c594642d6b13d225e10df5c16ee15b3398e21a35ecd6aee824f107a625690374", - "sha256:c87c22bd6a987aca976e3d3e23806d17f65426191db36d40da4ae16a6a494cbc", - "sha256:c9ac1c2678abf9270e7228133e5b77c6c3c930ad33a3c1dfbdd76ff2c33b7b50", - "sha256:d0e5ffc763678d48ecc8da836f2ae2dd1b6eb2d27a48671066f91694e575173c", - "sha256:d0f402e787e6e7ee7876c8b05e2fe6464820d9f35ba3f172e95b5f8b699f6c7f", - "sha256:d222a9ed082cd9f38b58923775152003765016342a12f08f8c123bf893461f28", - "sha256:d94245caa3c61f760c4ce4953cfa76e7739b6f2cbfc94cc46fff6c050c2390c5", - "sha256:de9a2792612ec6def556d1dc621fd6b2073aff015d64fba9f3e53349ad292734", - "sha256:e2f5a398b5e77bb01b23d92872255e1bcb3c0c719a3be40b8df146570fe7781a", - "sha256:e8dd53a8706b15bc0e34f00e6150fbefb35d2fd9235d095b4f83b3c5ed4fa11d", - "sha256:e9eb3cff1b7d71aa50c89a0536f469cb8d6dcdd585d8f14fb8500d822f3bdee4", - "sha256:ed628c1431100b0b65387419551e822987396bee3c088a15d68446d92f554e0c", - "sha256:ef7938a78447174e2616be223f496ddccdbf7854f7bf2ce716dbccd958cc7d13", - "sha256:f1c70249b15e4ce1a7d5340c97670a95f305ca79f376887759b43bb33288c973", - "sha256:f3c7363a8cb8c5238878ec96c5eb0fc5ca2cb11fc0c7d2379863d342c6ee367a", - "sha256:fbbcc6b0639aa09b9649f36f1bcb347b19403fe44109948392fbb5ea69e48c3e", - "sha256:febffa5b1eda6622d44b245b0685aff6fb555ce0ed734e2d7b1c3acd018a2cff", - "sha256:ff836cd4041e16003549449cc0a5e372f6b6f871eb89007ab0ee18fb2800fded" - ], - "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==3.19.2" - }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", From bc29d225262f90cc65ad400956dc0395c7b25bf1 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Jul 2024 15:23:38 +0900 Subject: [PATCH 04/42] =?UTF-8?q?feat:=20=E8=A8=AD=E8=A8=88=E3=81=AB?= =?UTF-8?q?=E5=90=88=E3=82=8F=E3=81=9B=E3=81=A6=E5=AE=9F=E8=A3=85=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/transfer-medpass-data/main.py | 131 +++++++++++++++++---------- 1 file changed, 82 insertions(+), 49 deletions(-) diff --git a/lambda/transfer-medpass-data/main.py b/lambda/transfer-medpass-data/main.py index efb7e42c..096d36a5 100644 --- a/lambda/transfer-medpass-data/main.py +++ b/lambda/transfer-medpass-data/main.py @@ -8,13 +8,14 @@ import pyzipper from pyzipper.zipfile import BadZipFile # 環境変数 -# mbj-newdwh2021-staging-data DATA_IMPORT_BUCKET = os.environ["DATA_IMPORT_BUCKET"] -# mbj-newdwh2021-staging-backup-medpass -MEDPASS_BACKUP_BUCKET = os.environ["MEDPASS_BACKUP_BUCKET"] -MEDPASS_TARGET_FOLDER = os.environ["MEDPASS_TARGET_FOLDER"] # medpass/target -# /staging/s3/medpassZipExtractPassword +HCP_WEB_TARGET_FOLDER = os.environ["HCP_WEB_TARGET_FOLDER"] +HCP_WEB_BACKUP_BUCKET = os.environ["HCP_WEB_BACKUP_BUCKET"] +BACKUP_ZIPFILE_FOLDER = os.environ["BACKUP_ZIPFILE_FOLDER"] +BACKUP_DATA_IMPORT_FOLDER = os.environ["BACKUP_DATA_IMPORT_FOLDER"] +DATA_IMPORT_FILENAME = os.environ["DATA_IMPORT_FILENAME"] MEDPASS_ZIP_PASSWORD_PARAMETER_STORE_KEY = os.environ["MEDPASS_ZIP_PASSWORD_PARAMETER_STORE_KEY"] + LOG_LEVEL = os.environ["LOG_LEVEL"] TZ = os.environ["TZ"] @@ -55,13 +56,15 @@ if not isinstance(level, int): logger.setLevel(level) -def extract_zip_with_password(zip_filepath: str, extract_to_folder: str, password: str): +def extract_zip_with_password(zip_filepath: str, extract_to_folder: str, password: str) -> os.path: """ 暗号化ZIPを解凍する。 :param zip_filepath: ZIPファイルが保管されているフォルダパス :param extract_to_folder: ZIPファイルの解凍先フォルダ :param password: ZIPパスワード + + :return 解凍されたファイルパス """ # ZIPを解凍 try: @@ -69,40 +72,45 @@ def extract_zip_with_password(zip_filepath: str, extract_to_folder: str, passwor # ZIP内のファイルは1つのみ inner_filename = z.filelist[0].filename z.extractall(path=extract_to_folder, pwd=password.encode()) - except BadZipFile as e: - pass except Exception as e: - pass + raise e - logger.info(f'解凍先フォルダ: {os.listdir("/tmp")}') - logger.info(f'解凍ファイルパス: {os.path.join(extract_to_folder, inner_filename)}') return os.path.join(extract_to_folder, inner_filename) -def get_s3_event_parameter(event: dict): +def get_s3_event_parameter(event: dict) -> tuple[str, str, str, str]: s3_event = event["Records"][0]["s3"] - event_bucket_name = s3_event["bucket"]["name"] - event_object_key = s3_event["object"]["key"] - event_file_name = os.path.basename(event_object_key) - event_folder_name = os.path.dirname(event_object_key).split('/')[0] + event_bucket_name: str = s3_event["bucket"]["name"] + event_object_key: str = s3_event["object"]["key"] + event_file_name: str = os.path.basename(event_object_key) + event_folder_name: str = os.path.dirname(event_object_key).split('/')[0] return event_bucket_name, event_object_key, event_file_name, event_folder_name -def get_ssm_params(parameter_key: str, with_decryption: bool = True): +def get_ssm_params(parameter_key: str, with_decryption: bool = True) -> str: + """SSMパラメータストアから指定されたパラメータ名の値を取得する""" response = ssm_client.get_parameter( Name=parameter_key, WithDecryption=with_decryption) - parameter_value = response['Parameter']['Value'] + parameter_value: str = response['Parameter']['Value'] return parameter_value -def delete_doing_file(event_bucket_name: str, event_object_key: str): +def delete_doing_file(event: dict) -> None: + """.doingファイルをバケット上から削除する""" + # イベント情報を取得 + ( + event_bucket_name, + event_object_key, + _, + _ + ) = get_s3_event_parameter(event) # ⑨ メモリに保持したバケット名/フォルダ名内の「受信データファイル名.doing」ファイルを削除する s3_client.delete_object( Bucket=event_bucket_name, Key=f'{event_object_key}{EXCLUSIVE_CONTROL_FILE_EXT}') -def handler(event, context): +def handler(event, context) -> None: try: # ① 処理開始ログを出力する logger.info('I-01-01 処理開始　medパスデータ解凍・復号化・転送処理') @@ -141,7 +149,8 @@ def handler(event, context): logger.info('I-04-02 暗号化ZIPファイルをダウンロードしました') except Exception as e: logger.exception(f'E-04-01 暗号化ZIPファイルのダウンロードに失敗しました　エラー内容：{e}') - raise e + delete_doing_file(event) + return # ⑤ ZIP解凍パスワードをSSM パラメータストアから取得する try: @@ -151,54 +160,78 @@ def handler(event, context): MEDPASS_ZIP_PASSWORD_PARAMETER_STORE_KEY) except Exception as e: logger.exception(f'E-05-01 ZIP解凍パスワードの読み込みに失敗しました　エラー内容：{e}') - raise e + delete_doing_file(event) + return # ⑥ ZIPファイルを解凍してローカルに保存 try: - logger.info(f'I-06-01 ZIP解凍開始') + logger.info(f'I-05-02 ZIP解凍開始') extracted_zip_file_path = extract_zip_with_password( os.path.join(PATH_TMP, event_file_name), PATH_TMP, zip_password) - except Exception as e: - logger.exception(f'E-06-01 ZIPの解凍に失敗しました　エラー内容：{e}') - raise e - logger.info(f'I-06-02 ZIP解凍成功') - # ⑦ バックアップS3バケットにコピー - copy_source = {'Bucket': event_bucket_name, 'Key': event_object_key} - execute_date_yyyymm = datetime.date.today().strftime('%Y/%m/%d') + except RuntimeError as e: + if 'password' in str(e).lower(): + # パスワードが間違っている場合のエラー + logger.exception( + f'E-05-02 ZIPのパスワードが不正のため、解凍に失敗しました　エラー内容：{e}') + delete_doing_file(event) + return + else: + # 想定外のエラー + raise e + # ZIPファイルが壊れている場合のエラー + except BadZipFile as e: + logger.exception(f'E-05-03 ZIPの形式が不正のため、解凍に失敗しました　エラー内容：{e}') + delete_doing_file(event) + return + + # データ登録用にファイルをリネーム + # ZIPファイル名がyyyymmdd.zipのため、年月日部分をデータ登録用ファイル名の末尾につけ、拡張子をCSVに変更 + data_import_file_name = f'{DATA_IMPORT_FILENAME}_{event_file_name.replace(ZIP_FILE_EXT, CSV_FILE_EXT)}' + logger.info(f'I-05-03 ZIP解凍成功') + + # ⑥ 受信した暗号化ZIPファイルと解凍後のファイルをバックアップする + backup_copy_source = { + 'Bucket': event_bucket_name, 'Key': event_object_key} + execute_date_yyyymmdd = datetime.date.today().strftime('%Y/%m/%d') + + # ZIPファイルのバックアップ s3_client.copy_object( - Bucket=MEDPASS_BACKUP_BUCKET, - Key=f'{execute_date_yyyymm}/{event_file_name}', - CopySource=copy_source + Bucket=HCP_WEB_BACKUP_BUCKET, + Key=f'{BACKUP_ZIPFILE_FOLDER}/{execute_date_yyyymmdd}/{event_file_name}', + CopySource=backup_copy_source ) logger.info( - f'I-07-01 medパス受信データのバックアップ完了：{MEDPASS_BACKUP_BUCKET}/{execute_date_yyyymm}/{event_file_name}') + f'I-06-01 medパス受信データのバックアップ完了：{BACKUP_ZIPFILE_FOLDER}/{HCP_WEB_BACKUP_BUCKET}/{execute_date_yyyymmdd}/{event_file_name}') - # ⑧ データ登録S3バケットにアップロード - csv_file_name = extracted_zip_file_path.split('/')[-1] - logger.info(csv_file_name) + # 解凍後ファイルのバックアップ s3_client.upload_file( extracted_zip_file_path, + Bucket=HCP_WEB_BACKUP_BUCKET, + Key=f'{HCP_WEB_TARGET_FOLDER}/{data_import_file_name}' + ) + logger.info( + f'I-06-02 medパス解凍後データのバックアップ完了：{BACKUP_DATA_IMPORT_FOLDER}/{HCP_WEB_BACKUP_BUCKET}/{execute_date_yyyymmdd}/{data_import_file_name}') + + # ⑦ 解凍後のファイルをデータ登録バケットに転送する + data_import_copy_source = {'Bucket': HCP_WEB_BACKUP_BUCKET, + 'Key': f'{BACKUP_DATA_IMPORT_FOLDER}/{execute_date_yyyymmdd}/{data_import_file_name}'} + s3_client.copy_object( Bucket=DATA_IMPORT_BUCKET, - Key=f'{MEDPASS_TARGET_FOLDER}/{csv_file_name}' + Key=f'{HCP_WEB_TARGET_FOLDER}/{data_import_file_name}', + CopySource=data_import_copy_source ) # アップロード後、元のバケットからは削除する s3_client.delete_object(Bucket=event_bucket_name, Key=event_object_key) logger.info( - f'I-08-01 Encise受信データの転送完了：{DATA_IMPORT_BUCKET}/{MEDPASS_TARGET_FOLDER}/{csv_file_name}') + f'I-07-01 medパス解凍後データの転送完了：{DATA_IMPORT_BUCKET}/{HCP_WEB_TARGET_FOLDER}/{data_import_file_name}') - # ⑨ メモリに保持したバケット名/フォルダ名内の「受信データファイル名.doing」ファイルを削除する - delete_doing_file(event_bucket_name, event_object_key) + # ⑧ メモリに保持したバケット名/フォルダ名内の「受信データファイル名.doing」ファイルを削除する + delete_doing_file(event) - logger.info('I-08-09 処理終了　Medpass受信データ転送処理') + logger.info('I-08-01 処理終了　medパスデータ解凍・復号化・転送処理') except Exception as e: logger.exception(f'想定外のエラーが発生しました。処理を終了します。　例外内容：{e}') - ( - event_bucket_name, - event_object_key, - _, - _ - ) = get_s3_event_parameter(event) - delete_doing_file(event_bucket_name, event_object_key) + delete_doing_file(event) raise e From d16762a7018bd6fc0e0fb86576e564961fe88fe3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Jul 2024 16:09:49 +0900 Subject: [PATCH 05/42] =?UTF-8?q?fix:=20=E3=83=AD=E3=82=B0=E5=87=BA?= =?UTF-8?q?=E5=8A=9B=E6=96=87=E8=A8=80=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/transfer-medpass-data/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lambda/transfer-medpass-data/main.py b/lambda/transfer-medpass-data/main.py index 096d36a5..880f3fe0 100644 --- a/lambda/transfer-medpass-data/main.py +++ b/lambda/transfer-medpass-data/main.py @@ -201,16 +201,16 @@ def handler(event, context) -> None: CopySource=backup_copy_source ) logger.info( - f'I-06-01 medパス受信データのバックアップ完了：{BACKUP_ZIPFILE_FOLDER}/{HCP_WEB_BACKUP_BUCKET}/{execute_date_yyyymmdd}/{event_file_name}') + f'I-06-01 medパス受信データのバックアップ完了：{HCP_WEB_BACKUP_BUCKET}/{BACKUP_ZIPFILE_FOLDER}/{execute_date_yyyymmdd}/{event_file_name}') # 解凍後ファイルのバックアップ s3_client.upload_file( extracted_zip_file_path, Bucket=HCP_WEB_BACKUP_BUCKET, - Key=f'{HCP_WEB_TARGET_FOLDER}/{data_import_file_name}' + Key=f'{BACKUP_DATA_IMPORT_FOLDER}/{execute_date_yyyymmdd}/{data_import_file_name}' ) logger.info( - f'I-06-02 medパス解凍後データのバックアップ完了：{BACKUP_DATA_IMPORT_FOLDER}/{HCP_WEB_BACKUP_BUCKET}/{execute_date_yyyymmdd}/{data_import_file_name}') + f'I-06-02 medパス解凍後データのバックアップ完了：{HCP_WEB_BACKUP_BUCKET}/{BACKUP_DATA_IMPORT_FOLDER}/{execute_date_yyyymmdd}/{data_import_file_name}') # ⑦ 解凍後のファイルをデータ登録バケットに転送する data_import_copy_source = {'Bucket': HCP_WEB_BACKUP_BUCKET, From d26f4c5a3fd9cbd1d6d0cb7c966d9c5aa5a65140 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Jul 2024 16:17:39 +0900 Subject: [PATCH 06/42] =?UTF-8?q?fix:=20=E3=83=AD=E3=82=B0=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/transfer-medpass-data/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lambda/transfer-medpass-data/main.py b/lambda/transfer-medpass-data/main.py index 880f3fe0..273a6e43 100644 --- a/lambda/transfer-medpass-data/main.py +++ b/lambda/transfer-medpass-data/main.py @@ -154,8 +154,7 @@ def handler(event, context) -> None: # ⑤ ZIP解凍パスワードをSSM パラメータストアから取得する try: - logger.info( - f'I-05-01 ZIP解凍パスワードを読み込み　読込元：{MEDPASS_ZIP_PASSWORD_PARAMETER_STORE_KEY}') + logger.info('I-05-01 ZIP解凍パスワードを読込') zip_password = get_ssm_params( MEDPASS_ZIP_PASSWORD_PARAMETER_STORE_KEY) except Exception as e: From ad8c18aa230f0dcd1b5875efc8ea9df33aadc2cd Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Jul 2024 16:23:15 +0900 Subject: [PATCH 07/42] =?UTF-8?q?style:=20=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/transfer-medpass-data/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/transfer-medpass-data/Dockerfile b/lambda/transfer-medpass-data/Dockerfile index b00422cf..19b9594f 100644 --- a/lambda/transfer-medpass-data/Dockerfile +++ b/lambda/transfer-medpass-data/Dockerfile @@ -15,5 +15,5 @@ RUN pip install --upgrade pip wheel setuptools && \ pipenv install --system --deploy && \ pip uninstall -y pipenv virtualenv-clone virtualenv - # lambdaハンドラを起動 +# lambdaハンドラを起動 CMD [ "main.handler" ] From 7156fab12e3c9710a2a5111794e8104a9fc458b7 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Jul 2024 16:30:24 +0900 Subject: [PATCH 08/42] =?UTF-8?q?feat:=20(=E3=81=9F=E3=81=B6=E3=82=93?= =?UTF-8?q?=E3=81=AA=E3=81=84=E3=81=91=E3=81=A9=E3=80=81)=E6=8B=A1?= =?UTF-8?q?=E5=BC=B5=E5=AD=90=E5=A4=A7=E6=96=87=E5=AD=97=E3=81=AE=E8=80=83?= =?UTF-8?q?=E6=85=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/transfer-medpass-data/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/transfer-medpass-data/main.py b/lambda/transfer-medpass-data/main.py index 273a6e43..19b6a95f 100644 --- a/lambda/transfer-medpass-data/main.py +++ b/lambda/transfer-medpass-data/main.py @@ -185,7 +185,7 @@ def handler(event, context) -> None: # データ登録用にファイルをリネーム # ZIPファイル名がyyyymmdd.zipのため、年月日部分をデータ登録用ファイル名の末尾につけ、拡張子をCSVに変更 - data_import_file_name = f'{DATA_IMPORT_FILENAME}_{event_file_name.replace(ZIP_FILE_EXT, CSV_FILE_EXT)}' + data_import_file_name = f'{DATA_IMPORT_FILENAME}_{event_file_name.lower().replace(ZIP_FILE_EXT, CSV_FILE_EXT)}' logger.info(f'I-05-03 ZIP解凍成功') # ⑥ 受信した暗号化ZIPファイルと解凍後のファイルをバックアップする From bd76231de4fabfff5e247153b693340706499eef Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Thu, 4 Jul 2024 17:27:26 +0900 Subject: [PATCH 09/42] =?UTF-8?q?=E9=80=94=E4=B8=AD=E3=82=B3=E3=83=9F?= =?UTF-8?q?=E3=83=83=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily-data-unreceive-check.py | 343 ++++++++++++++++++ 1 file changed, 343 insertions(+) create mode 100644 lambda/daily-data-unreceive-check/daily-data-unreceive-check.py diff --git a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py new file mode 100644 index 00000000..1bd3194e --- /dev/null +++ b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py @@ -0,0 +1,343 @@ +import csv +import datetime +import io +import json +import logging +import os +import re +from zoneinfo import ZoneInfo + +import boto3 +from dateutil.relativedelta import relativedelta + +# 環境変数 +CONFIG_BUCKET_NAME = os.environ["CONFIG_BUCKET_NAME"] +MBJ_NOTICE_TOPIC = os.environ["MBJ_NOTICE_TOPIC"] +PROCESSED_MESSAGE_DYNAMODB_TABLE_NAME = os.environ["PROCESSED_MESSAGE_DYNAMODB_TABLE_NAME"] +PROCESSED_MESSAGE_EXPIRES_PERIOD = int(os.environ["PROCESSED_MESSAGE_EXPIRES_PERIOD"]) +LOG_LEVEL = os.environ["LOG_LEVEL"] +TZ = os.environ["TZ"] + +# 定数 +ROW_COMMENT_SYMBOL = '#' +INDEX_REGEX = 0 +INDEX_DATA_NAME = 1 +INDEX_ROW_COMMENT_SYMBOL = 0 +INDEX_SPLIT_NUM = 1 +INDEX_LAST = -1 + +# メール本文に出力する不足ファイル名一覧のインデント +MAIL_INDENT = '　　' + +# AWS操作クライアント +s3_client = boto3.client('s3') +sns_client = boto3.client('sns') +dynamodb_client = boto3.client('dynamodb') + + +# logger設定 +def log_datetime_convert_tz(*arg): + """ログに出力するタイムスタンプのロケールを変更する（JST指定）""" + return datetime.datetime.now(ZoneInfo(TZ)).timetuple() + + +logger = logging.getLogger() +formatter = logging.Formatter( + '[%(levelname)s]\t%(asctime)s\t%(message)s\n', + '%Y-%m-%d %H:%M:%S' +) +formatter.converter = log_datetime_convert_tz +for handler in logger.handlers: + handler.setFormatter(formatter) + +level = logging.getLevelName(LOG_LEVEL) +if not isinstance(level, int): + level = logging.INFO +logger.setLevel(level) + + +def is_duplicate_message(message_id: str) -> bool: + """DynamoDBテーブルに処理済みのSQSメッセージIdが存在するかどうかを調査する + + Args: + message_id (str): SQSメッセージId + + Returns: + bool: 存在する場合はTrue + """ + return dynamodb_client.query( + TableName=PROCESSED_MESSAGE_DYNAMODB_TABLE_NAME, + Select='COUNT', + KeyConditionExpression='message_id = :message_id', + ExpressionAttributeValues={ + ':message_id': {'S': message_id} + } + )["Count"] != 0 + + +def put_success_messages_to_dynamo_db(batch_success_items: list[str]) -> bool: + """処理済みのSQSメッセージIdをDynamoDBにPushする + + Args: + batch_success_items (list[str]): SQSメッセージIdのリスト + + Returns: + bool: 登録成功の場合、True + """ + + # レコードの有効期限を算出 + now = datetime.datetime.now(ZoneInfo(TZ)) + record_expiration_datetime = now + \ + datetime.timedelta(minutes=PROCESSED_MESSAGE_EXPIRES_PERIOD) + record_expiration_time = record_expiration_datetime.timestamp() + + for message_id in batch_success_items: + dynamodb_client.put_item( + TableName=PROCESSED_MESSAGE_DYNAMODB_TABLE_NAME, + Item={ + 'message_id': {'S': message_id}, + 'record_expiration_time': {'N': f'{record_expiration_time}'} + } + ) + return True + + +def substitute_mail_template(mail_template: str, receive_timing: str, mail_msg: str) -> str: + """メールテンプレートのプレースホルダーを置き換える + + Args: + mail_template (str): 置き換え前のメールテンプレート + receive_timing (str): メールテンプレートのプレースホルダーを置き換える文言(受信タイミング) + mail_msg (str): メールテンプレートのプレースホルダーを置き換える文言(ファイル一覧) + + Returns: + str: 置き換え後のメール本文 + """ + substitute_dict = { + "receive_timing": receive_timing, + "notice_file_names": mail_msg + } + mail_str = mail_template.format_map(substitute_dict) + return mail_str + + +def make_failure_item_on_error(message_id: str) -> dict[str, str]: + """Report batch item failuresによる処理に失敗したメッセージの判別のためのレスポンスを作成する + @see + Args: + message_id (str): SQSメッセージId + + Returns: + dict[str, str]: Report batch item failuresで失敗したSQSメッセージを判別するための辞書オブジェクト + """ + return {"itemIdentifier": message_id} + + +def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[dict[str, str]], list[str]]: + """日次データ未受領チェック + + Args: + records (list): SQS Eventのレコードリスト + execute_date (str): 処理稼働年月日 + + Returns: + tuple[list[dict[str, str]], list[str]]: 失敗メッセージIdのリスト, 成功メッセージIdのリスト + """ + batch_failed_items = [] + batch_success_items = [] + + for record in records: + # メール挿入用文言を格納するためのメモリを保持する + mail_message = '' + try: + try: + # 1.SQSメッセージIDを取得する + message_id = record["messageId"] + # 2.DynamoDBテーブルからレコードを取得し、処理済みメッセージかどうかを判別する + if is_duplicate_message(message_id): + logger.info(f'I-02-02 受信したメッセージは既に処理済みのため、処理をスキップします。メッセージID: {message_id}') + continue + except Exception as e: + logger.exception(f"E-02-01 メッセージ重複チェック処理に失敗しました　エラー内容：{e}") + batch_failed_items.append(make_failure_item_on_error(message_id)) + continue + + # SQSパラメータをJSONシリアライズし、Pythonの辞書オブジェクト(イベントパラメータ)を取得する。 + event_parameter = json.loads(record['body']) + + # ③ 設定ファイル[受領チェック対象ファイルリスト]を読み込む + try: + logger.info('I-03-01 ' +'受領チェック対象ファイルリスト読込　読込元：' + f'{CONFIG_BUCKET_NAME}/{event_parameter["check_target_file_list"]}') + check_target_file_list_response = s3_client.get_object( + Bucket=CONFIG_BUCKET_NAME, + Key=f'{event_parameter["check_target_file_list"]}' + ) + logger.info('I-03-02 受領チェック対象ファイルリストを読み込みました') + except Exception as e: + logger.exception(f"E-03-01 受領チェック対象ファイルリストの読み込みに失敗しました　エラー内容：{e}") + batch_failed_items.append(make_failure_item_on_error(message_id)) + continue + + # ④ 受領チェック処理を行う + receive_timing = event_parameter['receive_timing'] + logger.info(f'I-04-01 日次データ受領チェック ({event_parameter['data_source_name']}) 処理開始') + object_prefix = f'{event_parameter["check_folder_prefix"]}/{execute_date}/' + + # 1.日次データバックアップ保管バケットの処理稼働月に該当するサブフォルダにあるファイル一覧を取得する + logger.info(f'I-04-02 オブジェクトリストの取得　取得先：{event_parameter['check_bucket_name']}/{object_prefix}') + receive_file_list_response = s3_client.list_objects_v2(Bucket=event_parameter['check_bucket_name'], Prefix=object_prefix) + receive_file_list = [] + for content in receive_file_list_response.get('Contents', []): + # オブジェクトのキーからファイル名を切り出してリストに追加 + obj_key = content['Key'].rsplit('/', INDEX_SPLIT_NUM) + receive_file_list.append(obj_key[INDEX_LAST]) + + # 2.I/Fファイルチェック処理 + logger.info(f'I-04-03 日次受信データ({event_parameter['data_source_name']}) 未受領チェック処理開始') + check_target_file_name_body = io.TextIOWrapper(io.BytesIO( + check_target_file_list_response["Body"].read()), encoding='utf-8') + match_count = 0 + row_count = 0 + for tsv_row in csv.reader(check_target_file_name_body, delimiter='\t'): + # 「④１．」で取得したリストが「③」で読み込んだファイル内に存在するか確認する + is_file_not_exists = True + for file_name in receive_file_list: + match_result = re.fullmatch(tsv_row[INDEX_REGEX], file_name) + # 「③」で読み込んだファイルに記載されている全てが「④１．」で取得したリストに存在した場合 + if match_result is not None: + is_file_not_exists = False + # 存在したファイルの年月部分を抜き出し、チェック対象年月（処理稼働月-1)である場合 + match_group_yyyymm = match_result.group(1) + today = datetime.datetime.now(ZoneInfo(TZ)) + minus_1_month = today + relativedelta(months=-1) + check_target_yyyymm = minus_1_month.strftime('%Y%m') + if match_group_yyyymm == check_target_yyyymm: + logger.info(f'I-04-04 I/Fファイルの受領を確認しました　ファイル名：{file_name}') + match_count += 1 + else: + logger.info(f'I-04-06 I/Fファイルに不足があります　ファイル名：{file_name}') + mail_message += f'{MAIL_INDENT}{tsv_row[INDEX_DATA_NAME]}(受領年月が不正:{file_name})\n' + break + if is_file_not_exists: + logger.info(f'E-04-06 月次I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') + mail_message += f'{MAIL_INDENT}{tsv_row[INDEX_DATA_NAME]}\n' + + row_count += 1 + + if row_count == match_count: + logger.info('I-04-05 I/Fファイルは全て受領していることを確認しました') + + # ⑤ 「①」でメモリ保持しているメール挿入用文言に出力内容が存在するか確認する + logger.info('I-05-01 メール送信処理開始') + + if len(mail_message) == 0: + logger.info( + f'I-05-09 {execute_date} {event_parameter["data_source_name"]}データI/Fファイルに不足が無いため、メール送信処理をスキップします') + batch_success_items.append(message_id) + continue + + # 1.存在した場合 + logger.info( + f'I-05-02 {execute_date} {event_parameter["data_source_name"]}データI/Fファイルに不足があるため、メール送信処理を開始します') + + try: + logger.info('I-05-03 ' +f'通知メール（タイトル）テンプレートファイル読込　読込元：{CONFIG_BUCKET_NAME}/{event_parameter["notice_mail_title_template"]}') + mail_title_response = s3_client.get_object( + Bucket=CONFIG_BUCKET_NAME, + Key=f'{event_parameter["notice_mail_title_template"]}' + ) + mail_title_template = (mail_title_response['Body'].read().decode('utf-8')) + # 改行を取り除く + mail_title = substitute_mail_template(mail_title_template, receive_timing, mail_message) + mail_title_without_line_break = mail_title.splitlines()[0] + logger.info('I-05-04 通知メール（タイトル）テンプレートファイルを読み込みました') + except Exception as e: + logger.exception( + f'E-05-01 通知メール（タイトル）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') + batch_failed_items.append( + make_failure_item_on_error(message_id)) + continue + + try: + logger.info('I-05-05 ' +f'通知メール（本文）テンプレートファイル読込　読込元：{CONFIG_BUCKET_NAME}/{event_parameter["notice_mail_body_template"]}') + mail_body_template_response = s3_client.get_object( + Bucket=CONFIG_BUCKET_NAME, + Key=f'{event_parameter["notice_mail_body_template"]}' + ) + mail_body_template = (mail_body_template_response['Body'].read().decode('utf-8')) + # メール本文内のプレースホルダーを置き換える + mail_body = substitute_mail_template(mail_body_template, receive_timing, mail_message) + logger.info('I-05-06 通知メール（本文）テンプレートファイルを読み込みました') + except Exception as e: + logger.exception(f'E-05-02 通知メール（本文）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') + batch_failed_items.append(make_failure_item_on_error(message_id)) + continue + + logger.info(f'I-05-07 メール送信指示をします　送信先トピック：{MBJ_NOTICE_TOPIC}') + params = { + 'TopicArn': MBJ_NOTICE_TOPIC, + 'Subject': mail_title_without_line_break, + 'Message': mail_body + } + sns_client.publish(**params) + logger.info('I-05-08 メール送信指示をしました') + + batch_success_items.append(message_id) + except Exception as e: + logger.exception(f'E-99 想定外のエラーが発生しました　エラー内容：{e}') + batch_failed_items.append(make_failure_item_on_error(message_id)) + continue + + return batch_failed_items, batch_success_items + + +def lambda_handler(event, context): + try: + # ① 処理開始ログを出力する + logger.info('I-01-01 処理開始　日次データ受領チェック処理') + # 処理稼働年月を取得しメモリに保持する + execute_date = datetime.date.today() + # 処理成功メッセージIDリストをメモリに保持する（初期値＝空のリスト） + batch_success_items = [] + # 処理失敗メッセージIDリストをメモリに保持する（初期値＝空のリスト） + batch_failed_items = [] + + # ② SQSメッセージ重複排除処理を行う + logger.info('I-02-01 メッセージ処理開始') + batch_failed_items, batch_success_items = daily_data_unreceive_check(event["Records"], execute_date) + logger.info('I-06-01 すべてのメッセージの処理完了') + + # ⑦ メッセージを処理済として、以下のDynamoDBテーブルに記録する + put_success_messages_to_dynamo_db(batch_success_items) + logger.info('I-07-01 処理済みメッセージIDの記録完了') + logger.info('I-07-02 処理終了　日次データ受領チェック処理') + + except Exception as e: + logger.exception(f'E-99 想定外のエラーが発生しました　エラー内容：{e}') + raise e + + return batch_failed_items + + +# 動作確認用のコード +if __name__ == '__main__': + lambda_handler({ + "Records": [ + { + "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78", + "receiptHandle": "MessageReceiptHandle", + "body": "{\"data_source_name\":\"data_source_name\",\"check_bucket_name\":\"check_bucket_name\",\"check_folder_prefix\":\"check_folder_prefix\",\"check_target_file_list\":\"check_target_file_list\",\"notice_mail_title_template\":\"notice_mail_title_template\",\"notice_mail_body_template\":\"notice_mail_body_template\"\r\n}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1523232000000", + "SenderId": "123456789012", + "ApproximateFirstReceiveTimestamp": "1523232000001" + }, + "messageAttributes": {}, + "md5OfBody": "{{{md5_of_body}}}", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue", + "awsRegion": "us-east-1" + } +] +}, {}) From 0b5e0653d7aa663625b2d770eb5e580bc55f2f7e Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 11 Jul 2024 11:31:35 +0900 Subject: [PATCH 10/42] =?UTF-8?q?feat:HCP=E3=82=A6=E3=82=A7=E3=83=96?= =?UTF-8?q?=E5=80=8B=E5=88=A5=E8=A8=AD=E5=AE=9A=E3=83=9E=E3=83=83=E3=83=94?= =?UTF-8?q?=E3=83=B3=E3=82=B0=E3=83=AA=E3=82=B9=E3=83=88=E3=81=AE=E4=BD=9C?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/hcp_web/settings/configmap.config | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 s3/data/hcp_web/settings/configmap.config diff --git a/s3/data/hcp_web/settings/configmap.config b/s3/data/hcp_web/settings/configmap.config new file mode 100644 index 00000000..f4aee336 --- /dev/null +++ b/s3/data/hcp_web/settings/configmap.config @@ -0,0 +1,4 @@ +/* medパス社データ */ +hcp_web_medpass_user_mst_[0-9]{14}\.(CSV|csv) hcp_web_medpass_user_mst.txt +/* Parantir ウェブアクセスログデータ */ +hcpweb_accesslog_[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{6}\.(CSV|csv) hcpweb_accesslog.txt \ No newline at end of file From 0506f20484fe90575955942f770c600d22e32d87 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 11 Jul 2024 11:32:48 +0900 Subject: [PATCH 11/42] =?UTF-8?q?feat:HCP=E3=82=A6=E3=82=A7=E3=83=96=20Vie?= =?UTF-8?q?w=E3=82=AA=E3=83=97=E3=82=B7=E3=83=A7=E3=83=B3=E3=83=81?= =?UTF-8?q?=E3=82=A7=E3=83=83=E3=82=AF=E5=AF=BE=E8=B1=A1=E3=82=B9=E3=82=AD?= =?UTF-8?q?=E3=83=BC=E3=83=9E=E5=90=8D=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/config/view_check/check_target_schemas.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/config/view_check/check_target_schemas.json b/s3/config/view_check/check_target_schemas.json index c8ec5d7f..07b1df0e 100644 --- a/s3/config/view_check/check_target_schemas.json +++ b/s3/config/view_check/check_target_schemas.json @@ -1,3 +1,3 @@ { - "check_target_schemas": ["custom01", "custom02", "custom03", "custom04"] + "check_target_schemas": ["custom01", "custom02", "custom03", "custom04","custom05"] } From 8a7fb7381c5b65a19cefaa4261c299cc9a85c784 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 11 Jul 2024 11:35:09 +0900 Subject: [PATCH 12/42] =?UTF-8?q?feat:HCP=E3=82=A6=E3=82=A7=E3=83=96=20?= =?UTF-8?q?=E5=80=8B=E5=88=A5=E8=A8=AD=E5=AE=9A=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hcp_web/settings/hcp_web_medpass_user_mst.txt | 13 +++++++++++++ s3/data/hcp_web/settings/hcpweb_accesslog.txt | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt create mode 100644 s3/data/hcp_web/settings/hcpweb_accesslog.txt diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt new file mode 100644 index 00000000..31448f96 --- /dev/null +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt @@ -0,0 +1,13 @@ +hcp_web +, +utf-8 +" +LF +1 +39 +Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate,Gender,CreateDate,LastUpdate,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes +Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate_org,Gender,CreateDate_org,LastUpdate_org,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes +src06.hcp_web_medpass_user_mst +org06.hcp_web_medpass_user_mst +hcp_web_medpass_user_mst_ex.sql + diff --git a/s3/data/hcp_web/settings/hcpweb_accesslog.txt b/s3/data/hcp_web/settings/hcpweb_accesslog.txt new file mode 100644 index 00000000..d5bb453e --- /dev/null +++ b/s3/data/hcp_web/settings/hcpweb_accesslog.txt @@ -0,0 +1,13 @@ +hcp_web +, +utf-8 +" +LF +1 +178 +id,browser,campaign,carrier,click_context,date_time,duplicate_events,duplicated_from,evar1,evar10,evar16,evar24,evar4,evar54,evar55,evar71,evar73,evar74,evar8,evar9,event_list,exclude_hit,filename,first_hit_page_url,first_hit_ref_type,first_hit_referrer,first_hit_time_gmt,geo_city,geo_country,geo_dma,geo_region,hit_source,hit_time_gmt,hitid_high,hitid_low,last_hit_time_gmt,mobile_id,mobiledevice,os,page_event,page_event_var1,page_event_var2,page_url,pagename,paid_search,post_campaign,post_channel,post_cust_hit_time_gmt,post_evar1,post_evar10,post_evar100,post_evar11,post_evar12,post_evar13,post_evar14,post_evar15,post_evar16,post_evar17,post_evar18,post_evar19,post_evar2,post_evar20,post_evar21,post_evar22,post_evar23,post_evar24,post_evar25,post_evar26,post_evar27,post_evar28,post_evar29,post_evar3,post_evar30,post_evar31,post_evar32,post_evar33,post_evar34,post_evar35,post_evar36,post_evar37,post_evar38,post_evar39,post_evar4,post_evar40,post_evar41,post_evar42,post_evar43,post_evar44,post_evar45,post_evar46,post_evar47,post_evar48,post_evar49,post_evar5,post_evar50,post_evar51,post_evar52,post_evar53,post_evar54,post_evar55,post_evar56,post_evar57,post_evar58,post_evar59,post_evar6,post_evar60,post_evar61,post_evar62,post_evar63,post_evar64,post_evar65,post_evar66,post_evar67,post_evar68,post_evar69,post_evar7,post_evar70,post_evar71,post_evar72,post_evar73,post_evar74,post_evar75,post_evar76,post_evar77,post_evar78,post_evar79,post_evar8,post_evar80,post_evar81,post_evar82,post_evar83,post_evar84,post_evar85,post_evar86,post_evar87,post_evar88,post_evar89,post_evar9,post_evar90,post_evar91,post_evar92,post_evar93,post_evar94,post_evar95,post_evar96,post_evar97,post_evar98,post_evar99,post_event_list,post_mobiledevice,post_page_event,post_page_url,post_pagename,post_referrer,post_search_engine,post_visid_high,post_visid_low,post_visid_type,ref_domain,ref_type,referrer,sampled_hit,secondary_hit,truncated_hit,user_agent,user_server,username,va_closer_detail,va_finder_detail,visid_new,visit_num,visit_page_num,visit_ref_domain,visit_ref_type,visit_referrer,visit_search_engine,visit_start_page_url,visit_start_time_gmt +id,browser,campaign,carrier,click_context,date_time_org,duplicate_events,duplicated_from,evar1,evar10,evar16,evar24,evar4,evar54,evar55,evar71,evar73,evar74,evar8,evar9,event_list,exclude_hit,filename,first_hit_page_url,first_hit_ref_type,first_hit_referrer,first_hit_time_gmt,geo_city,geo_country,geo_dma,geo_region,hit_source,hit_time_gmt,hitid_high,hitid_low,last_hit_time_gmt,mobile_id,mobiledevice,os,page_event,page_event_var1,page_event_var2,page_url,pagename,paid_search,post_campaign,post_channel,post_cust_hit_time_gmt,post_evar1,post_evar10,post_evar100,post_evar11,post_evar12,post_evar13,post_evar14,post_evar15,post_evar16,post_evar17,post_evar18,post_evar19,post_evar2,post_evar20,post_evar21,post_evar22,post_evar23,post_evar24,post_evar25,post_evar26,post_evar27,post_evar28,post_evar29_org,post_evar3,post_evar30,post_evar31,post_evar32,post_evar33,post_evar34,post_evar35,post_evar36,post_evar37,post_evar38,post_evar39,post_evar4,post_evar40,post_evar41,post_evar42,post_evar43,post_evar44,post_evar45,post_evar46,post_evar47,post_evar48,post_evar49,post_evar5,post_evar50,post_evar51,post_evar52,post_evar53,post_evar54,post_evar55,post_evar56,post_evar57,post_evar58,post_evar59,post_evar6,post_evar60,post_evar61,post_evar62,post_evar63,post_evar64,post_evar65,post_evar66,post_evar67,post_evar68,post_evar69,post_evar7,post_evar70,post_evar71,post_evar72,post_evar73,post_evar74,post_evar75,post_evar76,post_evar77,post_evar78,post_evar79,post_evar8,post_evar80,post_evar81,post_evar82,post_evar83,post_evar84,post_evar85,post_evar86,post_evar87,post_evar88,post_evar89,post_evar9,post_evar90,post_evar91,post_evar92,post_evar93,post_evar94,post_evar95,post_evar96,post_evar97,post_evar98,post_evar99,post_event_list,post_mobiledevice,post_page_event,post_page_url,post_pagename,post_referrer,post_search_engine,post_visid_high,post_visid_low,post_visid_type,ref_domain,ref_type,referrer,sampled_hit,secondary_hit,truncated_hit,user_agent,user_server,username,va_closer_detail,va_finder_detail,visid_new,visit_num,visit_page_num,visit_ref_domain,visit_ref_type,visit_referrer,visit_search_engine,visit_start_page_url,visit_start_time_gmt +src06.hcp_web_access_log +org06.hcp_web_access_log +hcp_web_accsess_log_ex.sql + From 6ee8085a075d52102ed7fcd66dbf484ed28972ff Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 11 Jul 2024 12:53:31 +0900 Subject: [PATCH 13/42] =?UTF-8?q?feat:HCP=E3=82=A6=E3=82=A7=E3=83=96=20?= =?UTF-8?q?=E6=8B=A1=E5=BC=B5SQL=E3=81=AE=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hcp_web/settings/hcp_web_accsess_log_ex.sql | 9 +++++++++ .../settings/hcp_web_medpass_user_mst_ex.sql | 14 ++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql create mode 100644 s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql diff --git a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql new file mode 100644 index 00000000..454fc632 --- /dev/null +++ b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql @@ -0,0 +1,9 @@ +/* date_time(日付型)がnullのレコードを抽出し、date_time(文字型) を日付型に変換してセットする。 */ +update src06.hcp_web_access_log set +date_time = STR_TO_DATE(date_time_org, '%Y-%m-%d %H:%i:%s')　+ INTERVAL 9 HOUR +where date_time_org　is　not null and date_time is null and ins_date >= (now() - INTERVAL 1 YEAR); + +/* post_evar29(日付型) がnullのレコードを抽出し、post_evar29(文字型)を日付型に変換してセットする。 */ +update src06.hcp_web_access_log set +post_evar29 = STR_TO_DATE(post_evar29_org, '%Y-%m-%dT%H:%i:%s')　+ INTERVAL 9 HOUR +where post_evar29_org　is　not null and post_evar29 is null and ins_date >= (now() - INTERVAL 1 YEAR); \ No newline at end of file diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql new file mode 100644 index 00000000..f53d0acd --- /dev/null +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql @@ -0,0 +1,14 @@ +/* 生年月日(日付型)がnullのレコードを抽出し、生年月日(文字型) を日付型に変換してセットする。 */ +update src06.hcp_web_medpass_user_mst set +BirthDate = STR_TO_DATE(BirthDate_org, '%Y-%m-%d') +where BirthDate_org　is　not null and BirthDate is　null; + +/* 登録日時(日付型) がnullのレコードを抽出し、登録日時(文字型)を日付型に変換してセットする。 */ +update src06.hcp_web_medpass_user_mst set +CreateDate = STR_TO_DATE(CreateDate_org, '%Y-%m-%dT%H:%i:%s') +where CreateDate_org　is　not null and CreateDate is　null; + +/* 最終更新日時(日付型) がnullのレコードを抽出し、最終更新日時(文字型)を日付型に変換してセットする。 */ +update src06.hcp_web_medpass_user_mst set +LastUpdate = STR_TO_DATE(LastUpdate_org, '%Y-%m-%dT%H:%i:%s') +where LastUpdate_org　is　not null and LastUpdate is　null; From 473328c639ca722dbb23cb26d17e1fc2e0bf70bf Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 11 Jul 2024 14:45:01 +0900 Subject: [PATCH 14/42] =?UTF-8?q?feat:SQL=E3=81=AE=E4=BA=88=E7=B4=84?= =?UTF-8?q?=E8=AA=9E=E3=82=92=E5=A4=A7=E6=96=87=E5=AD=97=E3=81=AB=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E3=80=81JST=E5=A4=89=E6=8F=9B=E3=81=AB=E3=81=A4?= =?UTF-8?q?=E3=81=84=E3=81=A6=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E8=A8=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../settings/hcp_web_accsess_log_ex.sql | 14 ++++++++------ .../settings/hcp_web_medpass_user_mst_ex.sql | 18 +++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql index 454fc632..b09e56ec 100644 --- a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql @@ -1,9 +1,11 @@ -/* date_time(日付型)がnullのレコードを抽出し、date_time(文字型) を日付型に変換してセットする。 */ -update src06.hcp_web_access_log set +/* date_time(日付型)がNULLのレコードを抽出し、date_time(文字型) を日付型に変換してセットする。 */ +/* date_time(文字型) はUTCの日付を取得するため、date_time(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ +UPDATE src06.hcp_web_access_log SET date_time = STR_TO_DATE(date_time_org, '%Y-%m-%d %H:%i:%s')　+ INTERVAL 9 HOUR -where date_time_org　is　not null and date_time is null and ins_date >= (now() - INTERVAL 1 YEAR); +WHERE date_time_org　IS　NOT NULL and date_time IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); -/* post_evar29(日付型) がnullのレコードを抽出し、post_evar29(文字型)を日付型に変換してセットする。 */ -update src06.hcp_web_access_log set +/* post_evar29(日付型) がNULLのレコードを抽出し、post_evar29(文字型)を日付型に変換してセットする。 */ +/* date_time(文字型) はUTCの日付を取得するため、date_time(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ +UPDATE src06.hcp_web_access_log SET post_evar29 = STR_TO_DATE(post_evar29_org, '%Y-%m-%dT%H:%i:%s')　+ INTERVAL 9 HOUR -where post_evar29_org　is　not null and post_evar29 is null and ins_date >= (now() - INTERVAL 1 YEAR); \ No newline at end of file +WHERE post_evar29_org　IS　NOT NULL and post_evar29 IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); \ No newline at end of file diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql index f53d0acd..7042aa2d 100644 --- a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql @@ -1,14 +1,14 @@ -/* 生年月日(日付型)がnullのレコードを抽出し、生年月日(文字型) を日付型に変換してセットする。 */ -update src06.hcp_web_medpass_user_mst set +/* 生年月日(日付型)がNULLのレコードを抽出し、生年月日(文字型) を日付型に変換してセットする。 */ +UPDATE src06.hcp_web_medpass_user_mst SET BirthDate = STR_TO_DATE(BirthDate_org, '%Y-%m-%d') -where BirthDate_org　is　not null and BirthDate is　null; +WHERE BirthDate_org　IS　NOT NULL and BirthDate IS　NULL; -/* 登録日時(日付型) がnullのレコードを抽出し、登録日時(文字型)を日付型に変換してセットする。 */ -update src06.hcp_web_medpass_user_mst set +/* 登録日時(日付型) がNULLのレコードを抽出し、登録日時(文字型)を日付型に変換してセットする。 */ +UPDATE src06.hcp_web_medpass_user_mst SET CreateDate = STR_TO_DATE(CreateDate_org, '%Y-%m-%dT%H:%i:%s') -where CreateDate_org　is　not null and CreateDate is　null; +WHERE CreateDate_org　IS　NOT NULL and CreateDate IS　NULL; -/* 最終更新日時(日付型) がnullのレコードを抽出し、最終更新日時(文字型)を日付型に変換してセットする。 */ -update src06.hcp_web_medpass_user_mst set +/* 最終更新日時(日付型) がNULLのレコードを抽出し、最終更新日時(文字型)を日付型に変換してセットする。 */ +UPDATE src06.hcp_web_medpass_user_mst SET LastUpdate = STR_TO_DATE(LastUpdate_org, '%Y-%m-%dT%H:%i:%s') -where LastUpdate_org　is　not null and LastUpdate is　null; +WHERE LastUpdate_org　IS　NOT NULL and LastUpdate IS　NULL; From bc7795ee3dabaec950d47836a415aba483f10fe2 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 11 Jul 2024 14:46:03 +0900 Subject: [PATCH 15/42] =?UTF-8?q?feat:hcp=5Fweb=5Faccess=5Flog=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql index b09e56ec..d1f59b83 100644 --- a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql @@ -4,8 +4,8 @@ UPDATE src06.hcp_web_access_log SET date_time = STR_TO_DATE(date_time_org, '%Y-%m-%d %H:%i:%s')　+ INTERVAL 9 HOUR WHERE date_time_org　IS　NOT NULL and date_time IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); -/* post_evar29(日付型) がNULLのレコードを抽出し、post_evar29(文字型)を日付型に変換してセットする。 */ -/* date_time(文字型) はUTCの日付を取得するため、date_time(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ +/* post_evar29(日付型)がNULLのレコードを抽出し、post_evar29(文字型)を日付型に変換してセットする。 */ +/* post_evar29(文字型)はUTCの日付を取得するため、post_evar29(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ UPDATE src06.hcp_web_access_log SET post_evar29 = STR_TO_DATE(post_evar29_org, '%Y-%m-%dT%H:%i:%s')　+ INTERVAL 9 HOUR WHERE post_evar29_org　IS　NOT NULL and post_evar29 IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); \ No newline at end of file From d3d3b95f343ce9948fd3c306cf294e87c1c94b94 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 11 Jul 2024 14:56:06 +0900 Subject: [PATCH 16/42] =?UTF-8?q?feat:=E5=85=A8=E8=A7=92=E3=82=B9=E3=83=9A?= =?UTF-8?q?=E3=83=BC=E3=82=B9=E3=82=92=E5=8D=8A=E8=A7=92=E3=82=B9=E3=83=9A?= =?UTF-8?q?=E3=83=BC=E3=82=B9=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql | 8 ++++---- s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql index d1f59b83..ba682ead 100644 --- a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql @@ -1,11 +1,11 @@ /* date_time(日付型)がNULLのレコードを抽出し、date_time(文字型) を日付型に変換してセットする。 */ /* date_time(文字型) はUTCの日付を取得するため、date_time(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ UPDATE src06.hcp_web_access_log SET -date_time = STR_TO_DATE(date_time_org, '%Y-%m-%d %H:%i:%s')　+ INTERVAL 9 HOUR -WHERE date_time_org　IS　NOT NULL and date_time IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); +date_time = STR_TO_DATE(date_time_org, '%Y-%m-%d %H:%i:%s') + INTERVAL 9 HOUR +WHERE date_time_org IS NOT NULL and date_time IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); /* post_evar29(日付型)がNULLのレコードを抽出し、post_evar29(文字型)を日付型に変換してセットする。 */ /* post_evar29(文字型)はUTCの日付を取得するため、post_evar29(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ UPDATE src06.hcp_web_access_log SET -post_evar29 = STR_TO_DATE(post_evar29_org, '%Y-%m-%dT%H:%i:%s')　+ INTERVAL 9 HOUR -WHERE post_evar29_org　IS　NOT NULL and post_evar29 IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); \ No newline at end of file +post_evar29 = STR_TO_DATE(post_evar29_org, '%Y-%m-%dT%H:%i:%s') + INTERVAL 9 HOUR +WHERE post_evar29_org IS NOT NULL and post_evar29 IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); \ No newline at end of file diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql index 7042aa2d..6b002b68 100644 --- a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql @@ -1,14 +1,14 @@ /* 生年月日(日付型)がNULLのレコードを抽出し、生年月日(文字型) を日付型に変換してセットする。 */ UPDATE src06.hcp_web_medpass_user_mst SET BirthDate = STR_TO_DATE(BirthDate_org, '%Y-%m-%d') -WHERE BirthDate_org　IS　NOT NULL and BirthDate IS　NULL; +WHERE BirthDate_org IS NOT NULL and BirthDate IS NULL; /* 登録日時(日付型) がNULLのレコードを抽出し、登録日時(文字型)を日付型に変換してセットする。 */ UPDATE src06.hcp_web_medpass_user_mst SET CreateDate = STR_TO_DATE(CreateDate_org, '%Y-%m-%dT%H:%i:%s') -WHERE CreateDate_org　IS　NOT NULL and CreateDate IS　NULL; +WHERE CreateDate_org IS NOT NULL and CreateDate IS NULL; /* 最終更新日時(日付型) がNULLのレコードを抽出し、最終更新日時(文字型)を日付型に変換してセットする。 */ UPDATE src06.hcp_web_medpass_user_mst SET LastUpdate = STR_TO_DATE(LastUpdate_org, '%Y-%m-%dT%H:%i:%s') -WHERE LastUpdate_org　IS　NOT NULL and LastUpdate IS　NULL; +WHERE LastUpdate_org IS NOT NULL and LastUpdate IS NULL; From 21126da1bae2515a1ce48020424f03b2505ad51f Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Tue, 16 Jul 2024 09:06:34 +0900 Subject: [PATCH 17/42] =?UTF-8?q?feat:=E6=8B=A1=E5=BC=B5SQL=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql | 12 ++++++++++-- .../hcp_web/settings/hcp_web_medpass_user_mst_ex.sql | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql index ba682ead..6b3d9d00 100644 --- a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql @@ -1,11 +1,19 @@ /* date_time(日付型)がNULLのレコードを抽出し、date_time(文字型) を日付型に変換してセットする。 */ /* date_time(文字型) はUTCの日付を取得するため、date_time(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ UPDATE src06.hcp_web_access_log SET -date_time = STR_TO_DATE(date_time_org, '%Y-%m-%d %H:%i:%s') + INTERVAL 9 HOUR + date_time = IF ( + medaca_common.convert_to_date(date_time_org, '%Y-%m-%d %H:%i:%s') IS NULL, + NULL, + medaca_common.convert_to_date(date_time_org, '%Y-%m-%d %H:%i:%s') + INTERVAL 9 HOUR +) WHERE date_time_org IS NOT NULL and date_time IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); /* post_evar29(日付型)がNULLのレコードを抽出し、post_evar29(文字型)を日付型に変換してセットする。 */ /* post_evar29(文字型)はUTCの日付を取得するため、post_evar29(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ UPDATE src06.hcp_web_access_log SET -post_evar29 = STR_TO_DATE(post_evar29_org, '%Y-%m-%dT%H:%i:%s') + INTERVAL 9 HOUR + post_evar29 = IF ( + medaca_common.convert_to_date(post_evar29_org, '%Y-%m-%dT%H:%i:%s.%fZ') IS NULL, + NULL, + medaca_common.convert_to_date(post_evar29_org, '%Y-%m-%dT%H:%i:%s.%fZ') + INTERVAL 9 HOUR +) WHERE post_evar29_org IS NOT NULL and post_evar29 IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); \ No newline at end of file diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql index 6b002b68..a097ce00 100644 --- a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql @@ -5,10 +5,10 @@ WHERE BirthDate_org IS NOT NULL and BirthDate IS NULL; /* 登録日時(日付型) がNULLのレコードを抽出し、登録日時(文字型)を日付型に変換してセットする。 */ UPDATE src06.hcp_web_medpass_user_mst SET -CreateDate = STR_TO_DATE(CreateDate_org, '%Y-%m-%dT%H:%i:%s') +CreateDate = STR_TO_DATE(CreateDate_org, '%Y-%m-%dT%H:%i:%s+09:00') WHERE CreateDate_org IS NOT NULL and CreateDate IS NULL; /* 最終更新日時(日付型) がNULLのレコードを抽出し、最終更新日時(文字型)を日付型に変換してセットする。 */ UPDATE src06.hcp_web_medpass_user_mst SET -LastUpdate = STR_TO_DATE(LastUpdate_org, '%Y-%m-%dT%H:%i:%s') +LastUpdate = STR_TO_DATE(LastUpdate_org, '%Y-%m-%dT%H:%i:%s+09:00') WHERE LastUpdate_org IS NOT NULL and LastUpdate IS NULL; From 8f0e0ec65fe7785a56995703adee3521d6831d2c Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Tue, 16 Jul 2024 10:25:43 +0900 Subject: [PATCH 18/42] =?UTF-8?q?feat:=E3=82=B9=E3=83=88=E3=82=A2=E3=83=89?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=83=B3=E3=82=AF=E3=82=B7=E3=83=A7=E3=83=B3?= =?UTF-8?q?=E3=81=AE=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../medaca_common/convert_to_date.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 rds_mysql/stored_procedure/medaca_common/convert_to_date.sql diff --git a/rds_mysql/stored_procedure/medaca_common/convert_to_date.sql b/rds_mysql/stored_procedure/medaca_common/convert_to_date.sql new file mode 100644 index 00000000..e5bcf3dd --- /dev/null +++ b/rds_mysql/stored_procedure/medaca_common/convert_to_date.sql @@ -0,0 +1,17 @@ +-- A5M2で実行時に[SQL] - [スラッシュ(/)のみの行でSQLを区切る]に変えてから実行する +CREATE FUNCTION medaca_common.convert_to_date(date_string VARCHAR(255), _format VARCHAR(50)) RETURNS datetime + DETERMINISTIC +BEGIN + DECLARE converted_date DATETIME; + + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + -- エラー発生時にNULLを返す + RETURN NULL; + END; + + -- 日付変換の実行 + SET converted_date = STR_TO_DATE(date_string, _format); + + RETURN converted_date; +END \ No newline at end of file From 838f5122fe999bf9600c006e524a0bdc56aecf89 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Jul 2024 16:06:04 +0900 Subject: [PATCH 19/42] =?UTF-8?q?feat:=20S3=E3=83=90=E3=82=B1=E3=83=83?= =?UTF-8?q?=E3=83=88=E9=96=93=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E8=BB=A2?= =?UTF-8?q?=E9=80=81=E5=87=A6=E7=90=86=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../transfer-receive-file-between-s3.py | 168 ++++++++++++++++++ .../bucket_transfer_config.json | 16 ++ 2 files changed, 184 insertions(+) create mode 100644 lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py create mode 100644 s3/config/bucket_transfer_config/bucket_transfer_config.json diff --git a/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py b/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py new file mode 100644 index 00000000..7cf53295 --- /dev/null +++ b/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py @@ -0,0 +1,168 @@ +import datetime +import json +import logging +import os +import re +from zoneinfo import ZoneInfo + +import boto3 + +# 環境変数 +CONFIG_BUCKET_NAME = os.environ["CONFIG_BUCKET_NAME"] +BUCKET_TRANSFER_SETTING_FILE_FOLDER = os.environ["BUCKET_TRANSFER_SETTING_FILE_FOLDER"] +BUCKET_TRANSFER_SETTING_FILE_NAME = os.environ["BUCKET_TRANSFER_SETTING_FILE_NAME"] +LOG_LEVEL = os.environ["LOG_LEVEL"] +TZ = os.environ["TZ"] +ENV = os.environ["ENV"] + +# 定数 +EXCLUSIVE_CONTROL_FILE_EXT = '.doing' + +# S3クライアント +s3_client = boto3.client('s3') + +# logger設定 +logger = logging.getLogger() + + +def log_datetime_convert_tz(*arg): + """ログに出力するタイムスタンプのロケールを変更する（JST指定）""" + return datetime.datetime.now(ZoneInfo(TZ)).timetuple() + + +formatter = logging.Formatter( + '[%(levelname)s]\t%(asctime)s\t%(message)s\n', + '%Y-%m-%d %H:%M:%S' +) +formatter.converter = log_datetime_convert_tz +for handler in logger.handlers: + handler.setFormatter(formatter) + +level = logging.getLevelName(LOG_LEVEL) +if not isinstance(level, int): + level = logging.INFO +logger.setLevel(level) + + +def get_s3_event_parameter(event: dict) -> tuple[str, str, str, str]: + """Lambdaに送信されたEvent情報からS3のイベントを取得する""" + s3_event = event["Records"][0]["s3"] + event_bucket_name: str = s3_event["bucket"]["name"] + event_object_key: str = s3_event["object"]["key"] + event_file_name: str = os.path.basename(event_object_key) + event_folder_name: str = os.path.dirname(event_object_key).split('/')[0] + + return event_bucket_name, event_object_key, event_file_name, event_folder_name + + +def delete_doing_file(event: dict) -> None: + """.doingファイルをバケット上から削除する""" + # イベント情報を取得 + ( + event_bucket_name, + event_object_key, + _, + _ + ) = get_s3_event_parameter(event) + # ⑨ メモリに保持したバケット名/フォルダ名内の「受信データファイル名.doing」ファイルを削除する + s3_client.delete_object( + Bucket=event_bucket_name, Key=f'{event_object_key}{EXCLUSIVE_CONTROL_FILE_EXT}') + + +def lambda_handler(event, context): + """Lambdaハンドラー関数""" + # ① 処理開始ログを出力する + logger.info('I-01-01 処理開始　S3バケット間ファイル転送処理') + + # ② 処理開始時に受け取ったイベント情報をログに出力する + # バケット名・フォルダ名・受信データファイル名をメモリに保持 + ( + event_bucket_name, + event_object_key, + event_file_name, + event_folder_name + ) = get_s3_event_parameter(event) + + try: + logger.info(f'I-02-01 受信バケット：{event_bucket_name}') + logger.info(f'I-02-01 フォルダ名：{event_folder_name}') + logger.info(f'I-02-01 受信ファイル名：{event_file_name}') + + # ③ S3イベントによるLambdaの重複発火防止の為、メモリに保持したバケット名/フォルダ名内に、「受信データファイル名.doing」ファイルが存在するかチェックする + try: + s3_client.head_object( + Bucket=event_bucket_name, Key=f'{event_object_key}{EXCLUSIVE_CONTROL_FILE_EXT}') + logger.error( + f'E-01-01 {event_bucket_name}/{event_object_key}は現在処理中です。処理を終了します。') + return + except Exception: + # .doingファイルが見つからなかった場合は、処理を続行する + # メモリに保持したバケット名/フォルダ名内に、「受信データファイル名.doing」ファイルを作成する + logger.info( + f'I-03-01 {event_bucket_name}/{event_object_key}を転送します') + s3_client.put_object( + Bucket=event_bucket_name, Key=f'{event_object_key}{EXCLUSIVE_CONTROL_FILE_EXT}', Body=b'') + + # ④ バケット転送設定ファイルを特定する + transfer_config_file_response = s3_client.get_object( + Bucket=CONFIG_BUCKET_NAME, Key=f'{BUCKET_TRANSFER_SETTING_FILE_FOLDER}/{BUCKET_TRANSFER_SETTING_FILE_NAME}') + transfer_config_file = json.loads( + transfer_config_file_response['Body'].read().decode('utf8')) + + # ⑤ バケット転送設定ファイルのキー[受信ファイル名正規表現パターン]と、[メモリに保持した受信ファイル名]と一致するものを取得する。 + + transfer_config = None + for key in transfer_config_file.keys(): + match_result = re.fullmatch(event_file_name, key) + if match_result is not None: + transfer_config = transfer_config_file[key] + break + + if transfer_config is None: + logger.error( + f'E-03-01 S3バケットの転送設定が見つかりません。{CONFIG_BUCKET_NAME}/{BUCKET_TRANSFER_SETTING_FILE_FOLDER}/{BUCKET_TRANSFER_SETTING_FILE_NAME}') + return + + # ⑥ 受信ファイルを、⑤でメモリ上に保持したJSONオブジェクトの設定内容に基づき、バックアップする + if transfer_config.get('backup_setting', None) is not None: + backup_setting = transfer_config['backup_setting'] + copy_source = {'Bucket': event_bucket_name, + 'Key': event_object_key} + backup_buccet = backup_setting['backup_bucet'].format(env=ENV) + backup_date_pattern = datetime.date.today().strftime( + backup_setting['date_pattern']) + s3_client.copy_object( + Bucket=backup_buccet, + Key=f'{event_folder_name}/{backup_date_pattern}/{event_file_name}', + CopySource=copy_source + ) + logger.info( + f'I-04-01 受信ファイルのバックアップ完了：：{backup_buccet}/{event_folder_name}/{backup_date_pattern}/{event_file_name}') + + # ⑦ 受信ファイルを、⑤でメモリ上に保持したJSONオブジェクトの設定内容に基づき、移動する + destination_bucket = transfer_config['destination_bucket'].format( + env=ENV) + destination_folder = transfer_config['destination_folder'] + s3_client.copy_object( + Bucket=destination_bucket, + Key=f"{destination_folder}/{event_file_name}", + CopySource=copy_source + ) + # コピー後、元のバケットからは削除する + s3_client.delete_object(Bucket=event_bucket_name, Key=event_object_key) + logger.info( + f'I-05-01 受信ファイルの転送完了：{destination_bucket}/{destination_folder}/{event_file_name}') + + # ⑧ メモリに保持したバケット名/フォルダ名内の「受信データファイル名.doing」ファイルを削除する + delete_doing_file(event) + + logger.info('I-01-06 処理終了　S3バケット間ファイル転送処理') + + except Exception as e: + logger.exception( + f'想定外のエラーが発生しました。処理を終了します。ファイル名フルパス: {event_bucket_name}/{event_object_key}　例外内容：{e}' + ) + delete_doing_file(event) + raise e + + return diff --git a/s3/config/bucket_transfer_config/bucket_transfer_config.json b/s3/config/bucket_transfer_config/bucket_transfer_config.json new file mode 100644 index 00000000..5f315d88 --- /dev/null +++ b/s3/config/bucket_transfer_config/bucket_transfer_config.json @@ -0,0 +1,16 @@ +{ + "hcpweb_accesslog_\\d{4}-\\d{2}-\\d{2}\\.(csv|CSV)": { + "Description": { + "データソース名": "hcp_web", + "送信元システム名": "Palantir", + "データの論理名（説明）": "HCPウェブアクセスログデータ", + "特記事項": "日〜金 AM1:00にMeDaCAへ連携" + }, + "destination_bucket": "mbj-newdwh2021-{env}-data", + "destination_folder": "hcp_web/target", + "backup_setting": { + "backup_bucket": "mbj-newdwh2021-{env}-backup-hcp-web", + "date_pattern": "%Y/%m/%d" + } + } +} From 42fe7804c4fe9569195ad6687a3fcafc9952136b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Jul 2024 16:09:00 +0900 Subject: [PATCH 20/42] =?UTF-8?q?fix:=20Palantir=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E5=90=8D=E6=AD=A3=E8=A6=8F=E8=A1=A8?= =?UTF-8?q?=E7=8F=BE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/config/bucket_transfer_config/bucket_transfer_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/config/bucket_transfer_config/bucket_transfer_config.json b/s3/config/bucket_transfer_config/bucket_transfer_config.json index 5f315d88..96c957ab 100644 --- a/s3/config/bucket_transfer_config/bucket_transfer_config.json +++ b/s3/config/bucket_transfer_config/bucket_transfer_config.json @@ -1,5 +1,5 @@ { - "hcpweb_accesslog_\\d{4}-\\d{2}-\\d{2}\\.(csv|CSV)": { + "hcpweb_accesslog_\\d{4}-\\d{2}-\\d{2}-\\d{6}\\.(csv|CSV)": { "Description": { "データソース名": "hcp_web", "送信元システム名": "Palantir", From 384e069fb8a6e750d1182f2d88faef47edc10433 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Tue, 16 Jul 2024 16:38:45 +0900 Subject: [PATCH 21/42] =?UTF-8?q?=E6=9C=AA=E5=8F=97=E9=A0=98=E3=83=81?= =?UTF-8?q?=E3=82=A7=E3=83=83=E3=82=AF=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily-data-unreceive-check.py | 60 ++++++++----------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py index 1bd3194e..cca7848d 100644 --- a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py +++ b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py @@ -179,9 +179,9 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d continue # ④ 受領チェック処理を行う - receive_timing = event_parameter['receive_timing'] + receive_timing = execute_date.strftime('%Y/%m/%d') logger.info(f'I-04-01 日次データ受領チェック ({event_parameter['data_source_name']}) 処理開始') - object_prefix = f'{event_parameter["check_folder_prefix"]}/{execute_date}/' + object_prefix = f'{event_parameter["check_folder_prefix"]}/{receive_timing}/' # 1.日次データバックアップ保管バケットの処理稼働月に該当するサブフォルダにあるファイル一覧を取得する logger.info(f'I-04-02 オブジェクトリストの取得　取得先：{event_parameter['check_bucket_name']}/{object_prefix}') @@ -206,18 +206,8 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d # 「③」で読み込んだファイルに記載されている全てが「④１．」で取得したリストに存在した場合 if match_result is not None: is_file_not_exists = False - # 存在したファイルの年月部分を抜き出し、チェック対象年月（処理稼働月-1)である場合 - match_group_yyyymm = match_result.group(1) - today = datetime.datetime.now(ZoneInfo(TZ)) - minus_1_month = today + relativedelta(months=-1) - check_target_yyyymm = minus_1_month.strftime('%Y%m') - if match_group_yyyymm == check_target_yyyymm: - logger.info(f'I-04-04 I/Fファイルの受領を確認しました　ファイル名：{file_name}') - match_count += 1 - else: - logger.info(f'I-04-06 I/Fファイルに不足があります　ファイル名：{file_name}') - mail_message += f'{MAIL_INDENT}{tsv_row[INDEX_DATA_NAME]}(受領年月が不正:{file_name})\n' - break + logger.info(f'I-04-04 I/Fファイルの受領を確認しました　ファイル名：{file_name}') + match_count += 1 if is_file_not_exists: logger.info(f'E-04-06 月次I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') mail_message += f'{MAIL_INDENT}{tsv_row[INDEX_DATA_NAME]}\n' @@ -320,24 +310,24 @@ def lambda_handler(event, context): # 動作確認用のコード -if __name__ == '__main__': - lambda_handler({ - "Records": [ - { - "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78", - "receiptHandle": "MessageReceiptHandle", - "body": "{\"data_source_name\":\"data_source_name\",\"check_bucket_name\":\"check_bucket_name\",\"check_folder_prefix\":\"check_folder_prefix\",\"check_target_file_list\":\"check_target_file_list\",\"notice_mail_title_template\":\"notice_mail_title_template\",\"notice_mail_body_template\":\"notice_mail_body_template\"\r\n}", - "attributes": { - "ApproximateReceiveCount": "1", - "SentTimestamp": "1523232000000", - "SenderId": "123456789012", - "ApproximateFirstReceiveTimestamp": "1523232000001" - }, - "messageAttributes": {}, - "md5OfBody": "{{{md5_of_body}}}", - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue", - "awsRegion": "us-east-1" - } -] -}, {}) +# if __name__ == '__main__': +# lambda_handler({ +# "Records": [ +# { +# "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78", +# "receiptHandle": "MessageReceiptHandle", +# "body": "{\"data_source_name\":\"data_source_name\",\"check_bucket_name\":\"check_bucket_name\",\"check_folder_prefix\":\"check_folder_prefix\",\"check_target_file_list\":\"check_target_file_list\",\"notice_mail_title_template\":\"notice_mail_title_template\",\"notice_mail_body_template\":\"notice_mail_body_template\"\r\n}", +# "attributes": { +# "ApproximateReceiveCount": "1", +# "SentTimestamp": "1523232000000", +# "SenderId": "123456789012", +# "ApproximateFirstReceiveTimestamp": "1523232000001" +# }, +# "messageAttributes": {}, +# "md5OfBody": "{{{md5_of_body}}}", +# "eventSource": "aws:sqs", +# "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue", +# "awsRegion": "us-east-1" +# } +# ] +# }, {}) From fd975d55c84022cdec0640f210f28a7e287ebc79 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Wed, 17 Jul 2024 09:23:58 +0900 Subject: [PATCH 22/42] =?UTF-8?q?=E3=83=A1=E3=83=BC=E3=83=AB=E4=BD=9C?= =?UTF-8?q?=E6=88=90=E6=A9=9F=E8=83=BD=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily-data-unreceive-check.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py index cca7848d..e30b0e6f 100644 --- a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py +++ b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py @@ -102,19 +102,17 @@ def put_success_messages_to_dynamo_db(batch_success_items: list[str]) -> bool: return True -def substitute_mail_template(mail_template: str, receive_timing: str, mail_msg: str) -> str: +def substitute_mail_template(mail_template: str, mail_msg: str) -> str: """メールテンプレートのプレースホルダーを置き換える Args: mail_template (str): 置き換え前のメールテンプレート - receive_timing (str): メールテンプレートのプレースホルダーを置き換える文言(受信タイミング) mail_msg (str): メールテンプレートのプレースホルダーを置き換える文言(ファイル一覧) Returns: str: 置き換え後のメール本文 """ substitute_dict = { - "receive_timing": receive_timing, "notice_file_names": mail_msg } mail_str = mail_template.format_map(substitute_dict) @@ -179,9 +177,9 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d continue # ④ 受領チェック処理を行う - receive_timing = execute_date.strftime('%Y/%m/%d') + receive_date = execute_date.strftime('%Y/%m/%d') logger.info(f'I-04-01 日次データ受領チェック ({event_parameter['data_source_name']}) 処理開始') - object_prefix = f'{event_parameter["check_folder_prefix"]}/{receive_timing}/' + object_prefix = f'{event_parameter["check_folder_prefix"]}/{receive_date}/' # 1.日次データバックアップ保管バケットの処理稼働月に該当するサブフォルダにあるファイル一覧を取得する logger.info(f'I-04-02 オブジェクトリストの取得　取得先：{event_parameter['check_bucket_name']}/{object_prefix}') @@ -238,8 +236,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d ) mail_title_template = (mail_title_response['Body'].read().decode('utf-8')) # 改行を取り除く - mail_title = substitute_mail_template(mail_title_template, receive_timing, mail_message) - mail_title_without_line_break = mail_title.splitlines()[0] + mail_title_without_line_break = mail_title_template.splitlines()[0] logger.info('I-05-04 通知メール（タイトル）テンプレートファイルを読み込みました') except Exception as e: logger.exception( @@ -256,7 +253,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d ) mail_body_template = (mail_body_template_response['Body'].read().decode('utf-8')) # メール本文内のプレースホルダーを置き換える - mail_body = substitute_mail_template(mail_body_template, receive_timing, mail_message) + mail_body = substitute_mail_template(mail_body_template, mail_message) logger.info('I-05-06 通知メール（本文）テンプレートファイルを読み込みました') except Exception as e: logger.exception(f'E-05-02 通知メール（本文）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') From 0d93f26bbf1ea93070dd3588aade249a2530b85a Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Wed, 17 Jul 2024 09:54:08 +0900 Subject: [PATCH 23/42] =?UTF-8?q?=E8=A8=AD=E5=AE=9A=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../log_unreceive_notice_mail_body.config | 9 +++++++++ .../log_unreceive_notice_mail_title.config | 1 + .../medpass_unreceive_notice_mail_body.config | 9 +++++++++ .../medpass_unreceive_notice_mail_title.config | 1 + .../hcp_web/unreceive_check_target/log_file_list.config | 1 + .../unreceive_check_target/medpass_file_list.config | 1 + 6 files changed, 22 insertions(+) create mode 100644 s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_body.config create mode 100644 s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_title.config create mode 100644 s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_body.config create mode 100644 s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_title.config create mode 100644 s3/config/hcp_web/unreceive_check_target/log_file_list.config create mode 100644 s3/config/hcp_web/unreceive_check_target/medpass_file_list.config diff --git a/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_body.config b/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_body.config new file mode 100644 index 00000000..724c7b04 --- /dev/null +++ b/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_body.config @@ -0,0 +1,9 @@ +【MeDaCa連携エラー通知】Palantir HCPウェブアクセスログデータ未受領 + +宛先各位 +　PalantirのHCPウェブアクセスログデータを受領できておりません。 + +　Palantir側の送信状況のご確認をお願いいたします。 + +　尚、本メールはシステム自動送信のため、返信は出来ません。 +　本件に関する問い合わせは、HCPウェブアクセスデータ担当者にお願いいたします。 \ No newline at end of file diff --git a/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_title.config b/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_title.config new file mode 100644 index 00000000..a2545bc4 --- /dev/null +++ b/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_title.config @@ -0,0 +1 @@ +【MeDaCa連携エラー通知】Parantir HCPウェブアクセスログデータ未受領 \ No newline at end of file diff --git a/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_body.config b/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_body.config new file mode 100644 index 00000000..a626f1ba --- /dev/null +++ b/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_body.config @@ -0,0 +1,9 @@ +【MeDaCa連携エラー通知】medパス HCPウェブアクセスユーザ情報未受領 + +宛先各位 +　medパス社のHCPウェブアクセスユーザ情報を受領できておりません。 + +　medパス社へ送信状況のご確認をお願いいたします。 + +　尚、本メールはシステム自動送信のため、返信は出来ません。 +　本件に関する問い合わせは、HCPウェブアクセスデータ担当者にお願いいたします。 \ No newline at end of file diff --git a/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_title.config b/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_title.config new file mode 100644 index 00000000..89f1cb28 --- /dev/null +++ b/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_title.config @@ -0,0 +1 @@ +【MeDaCa連携エラー通知】medパス HCPウェブアクセスユーザ情報未受領 \ No newline at end of file diff --git a/s3/config/hcp_web/unreceive_check_target/log_file_list.config b/s3/config/hcp_web/unreceive_check_target/log_file_list.config new file mode 100644 index 00000000..e98bbc76 --- /dev/null +++ b/s3/config/hcp_web/unreceive_check_target/log_file_list.config @@ -0,0 +1 @@ +hcpweb_accesslog_\d{4}-\d{2}-\d{2}-\d{6}\.(csv|CSV) Palantir HCPウェブアクセスログデータ diff --git a/s3/config/hcp_web/unreceive_check_target/medpass_file_list.config b/s3/config/hcp_web/unreceive_check_target/medpass_file_list.config new file mode 100644 index 00000000..4881eefd --- /dev/null +++ b/s3/config/hcp_web/unreceive_check_target/medpass_file_list.config @@ -0,0 +1 @@ +hcpweb_medpass_user_\d{14}\.(CSV|csv) medパス社ユーザ情報 From fa68ea7d8a4425cc565786c161096635a7858fb7 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Wed, 17 Jul 2024 10:12:40 +0900 Subject: [PATCH 24/42] =?UTF-8?q?feat:=E6=96=87=E5=AD=97=E5=9E=8B=E9=A0=85?= =?UTF-8?q?=E7=9B=AE(=5Forg)=E3=81=8CNULL=E5=88=A4=E5=AE=9A=E3=81=AE?= =?UTF-8?q?=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql index 6b3d9d00..67571273 100644 --- a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql @@ -6,7 +6,7 @@ UPDATE src06.hcp_web_access_log SET NULL, medaca_common.convert_to_date(date_time_org, '%Y-%m-%d %H:%i:%s') + INTERVAL 9 HOUR ) -WHERE date_time_org IS NOT NULL and date_time IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); +WHERE ins_date >= (now() - INTERVAL 1 YEAR); /* post_evar29(日付型)がNULLのレコードを抽出し、post_evar29(文字型)を日付型に変換してセットする。 */ /* post_evar29(文字型)はUTCの日付を取得するため、post_evar29(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ @@ -16,4 +16,4 @@ UPDATE src06.hcp_web_access_log SET NULL, medaca_common.convert_to_date(post_evar29_org, '%Y-%m-%dT%H:%i:%s.%fZ') + INTERVAL 9 HOUR ) -WHERE post_evar29_org IS NOT NULL and post_evar29 IS NULL and ins_date >= (now() - INTERVAL 1 YEAR); \ No newline at end of file +WHERE ins_date >= (now() - INTERVAL 1 YEAR); \ No newline at end of file From 54c3646cc4bc9a6839f536533fd6b66c1da2fa5d Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Wed, 17 Jul 2024 10:13:49 +0900 Subject: [PATCH 25/42] =?UTF-8?q?feat:NULL=E5=88=A4=E5=AE=9A=E3=81=AE?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../settings/hcp_web_medpass_user_mst_ex.sql | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql index a097ce00..eb63fb09 100644 --- a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql @@ -1,14 +1,23 @@ /* 生年月日(日付型)がNULLのレコードを抽出し、生年月日(文字型) を日付型に変換してセットする。 */ UPDATE src06.hcp_web_medpass_user_mst SET -BirthDate = STR_TO_DATE(BirthDate_org, '%Y-%m-%d') -WHERE BirthDate_org IS NOT NULL and BirthDate IS NULL; + BirthDate = IF( + medaca_common.convert_to_date(BirthDate_org, '%Y-%m-%d')IS NULL, + NULL, + medaca_common.convert_to_date(BirthDate_org, '%Y-%m-%d') +); /* 登録日時(日付型) がNULLのレコードを抽出し、登録日時(文字型)を日付型に変換してセットする。 */ UPDATE src06.hcp_web_medpass_user_mst SET -CreateDate = STR_TO_DATE(CreateDate_org, '%Y-%m-%dT%H:%i:%s+09:00') -WHERE CreateDate_org IS NOT NULL and CreateDate IS NULL; +CreateDate = IF( + medaca_common.convert_to_date(CreateDate_org, '%Y-%m-%dT%H:%i:%s+09:00')IS NULL, + NULL, + medaca_common.convert_to_date(CreateDate_org, '%Y-%m-%dT%H:%i:%s+09:00') +); /* 最終更新日時(日付型) がNULLのレコードを抽出し、最終更新日時(文字型)を日付型に変換してセットする。 */ UPDATE src06.hcp_web_medpass_user_mst SET -LastUpdate = STR_TO_DATE(LastUpdate_org, '%Y-%m-%dT%H:%i:%s+09:00') -WHERE LastUpdate_org IS NOT NULL and LastUpdate IS NULL; +LastUpdate = IF( + medaca_common.convert_to_date(LastUpdate_org, '%Y-%m-%dT%H:%i:%s+09:00')IS NULL, + NULL, + medaca_common.convert_to_date(LastUpdate_org, '%Y-%m-%dT%H:%i:%s+09:00') +); From b01f488b08d0233288ee8030064e07b2152884a0 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Wed, 17 Jul 2024 10:19:33 +0900 Subject: [PATCH 26/42] =?UTF-8?q?=E8=AA=A4=E5=AD=97=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E3=83=BB=E6=9C=AC=E6=96=87=E3=81=8B=E3=82=89=E3=82=BF=E3=82=A4?= =?UTF-8?q?=E3=83=88=E3=83=AB=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../notice_mail_template/log_unreceive_notice_mail_body.config | 2 -- .../notice_mail_template/log_unreceive_notice_mail_title.config | 2 +- .../medpass_unreceive_notice_mail_body.config | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_body.config b/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_body.config index 724c7b04..3c36acfd 100644 --- a/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_body.config +++ b/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_body.config @@ -1,5 +1,3 @@ -【MeDaCa連携エラー通知】Palantir HCPウェブアクセスログデータ未受領 - 宛先各位 　PalantirのHCPウェブアクセスログデータを受領できておりません。 diff --git a/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_title.config b/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_title.config index a2545bc4..c45c1e59 100644 --- a/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_title.config +++ b/s3/config/hcp_web/notice_mail_template/log_unreceive_notice_mail_title.config @@ -1 +1 @@ -【MeDaCa連携エラー通知】Parantir HCPウェブアクセスログデータ未受領 \ No newline at end of file +【MeDaCa連携エラー通知】Palantir HCPウェブアクセスログデータ未受領 \ No newline at end of file diff --git a/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_body.config b/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_body.config index a626f1ba..3d5119d4 100644 --- a/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_body.config +++ b/s3/config/hcp_web/notice_mail_template/medpass_unreceive_notice_mail_body.config @@ -1,5 +1,3 @@ -【MeDaCa連携エラー通知】medパス HCPウェブアクセスユーザ情報未受領 - 宛先各位 　medパス社のHCPウェブアクセスユーザ情報を受領できておりません。 From 3c761cd038b000444cafaa80bc2e87892352b5a9 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Jul 2024 10:29:59 +0900 Subject: [PATCH 27/42] =?UTF-8?q?feat:=20=E6=AD=A3=E8=A6=8F=E8=A1=A8?= =?UTF-8?q?=E7=8F=BE=E3=81=AE=E8=A7=A3=E9=87=88=E6=96=B9=E6=B3=95=E3=82=92?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=82=E3=82=B3=E3=83=B3=E3=83=95=E3=82=A3?= =?UTF-8?q?=E3=82=B0=E3=81=AE=E3=82=AD=E3=83=BC=E3=81=AE=E8=AA=A4=E3=82=8A?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../transfer-receive-file-between-s3.py | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py b/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py index 7cf53295..17f4cc4d 100644 --- a/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py +++ b/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py @@ -113,7 +113,8 @@ def lambda_handler(event, context): transfer_config = None for key in transfer_config_file.keys(): - match_result = re.fullmatch(event_file_name, key) + filename_regex = re.compile(key) + match_result = filename_regex.fullmatch(event_file_name) if match_result is not None: transfer_config = transfer_config_file[key] break @@ -121,6 +122,7 @@ def lambda_handler(event, context): if transfer_config is None: logger.error( f'E-03-01 S3バケットの転送設定が見つかりません。{CONFIG_BUCKET_NAME}/{BUCKET_TRANSFER_SETTING_FILE_FOLDER}/{BUCKET_TRANSFER_SETTING_FILE_NAME}') + delete_doing_file(event) return # ⑥ 受信ファイルを、⑤でメモリ上に保持したJSONオブジェクトの設定内容に基づき、バックアップする @@ -128,16 +130,16 @@ def lambda_handler(event, context): backup_setting = transfer_config['backup_setting'] copy_source = {'Bucket': event_bucket_name, 'Key': event_object_key} - backup_buccet = backup_setting['backup_bucet'].format(env=ENV) + backup_bucket = backup_setting['backup_bucket'].format(env=ENV) backup_date_pattern = datetime.date.today().strftime( backup_setting['date_pattern']) s3_client.copy_object( - Bucket=backup_buccet, + Bucket=backup_bucket, Key=f'{event_folder_name}/{backup_date_pattern}/{event_file_name}', CopySource=copy_source ) logger.info( - f'I-04-01 受信ファイルのバックアップ完了：：{backup_buccet}/{event_folder_name}/{backup_date_pattern}/{event_file_name}') + f'I-04-01 受信ファイルのバックアップ完了：：{backup_bucket}/{event_folder_name}/{backup_date_pattern}/{event_file_name}') # ⑦ 受信ファイルを、⑤でメモリ上に保持したJSONオブジェクトの設定内容に基づき、移動する destination_bucket = transfer_config['destination_bucket'].format( @@ -166,3 +168,47 @@ def lambda_handler(event, context): raise e return + +# 動作確認用のコード +# if __name__ == '__main__': +# lambda_handler( +# { +# "Records": [ +# { +# "eventVersion": "2.1", +# "eventSource": "aws:s3", +# "awsRegion": "ap-northeast-1", +# "eventTime": "2024-07-16T07:10:33.021Z", +# "eventName": "ObjectCreated:Put", +# "userIdentity": { +# "principalId": "AWS:AIDA4A3J5AIPDAT6MUJPZ" +# }, +# "requestParameters": { +# "sourceIPAddress": "118.238.231.215" +# }, +# "responseElements": { +# "x-amz-request-id": "0BST21P92A15BH55", +# "x-amz-id-2": "db9n9RpQxHEnq5o5ZLCeIGpuka54ghMHcbJ2Rj9aCcpjf111D4dyTZn5w5VvzV6W56rU89cSx/ihzkEHs8wk30ckbtRMYQ0byJn0UfK6bjg=" +# }, +# "s3": { +# "s3SchemaVersion": "1.0", +# "configurationId": "accesslog-receive-event2", +# "bucket": { +# "name": "mbj-newdwh2021-staging-hcp-web-receive", +# "ownerIdentity": { +# "principalId": "A1YQ10QIZBI5OE" +# }, +# "arn": "arn:aws:s3:::mbj-newdwh2021-staging-hcp-web-receive" +# }, +# "object": { +# "key": "palantir/hcpweb_accesslog_2024-07-16-071045.csv", +# "size": 597820, +# "eTag": "94299e880925b6f655c090521ff83d7a", +# "sequencer": "0066961CE8E7A670F2" +# } +# } +# } +# ] +# }, +# None +# ) From 9dbff34bf56dc32f9c380a793ca1dcae75eb5cf4 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Jul 2024 10:34:18 +0900 Subject: [PATCH 28/42] =?UTF-8?q?feat:=20=E6=8B=A1=E5=BC=B5SQL=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../settings/hcp_web_accsess_log_ex.sql | 8 +++---- .../settings/hcp_web_medpass_user_mst_ex.sql | 22 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql index 67571273..d7d27ded 100644 --- a/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_accsess_log_ex.sql @@ -1,14 +1,14 @@ -/* date_time(日付型)がNULLのレコードを抽出し、date_time(文字型) を日付型に変換してセットする。 */ +/* date_time(文字型) を日付型に変換してセットする。 */ /* date_time(文字型) はUTCの日付を取得するため、date_time(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ UPDATE src06.hcp_web_access_log SET date_time = IF ( - medaca_common.convert_to_date(date_time_org, '%Y-%m-%d %H:%i:%s') IS NULL, - NULL, + medaca_common.convert_to_date(date_time_org, '%Y-%m-%d %H:%i:%s') IS NULL, + NULL, medaca_common.convert_to_date(date_time_org, '%Y-%m-%d %H:%i:%s') + INTERVAL 9 HOUR ) WHERE ins_date >= (now() - INTERVAL 1 YEAR); -/* post_evar29(日付型)がNULLのレコードを抽出し、post_evar29(文字型)を日付型に変換してセットする。 */ +/* post_evar29(文字型)を日付型に変換してセットする。 */ /* post_evar29(文字型)はUTCの日付を取得するため、post_evar29(日付型)はMeDaCAのポリシーに合わせてJSTに変換する */ UPDATE src06.hcp_web_access_log SET post_evar29 = IF ( diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql index eb63fb09..a3de3b99 100644 --- a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst_ex.sql @@ -1,23 +1,23 @@ -/* 生年月日(日付型)がNULLのレコードを抽出し、生年月日(文字型) を日付型に変換してセットする。 */ +/* 生年月日(文字型) を日付型に変換してセットする。文字型のデータがNULLの場合はNULLをセットする */ UPDATE src06.hcp_web_medpass_user_mst SET - BirthDate = IF( - medaca_common.convert_to_date(BirthDate_org, '%Y-%m-%d')IS NULL, +BirthDate = IF( + BirthDate_org IS NULL, NULL, medaca_common.convert_to_date(BirthDate_org, '%Y-%m-%d') ); -/* 登録日時(日付型) がNULLのレコードを抽出し、登録日時(文字型)を日付型に変換してセットする。 */ +/* 登録日時(文字型)を日付型に変換してセットする。文字型のデータがNULLの場合はNULLをセットする */ UPDATE src06.hcp_web_medpass_user_mst SET CreateDate = IF( - medaca_common.convert_to_date(CreateDate_org, '%Y-%m-%dT%H:%i:%s+09:00')IS NULL, - NULL, - medaca_common.convert_to_date(CreateDate_org, '%Y-%m-%dT%H:%i:%s+09:00') + CreateDate_org IS NULL, + NULL, + medaca_common.convert_to_date(CreateDate_org, '%Y-%m-%dT%H:%i:%s+09:00') ); -/* 最終更新日時(日付型) がNULLのレコードを抽出し、最終更新日時(文字型)を日付型に変換してセットする。 */ +/* 最終更新日時(文字型)を日付型に変換してセットする。文字型のデータがNULLの場合はNULLをセットする */ UPDATE src06.hcp_web_medpass_user_mst SET LastUpdate = IF( - medaca_common.convert_to_date(LastUpdate_org, '%Y-%m-%dT%H:%i:%s+09:00')IS NULL, - NULL, - medaca_common.convert_to_date(LastUpdate_org, '%Y-%m-%dT%H:%i:%s+09:00') + LastUpdate_org IS NULL, + NULL, + medaca_common.convert_to_date(LastUpdate_org, '%Y-%m-%dT%H:%i:%s+09:00') ); From a058ef2d7521db76c4d4bce0e7dec22cc9e8d9dc Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Jul 2024 10:38:30 +0900 Subject: [PATCH 29/42] =?UTF-8?q?fix:=20=E3=83=AD=E3=82=B0=E3=81=8B?= =?UTF-8?q?=E3=82=89SNS=E3=81=ABPublish=E3=81=99=E3=82=8B=E6=A9=9F?= =?UTF-8?q?=E8=83=BD=E3=81=AESubject=E3=81=8C=E9=95=B7=E3=81=99=E3=81=8E?= =?UTF-8?q?=E3=82=8B=E3=81=A8=E3=82=A8=E3=83=A9=E3=83=BC=E3=81=AB=E3=81=AA?= =?UTF-8?q?=E3=82=8B=E5=95=8F=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/publish-from-log/publish_from_log.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lambda/publish-from-log/publish_from_log.py b/lambda/publish-from-log/publish_from_log.py index b6200cc7..aa2c4529 100644 --- a/lambda/publish-from-log/publish_from_log.py +++ b/lambda/publish-from-log/publish_from_log.py @@ -8,8 +8,9 @@ import boto3 sns_client = boto3.client('sns') + def lambda_handler(event, context): - awslogs_dict = event.get('awslogs') + awslogs_dict = event.get('awslogs') base64_data = awslogs_dict.get('data') try: decoded_gzip_data = base64.b64decode(base64_data) @@ -18,12 +19,16 @@ def lambda_handler(event, context): log_event_str = gzip.GzipFile(fileobj=BytesIO(decoded_gzip_data)).read() log_event = json.loads(log_event_str) + # SNSのSubjectパラメータは100文字までという制限があるため、100文字に切り出す + subject = f'Detect Error(or Warning) in {log_event.get("logGroup")}' + subject = subject[:97] + '...' if len(subject) > 100 else subject + publish_message = { - 'Subject': f'Detect Error(or Warning) in {log_event.get("logGroup")}', + 'Subject': subject, 'Message': '\n'.join([log.get('message') for log in log_event.get('logEvents')]), 'TopicArn': os.environ.get('SNS_TOPIC_ARN') } - + print(publish_message) - + sns_client.publish(**publish_message) From cada4f7b09d487c0992f635bc42d55720c55f41e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Jul 2024 10:43:56 +0900 Subject: [PATCH 30/42] =?UTF-8?q?style:=20=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/publish-from-log/publish_from_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/publish-from-log/publish_from_log.py b/lambda/publish-from-log/publish_from_log.py index aa2c4529..79f105b2 100644 --- a/lambda/publish-from-log/publish_from_log.py +++ b/lambda/publish-from-log/publish_from_log.py @@ -19,7 +19,7 @@ def lambda_handler(event, context): log_event_str = gzip.GzipFile(fileobj=BytesIO(decoded_gzip_data)).read() log_event = json.loads(log_event_str) - # SNSのSubjectパラメータは100文字までという制限があるため、100文字に切り出す + # SNSのSubjectパラメータは100文字までという制限があるため、100文字に切り出す(切り捨てた分は「...」に変換) subject = f'Detect Error(or Warning) in {log_event.get("logGroup")}' subject = subject[:97] + '...' if len(subject) > 100 else subject From 96a14447e68f74fc688bb32cb6a1fa327f5b2e65 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Wed, 17 Jul 2024 16:27:51 +0900 Subject: [PATCH 31/42] =?UTF-8?q?=E6=AD=A3=E8=A6=8F=E8=A1=A8=E7=8F=BE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hcp_web/unreceive_check_target/medpass_file_list.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/config/hcp_web/unreceive_check_target/medpass_file_list.config b/s3/config/hcp_web/unreceive_check_target/medpass_file_list.config index 4881eefd..43ad09d1 100644 --- a/s3/config/hcp_web/unreceive_check_target/medpass_file_list.config +++ b/s3/config/hcp_web/unreceive_check_target/medpass_file_list.config @@ -1 +1 @@ -hcpweb_medpass_user_\d{14}\.(CSV|csv) medパス社ユーザ情報 +hcp_web_medpass_user_mst_\d{14}\.(CSV|csv) medパス社ユーザ情報 From 84b2fbafab12acb6b09b3606d2080dba38f7de90 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Wed, 17 Jul 2024 18:13:36 +0900 Subject: [PATCH 32/42] =?UTF-8?q?=E3=83=AD=E3=82=B0=E8=A1=A8=E7=A4=BA?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../encise-data-unreceive-check/encise-data-unreceive-check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/encise-data-unreceive-check/encise-data-unreceive-check.py b/lambda/encise-data-unreceive-check/encise-data-unreceive-check.py index 389c82f9..1894096f 100644 --- a/lambda/encise-data-unreceive-check/encise-data-unreceive-check.py +++ b/lambda/encise-data-unreceive-check/encise-data-unreceive-check.py @@ -237,7 +237,7 @@ def encise_data_unreceive_check(records: list, execute_month: str) -> tuple[list break if is_file_not_exists: logger.info( - f'E-04-06 月次I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') + f'I-04-06 月次I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') mail_message += f'{MAIL_INDENT}{tsv_row[INDEX_DATA_NAME]}\n' row_count += 1 From 63c00f61b7d793ad61131d0fee4331f9548563a9 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Thu, 18 Jul 2024 09:54:24 +0900 Subject: [PATCH 33/42] =?UTF-8?q?Revert=20"=E3=83=AD=E3=82=B0=E8=A1=A8?= =?UTF-8?q?=E7=A4=BA=E5=A4=89=E6=9B=B4"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 84b2fbafab12acb6b09b3606d2080dba38f7de90. --- .../encise-data-unreceive-check/encise-data-unreceive-check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/encise-data-unreceive-check/encise-data-unreceive-check.py b/lambda/encise-data-unreceive-check/encise-data-unreceive-check.py index 1894096f..389c82f9 100644 --- a/lambda/encise-data-unreceive-check/encise-data-unreceive-check.py +++ b/lambda/encise-data-unreceive-check/encise-data-unreceive-check.py @@ -237,7 +237,7 @@ def encise_data_unreceive_check(records: list, execute_month: str) -> tuple[list break if is_file_not_exists: logger.info( - f'I-04-06 月次I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') + f'E-04-06 月次I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') mail_message += f'{MAIL_INDENT}{tsv_row[INDEX_DATA_NAME]}\n' row_count += 1 From 1b3995d4651b92802bc84c5f79a4445d2f7dc556 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Jul 2024 13:54:37 +0900 Subject: [PATCH 34/42] =?UTF-8?q?fix:=20=E3=83=AD=E3=82=B0=E6=96=87?= =?UTF-8?q?=E8=A8=80=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../transfer-receive-file-between-s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py b/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py index 17f4cc4d..d63cf064 100644 --- a/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py +++ b/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py @@ -158,7 +158,7 @@ def lambda_handler(event, context): # ⑧ メモリに保持したバケット名/フォルダ名内の「受信データファイル名.doing」ファイルを削除する delete_doing_file(event) - logger.info('I-01-06 処理終了　S3バケット間ファイル転送処理') + logger.info('I-06-01 処理終了　S3バケット間ファイル転送処理') except Exception as e: logger.exception( From e0ed18efaf6a640b99e027266174ea2353a68de7 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Jul 2024 14:16:48 +0900 Subject: [PATCH 35/42] =?UTF-8?q?fix:=20=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E3=83=AD=E3=82=B0=E3=81=AEID=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../transfer-receive-file-between-s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py b/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py index d63cf064..46011138 100644 --- a/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py +++ b/lambda/transfer-receive-file-between-s3/transfer-receive-file-between-s3.py @@ -162,7 +162,7 @@ def lambda_handler(event, context): except Exception as e: logger.exception( - f'想定外のエラーが発生しました。処理を終了します。ファイル名フルパス: {event_bucket_name}/{event_object_key}　例外内容：{e}' + f'E-99 想定外のエラーが発生しました。処理を終了します。ファイル名フルパス: {event_bucket_name}/{event_object_key}　例外内容：{e}' ) delete_doing_file(event) raise e From 610a0acdf18a6c3f2c3d57c45be9ea8c2abe2b12 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Thu, 18 Jul 2024 16:04:21 +0900 Subject: [PATCH 36/42] =?UTF-8?q?=E3=83=AD=E3=82=B0=E8=A1=A8=E7=A4=BA?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/daily-data-unreceive-check/daily-data-unreceive-check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py index e30b0e6f..5804ca8c 100644 --- a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py +++ b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py @@ -207,7 +207,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d logger.info(f'I-04-04 I/Fファイルの受領を確認しました　ファイル名：{file_name}') match_count += 1 if is_file_not_exists: - logger.info(f'E-04-06 月次I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') + logger.info(f'I-04-06 月次I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') mail_message += f'{MAIL_INDENT}{tsv_row[INDEX_DATA_NAME]}\n' row_count += 1 From 994c40ec55dcf6060afad2a883e64f5e5e099210 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Fri, 19 Jul 2024 14:59:19 +0900 Subject: [PATCH 37/42] =?UTF-8?q?=E4=BB=95=E6=A7=98=E3=81=AB=E5=90=88?= =?UTF-8?q?=E3=82=8F=E3=81=9B=E3=81=9F=E9=80=9A=E7=9F=A5=E8=A1=A8=E7=A4=BA?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily-data-unreceive-check.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py index 5804ca8c..1cf5a446 100644 --- a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py +++ b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py @@ -153,7 +153,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d message_id = record["messageId"] # 2.DynamoDBテーブルからレコードを取得し、処理済みメッセージかどうかを判別する if is_duplicate_message(message_id): - logger.info(f'I-02-02 受信したメッセージは既に処理済みのため、処理をスキップします。メッセージID: {message_id}') + logger.info(f'I-02-02 受信したメッセージは既に処理済みのため、処理をスキップします。メッセージID: {message_id} バケットディレクトリ: {event_parameter["check_folder_prefix"]}')# TODO ADD BUCKET DIRECTORY continue except Exception as e: logger.exception(f"E-02-01 メッセージ重複チェック処理に失敗しました　エラー内容：{e}") @@ -162,22 +162,22 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d # SQSパラメータをJSONシリアライズし、Pythonの辞書オブジェクト(イベントパラメータ)を取得する。 event_parameter = json.loads(record['body']) + receive_date = execute_date.strftime('%Y/%m/%d') # ③ 設定ファイル[受領チェック対象ファイルリスト]を読み込む try: - logger.info('I-03-01 ' +'受領チェック対象ファイルリスト読込　読込元：' + f'{CONFIG_BUCKET_NAME}/{event_parameter["check_target_file_list"]}') + logger.info('I-03-01 ' +'受領チェック対象ファイルリスト読込　読込元：' + f'{CONFIG_BUCKET_NAME}/{event_parameter["check_target_file_list"]}/{receive_date}/') check_target_file_list_response = s3_client.get_object( Bucket=CONFIG_BUCKET_NAME, Key=f'{event_parameter["check_target_file_list"]}' ) logger.info('I-03-02 受領チェック対象ファイルリストを読み込みました') except Exception as e: - logger.exception(f"E-03-01 受領チェック対象ファイルリストの読み込みに失敗しました　エラー内容：{e}") + logger.exception(f"E-03-01 受領チェック対象ファイルリストの読み込みに失敗しました　エラー内容：{e}　ファイル名：{event_parameter['check_target_file_list']}") batch_failed_items.append(make_failure_item_on_error(message_id)) continue # ④ 受領チェック処理を行う - receive_date = execute_date.strftime('%Y/%m/%d') logger.info(f'I-04-01 日次データ受領チェック ({event_parameter['data_source_name']}) 処理開始') object_prefix = f'{event_parameter["check_folder_prefix"]}/{receive_date}/' @@ -207,7 +207,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d logger.info(f'I-04-04 I/Fファイルの受領を確認しました　ファイル名：{file_name}') match_count += 1 if is_file_not_exists: - logger.info(f'I-04-06 月次I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') + logger.info(f'I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') mail_message += f'{MAIL_INDENT}{tsv_row[INDEX_DATA_NAME]}\n' row_count += 1 @@ -225,8 +225,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d continue # 1.存在した場合 - logger.info( - f'I-05-02 {execute_date} {event_parameter["data_source_name"]}データI/Fファイルに不足があるため、メール送信処理を開始します') + logger.info(f'I-05-02 {execute_date} {event_parameter["data_source_name"]} データI/Fファイルに不足があるため、メール送信処理を開始します') try: logger.info('I-05-03 ' +f'通知メール（タイトル）テンプレートファイル読込　読込元：{CONFIG_BUCKET_NAME}/{event_parameter["notice_mail_title_template"]}') @@ -240,7 +239,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d logger.info('I-05-04 通知メール（タイトル）テンプレートファイルを読み込みました') except Exception as e: logger.exception( - f'E-05-01 通知メール（タイトル）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') + f'E-05-01 通知メール（タイトル）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}　ファイル名：{event_parameter["notice_mail_title_template"]}') batch_failed_items.append( make_failure_item_on_error(message_id)) continue @@ -256,7 +255,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d mail_body = substitute_mail_template(mail_body_template, mail_message) logger.info('I-05-06 通知メール（本文）テンプレートファイルを読み込みました') except Exception as e: - logger.exception(f'E-05-02 通知メール（本文）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') + logger.exception(f'E-05-02 通知メール（本文）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}　ファイル名：{event_parameter["notice_mail_body_template"]}') batch_failed_items.append(make_failure_item_on_error(message_id)) continue From 5d55d438edba852517bc29484036de1e7477b7c2 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Fri, 19 Jul 2024 15:30:56 +0900 Subject: [PATCH 38/42] =?UTF-8?q?=E9=87=8D=E8=A4=87=E9=80=9A=E7=9F=A5?= =?UTF-8?q?=E3=81=AB=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=83=87=E3=82=A3?= =?UTF-8?q?=E3=83=AC=E3=82=AF=E3=83=88=E3=83=AA=E8=A1=A8=E7=A4=BA=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily-data-unreceive-check.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py index 1cf5a446..0266bc75 100644 --- a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py +++ b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py @@ -148,22 +148,22 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d # メール挿入用文言を格納するためのメモリを保持する mail_message = '' try: + # SQSパラメータをJSONシリアライズし、Pythonの辞書オブジェクト(イベントパラメータ)を取得する。 + event_parameter = json.loads(record['body']) + receive_date = execute_date.strftime('%Y/%m/%d') + try: # 1.SQSメッセージIDを取得する message_id = record["messageId"] # 2.DynamoDBテーブルからレコードを取得し、処理済みメッセージかどうかを判別する if is_duplicate_message(message_id): - logger.info(f'I-02-02 受信したメッセージは既に処理済みのため、処理をスキップします。メッセージID: {message_id} バケットディレクトリ: {event_parameter["check_folder_prefix"]}')# TODO ADD BUCKET DIRECTORY + logger.info(f'I-02-02 受信したメッセージは既に処理済みのため、処理をスキップします。メッセージID: {message_id} バケットディレクトリ: {event_parameter["check_bucket_name"]}/{event_parameter["check_folder_prefix"]}/{receive_date}/') continue except Exception as e: logger.exception(f"E-02-01 メッセージ重複チェック処理に失敗しました　エラー内容：{e}") batch_failed_items.append(make_failure_item_on_error(message_id)) continue - # SQSパラメータをJSONシリアライズし、Pythonの辞書オブジェクト(イベントパラメータ)を取得する。 - event_parameter = json.loads(record['body']) - receive_date = execute_date.strftime('%Y/%m/%d') - # ③ 設定ファイル[受領チェック対象ファイルリスト]を読み込む try: logger.info('I-03-01 ' +'受領チェック対象ファイルリスト読込　読込元：' + f'{CONFIG_BUCKET_NAME}/{event_parameter["check_target_file_list"]}/{receive_date}/') @@ -207,7 +207,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d logger.info(f'I-04-04 I/Fファイルの受領を確認しました　ファイル名：{file_name}') match_count += 1 if is_file_not_exists: - logger.info(f'I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') + logger.info(f'I-04-06 I/Fファイルに不足があります　ファイル名：{tsv_row[INDEX_DATA_NAME]}') mail_message += f'{MAIL_INDENT}{tsv_row[INDEX_DATA_NAME]}\n' row_count += 1 From 9388ce0ee776fdc0707eb49aba686c660b19c948 Mon Sep 17 00:00:00 2001 From: Nik Afiq Date: Fri, 19 Jul 2024 16:30:51 +0900 Subject: [PATCH 39/42] =?UTF-8?q?=E3=83=AD=E3=82=B0=E8=A1=A8=E7=A4=BA?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../daily-data-unreceive-check.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py index 0266bc75..e74550c7 100644 --- a/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py +++ b/lambda/daily-data-unreceive-check/daily-data-unreceive-check.py @@ -166,14 +166,14 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d # ③ 設定ファイル[受領チェック対象ファイルリスト]を読み込む try: - logger.info('I-03-01 ' +'受領チェック対象ファイルリスト読込　読込元：' + f'{CONFIG_BUCKET_NAME}/{event_parameter["check_target_file_list"]}/{receive_date}/') + logger.info('I-03-01 ' +'受領チェック対象ファイルリスト読込　読込元：' + f'{CONFIG_BUCKET_NAME}/{event_parameter["check_target_file_list"]}') check_target_file_list_response = s3_client.get_object( Bucket=CONFIG_BUCKET_NAME, Key=f'{event_parameter["check_target_file_list"]}' ) logger.info('I-03-02 受領チェック対象ファイルリストを読み込みました') except Exception as e: - logger.exception(f"E-03-01 受領チェック対象ファイルリストの読み込みに失敗しました　エラー内容：{e}　ファイル名：{event_parameter['check_target_file_list']}") + logger.exception(f"E-03-01 受領チェック対象ファイルリストの読み込みに失敗しました　エラー内容：{e}　読込元：{CONFIG_BUCKET_NAME}/{event_parameter["check_target_file_list"]}") batch_failed_items.append(make_failure_item_on_error(message_id)) continue @@ -239,7 +239,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d logger.info('I-05-04 通知メール（タイトル）テンプレートファイルを読み込みました') except Exception as e: logger.exception( - f'E-05-01 通知メール（タイトル）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}　ファイル名：{event_parameter["notice_mail_title_template"]}') + f'E-05-01 通知メール（タイトル）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}　読込元：{CONFIG_BUCKET_NAME}/{event_parameter["notice_mail_title_template"]}') batch_failed_items.append( make_failure_item_on_error(message_id)) continue @@ -255,7 +255,7 @@ def daily_data_unreceive_check(records: list, execute_date: str) -> tuple[list[d mail_body = substitute_mail_template(mail_body_template, mail_message) logger.info('I-05-06 通知メール（本文）テンプレートファイルを読み込みました') except Exception as e: - logger.exception(f'E-05-02 通知メール（本文）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}　ファイル名：{event_parameter["notice_mail_body_template"]}') + logger.exception(f'E-05-02 通知メール（本文）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}　読込元：{CONFIG_BUCKET_NAME}/{event_parameter["notice_mail_body_template"]}') batch_failed_items.append(make_failure_item_on_error(message_id)) continue From d049534ab9dcced2d79fd328fa57e400ba426fe5 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Mon, 22 Jul 2024 17:53:57 +0900 Subject: [PATCH 40/42] =?UTF-8?q?feat:Med=E3=83=91=E3=82=B9=E7=A4=BE?= =?UTF-8?q?=E3=83=87=E3=83=BC=E3=82=BF=E4=BB=95=E6=A7=98=E3=82=92=E5=90=B8?= =?UTF-8?q?=E5=8F=8E=E3=81=99=E3=82=8B=E3=81=9F=E3=82=81=E3=81=AE=E3=83=80?= =?UTF-8?q?=E3=83=9F=E3=83=BC=E9=A0=85=E7=9B=AE=E3=81=AE=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt index 31448f96..47fff4e1 100644 --- a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt @@ -4,9 +4,9 @@ utf-8 " LF 1 -39 -Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate,Gender,CreateDate,LastUpdate,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes -Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate_org,Gender,CreateDate_org,LastUpdate_org,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes +40 +Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate,Gender,CreateDate,LastUpdate,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes, +Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate_org,Gender,CreateDate_org,LastUpdate_org,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes,dummy01 src06.hcp_web_medpass_user_mst org06.hcp_web_medpass_user_mst hcp_web_medpass_user_mst_ex.sql From e46ac4e2e2097265608adcf0e605bce9c9430da8 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Tue, 23 Jul 2024 10:49:27 +0900 Subject: [PATCH 41/42] =?UTF-8?q?feat:=E3=83=80=E3=83=9F=E3=83=BC=E9=A0=85?= =?UTF-8?q?=E7=9B=AE=E3=82=92=E3=83=98=E3=83=83=E3=83=80=E9=A0=85=E7=9B=AE?= =?UTF-8?q?=E6=95=B0=E3=81=8B=E3=82=89=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt index 47fff4e1..2e27a137 100644 --- a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt @@ -4,7 +4,7 @@ utf-8 " LF 1 -40 +39 Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate,Gender,CreateDate,LastUpdate,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes, Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate_org,Gender,CreateDate_org,LastUpdate_org,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes,dummy01 src06.hcp_web_medpass_user_mst From 31c9ecba08714b4c531e6783deed7ffe5783b075 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Tue, 23 Jul 2024 11:43:45 +0900 Subject: [PATCH 42/42] =?UTF-8?q?feat:CSV=E9=A0=85=E7=9B=AE=E3=81=AE?= =?UTF-8?q?=E6=9C=AB=E5=B0=BE=E3=81=AE=E3=82=AB=E3=83=B3=E3=83=9E=E3=82=92?= =?UTF-8?q?=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt index 2e27a137..df490de6 100644 --- a/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt +++ b/s3/data/hcp_web/settings/hcp_web_medpass_user_mst.txt @@ -5,7 +5,7 @@ utf-8 LF 1 39 -Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate,Gender,CreateDate,LastUpdate,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes, +Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate,Gender,CreateDate,LastUpdate,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes Sub,UserName,FamilyName,GivenName,FamilyNameKana,GivenNameKana,BirthDate_org,Gender,CreateDate_org,LastUpdate_org,Job,Job_CareMgr,FacilityPrefecture,Workplace,School,GraduationYear,DoctorLicenseRegistYear,Service,SpecialistQualification,RawMailAddress,TestUser,MDBFacilityCode,FacilityPrefectureCode,FacilityMunicipalityCode,FacilityFullName,FacilityNameAbbr,CustomizedFacilityName,FacilityAddress,FacilityTelNumber,MDBFacilityPostalCode,MDBPersonalCode,MDBService,MDBServiceCategory,MDBWorkingDepartment,MDBGender,MDBOldSchool,MDBSubmitYear,MDBAllFacilityCode,RpOriginalAttributes,dummy01 src06.hcp_web_medpass_user_mst org06.hcp_web_medpass_user_mst