From e1acd610b0e17a6ce35a1cbb3777642453802065 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 1 Jul 2022 14:57:43 +0900 Subject: [PATCH 01/23] =?UTF-8?q?feat:=20Lambda=E9=96=A2=E6=95=B0=E3=81=AE?= =?UTF-8?q?=E3=83=87=E3=83=97=E3=83=AD=E3=82=A4=E3=81=BE=E3=81=A7=E3=81=AF?= =?UTF-8?q?=E5=AE=8C=E4=BA=86=E3=80=82=E9=96=A2=E6=95=B0=E3=81=AE=E4=B8=AD?= =?UTF-8?q?=E8=BA=AB=E3=81=AF=E3=81=BE=E3=81=A0=E7=A9=BA=E3=81=A3=E3=81=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- lambda/check-view-secutiry-option/Dockerfile | 15 ++ lambda/check-view-secutiry-option/Pipfile | 16 ++ .../check-view-secutiry-option/Pipfile.lock | 209 ++++++++++++++++++ lambda/check-view-secutiry-option/main.py | 5 + 5 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 lambda/check-view-secutiry-option/Dockerfile create mode 100644 lambda/check-view-secutiry-option/Pipfile create mode 100644 lambda/check-view-secutiry-option/Pipfile.lock create mode 100644 lambda/check-view-secutiry-option/main.py diff --git a/.gitignore b/.gitignore index e4f6d9fb..65e6a105 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json -lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* \ No newline at end of file +lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* +__pycache__/ \ No newline at end of file diff --git a/lambda/check-view-secutiry-option/Dockerfile b/lambda/check-view-secutiry-option/Dockerfile new file mode 100644 index 00000000..637e1e0c --- /dev/null +++ b/lambda/check-view-secutiry-option/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.9 + +ENV WORKDIR /function/ +ENV TZ="Asia/Tokyo" +WORKDIR ${WORKDIR} + +COPY Pipfile Pipfile.lock ${WORKDIR} +RUN pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv +COPY ./src $WORKDIRsrc +COPY main.py $WORKDIR + +ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] +CMD [ "main.handler" ] diff --git a/lambda/check-view-secutiry-option/Pipfile b/lambda/check-view-secutiry-option/Pipfile new file mode 100644 index 00000000..d6f3b1ee --- /dev/null +++ b/lambda/check-view-secutiry-option/Pipfile @@ -0,0 +1,16 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +awslambdaric = "*" +boto3 = "*" +pymysql = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.9" diff --git a/lambda/check-view-secutiry-option/Pipfile.lock b/lambda/check-view-secutiry-option/Pipfile.lock new file mode 100644 index 00000000..a93003e7 --- /dev/null +++ b/lambda/check-view-secutiry-option/Pipfile.lock @@ -0,0 +1,209 @@ +{ + "_meta": { + "hash": { + "sha256": "9521eb0e33f733846811775b587cd94d7660f2e612b8efcbd622fd4d19122916" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "awslambdaric": { + "hashes": [ + "sha256:059c7a66d4470169e01620d93f07424b80d302e3736cd11e68373f293a41e396", + "sha256:0e90053614f0e5e5d6d6ae6d164412ce95b5d549c6fb0f6ff4290d77c5e9d3e5", + "sha256:11a365164efec105aa670259dfe473d9609da8f6f2e468790b2dfc24969bfff1", + "sha256:19da28e8c892b1c52a9db4d2b986af303932e3a4c4632eb0c5d5eb6a673c6022", + "sha256:2eb2fdb1ae0f84669d37f193f247fa115a282a7777e051ced3a33620d6280646", + "sha256:2efff2292fc8f8484eb094ffd77808a67815353be898a7f0b33ce51b841af691", + "sha256:387b94cb0358662ae2b203f0aa2af25e80c6a2019a6b569f733ecd993a4f53d2", + "sha256:38f8ae67ecb5b4e9f7fc42746ee39765dd7ddab359cb7e8ebfda1de0f0c0b059", + "sha256:3fd0e1b3891987fa7ebb0c08d24c76af5fc17466f6efdfa9a59848dfb23930ec", + "sha256:63a82d21d66146b3fde7eb6086abd058b75bdcab4a02b02afe0e8e4a45edfb5b", + "sha256:676a741ad8f3aa27d651bcf3a2b83d5cee815f99c8b2b9abef3cb22ca7b29698", + "sha256:9b0781bd41c20a2f2a0b018464a1daa376f663bd5eb7b0b6ba78f483681b1519", + "sha256:bad98f2f94cecc90b89ac4e1d4feed96eb664e13c29b7ce232444cc9358e0d36", + "sha256:d64dcba8da9dbea62644133a48c75376a37bfe0f84096ad73bf7fc5b2eb31fc7", + "sha256:d8f280b25d8a7ae6b6ff92a9bbc6567b984264be8ef3e0fcb0402a1247f6c75d", + "sha256:dad646f566aa7ec9b7179f16ca6741a2bea148abec6ed5947f86d00607e0a9a2", + "sha256:dc7072f642fdd215387d4921bbd5ac91b96a4a705bce5e7853622d09fe59f57d", + "sha256:fbbd24446ce2f876335b178f04aa4ec7ec480afc0f9621ebfdd5f55ad4b7c06e", + "sha256:fe76893a1b42bcee4c91c6456092d2a42455818756e8f62d50e8c5adb22fa9e7" + ], + "index": "pypi", + "version": "==2.0.4" + }, + "boto3": { + "hashes": [ + "sha256:4a7cf5fddb1626d25c5935c5a82afdff9c7fe2faac2a68d37edf0264b3a85127", + "sha256:bd0b94428ae7cc57904d3c903d9393bdf4dd2b1274d1c51749f27f5bd76953e1" + ], + "index": "pypi", + "version": "==1.24.18" + }, + "botocore": { + "hashes": [ + "sha256:20a866351f9f65cfe27edc21d755de60e17a1fbb1273d73fc0006ed0d6f8ef86", + "sha256:74426179c75debd77c6dcc2d66cfd506e52962e605d2b9f2dbca290474539c8b" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.18" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "pymysql": { + "hashes": [ + "sha256:41fc3a0c5013d5f039639442321185532e3e2c8924687abe6537de157d403641", + "sha256:816927a350f38d56072aeca5dfb10221fe1dc653745853d30a216637f5d7ad36" + ], + "index": "pypi", + "version": "==1.0.2" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "simplejson": { + "hashes": [ + "sha256:034550078a11664d77bc1a8364c90bb7eef0e44c2dbb1fd0a4d92e3997088667", + "sha256:05b43d568300c1cd43f95ff4bfcff984bc658aa001be91efb3bb21df9d6288d3", + "sha256:0dd9d9c738cb008bfc0862c9b8fa6743495c03a0ed543884bf92fb7d30f8d043", + "sha256:10fc250c3edea4abc15d930d77274ddb8df4803453dde7ad50c2f5565a18a4bb", + "sha256:2862beabfb9097a745a961426fe7daf66e1714151da8bb9a0c430dde3d59c7c0", + "sha256:292c2e3f53be314cc59853bd20a35bf1f965f3bc121e007ab6fd526ed412a85d", + "sha256:2d3eab2c3fe52007d703a26f71cf649a8c771fcdd949a3ae73041ba6797cfcf8", + "sha256:2e7b57c2c146f8e4dadf84977a83f7ee50da17c8861fd7faf694d55e3274784f", + "sha256:311f5dc2af07361725033b13cc3d0351de3da8bede3397d45650784c3f21fbcf", + "sha256:344e2d920a7f27b4023c087ab539877a1e39ce8e3e90b867e0bfa97829824748", + "sha256:3fabde09af43e0cbdee407555383063f8b45bfb52c361bc5da83fcffdb4fd278", + "sha256:42b8b8dd0799f78e067e2aaae97e60d58a8f63582939af60abce4c48631a0aa4", + "sha256:4b3442249d5e3893b90cb9f72c7d6ce4d2ea144d2c0d9f75b9ae1e5460f3121a", + "sha256:55d65f9cc1b733d85ef95ab11f559cce55c7649a2160da2ac7a078534da676c8", + "sha256:5c659a0efc80aaaba57fcd878855c8534ecb655a28ac8508885c50648e6e659d", + "sha256:72d8a3ffca19a901002d6b068cf746be85747571c6a7ba12cbcf427bfb4ed971", + "sha256:75ecc79f26d99222a084fbdd1ce5aad3ac3a8bd535cd9059528452da38b68841", + "sha256:76ac9605bf2f6d9b56abf6f9da9047a8782574ad3531c82eae774947ae99cc3f", + "sha256:7d276f69bfc8c7ba6c717ba8deaf28f9d3c8450ff0aa8713f5a3280e232be16b", + "sha256:7f10f8ba9c1b1430addc7dd385fc322e221559d3ae49b812aebf57470ce8de45", + "sha256:8042040af86a494a23c189b5aa0ea9433769cc029707833f261a79c98e3375f9", + "sha256:813846738277729d7db71b82176204abc7fdae2f566e2d9fcf874f9b6472e3e6", + "sha256:845a14f6deb124a3bcb98a62def067a67462a000e0508f256f9c18eff5847efc", + "sha256:869a183c8e44bc03be1b2bbcc9ec4338e37fa8557fc506bf6115887c1d3bb956", + "sha256:8acf76443cfb5c949b6e781c154278c059b09ac717d2757a830c869ba000cf8d", + "sha256:8f713ea65958ef40049b6c45c40c206ab363db9591ff5a49d89b448933fa5746", + "sha256:934115642c8ba9659b402c8bdbdedb48651fb94b576e3b3efd1ccb079609b04a", + "sha256:9551f23e09300a9a528f7af20e35c9f79686d46d646152a0c8fc41d2d074d9b0", + "sha256:9a2b7543559f8a1c9ed72724b549d8cc3515da7daf3e79813a15bdc4a769de25", + "sha256:a55c76254d7cf8d4494bc508e7abb993a82a192d0db4552421e5139235604625", + "sha256:ad8f41c2357b73bc9e8606d2fa226233bf4d55d85a8982ecdfd55823a6959995", + "sha256:af4868da7dd53296cd7630687161d53a7ebe2e63814234631445697bd7c29f46", + "sha256:afebfc3dd3520d37056f641969ce320b071bc7a0800639c71877b90d053e087f", + "sha256:b59aa298137ca74a744c1e6e22cfc0bf9dca3a2f41f51bc92eb05695155d905a", + "sha256:bc00d1210567a4cdd215ac6e17dc00cb9893ee521cee701adfd0fa43f7c73139", + "sha256:c1cb29b1fced01f97e6d5631c3edc2dadb424d1f4421dad079cb13fc97acb42f", + "sha256:c94dc64b1a389a416fc4218cd4799aa3756f25940cae33530a4f7f2f54f166da", + "sha256:ceaa28a5bce8a46a130cd223e895080e258a88d51bf6e8de2fc54a6ef7e38c34", + "sha256:cff6453e25204d3369c47b97dd34783ca820611bd334779d22192da23784194b", + "sha256:d0b64409df09edb4c365d95004775c988259efe9be39697d7315c42b7a5e7e94", + "sha256:d4813b30cb62d3b63ccc60dd12f2121780c7a3068db692daeb90f989877aaf04", + "sha256:da3c55cdc66cfc3fffb607db49a42448785ea2732f055ac1549b69dcb392663b", + "sha256:e058c7656c44fb494a11443191e381355388443d543f6fc1a245d5d238544396", + "sha256:fed0f22bf1313ff79c7fc318f7199d6c2f96d4de3234b2f12a1eab350e597c06", + "sha256:ffd4e4877a78c84d693e491b223385e0271278f5f4e1476a4962dca6824ecfeb" + ], + "markers": "python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.17.2" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "urllib3": { + "hashes": [ + "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", + "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.9" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", + "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f" + ], + "index": "pypi", + "version": "==1.6.0" + }, + "flake8": { + "hashes": [ + "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", + "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + ], + "index": "pypi", + "version": "==4.0.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "pycodestyle": { + "hashes": [ + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" + }, + "pyflakes": { + "hashes": [ + "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", + "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + } + } +} diff --git a/lambda/check-view-secutiry-option/main.py b/lambda/check-view-secutiry-option/main.py new file mode 100644 index 00000000..31ce8430 --- /dev/null +++ b/lambda/check-view-secutiry-option/main.py @@ -0,0 +1,5 @@ +def handler(event, context): + try: + print('lambda handle') + except Exception as e: + print('exception') From f5d6e6b5252e9464fe75d3d533de3d5cfe27bba4 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 1 Jul 2022 14:57:43 +0900 Subject: [PATCH 02/23] =?UTF-8?q?feat:=20=E3=83=AD=E3=82=AC=E3=83=BC?= =?UTF-8?q?=E3=82=AF=E3=83=A9=E3=82=B9=E3=81=A8=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E3=82=AF=E3=83=A9=E3=82=B9=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/check-view-secutiry-option/main.py | 22 ++++++++++-- .../src/error/file_not_found_exception.py | 5 +++ .../src/error/madaca_exception.py | 7 ++++ .../src/logger/logger.py | 36 +++++++++++++++++++ .../src/system_vars/constants.py | 3 ++ .../src/system_vars/environments.py | 3 ++ 6 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 lambda/check-view-secutiry-option/src/error/file_not_found_exception.py create mode 100644 lambda/check-view-secutiry-option/src/error/madaca_exception.py create mode 100644 lambda/check-view-secutiry-option/src/logger/logger.py create mode 100644 lambda/check-view-secutiry-option/src/system_vars/constants.py create mode 100644 lambda/check-view-secutiry-option/src/system_vars/environments.py diff --git a/lambda/check-view-secutiry-option/main.py b/lambda/check-view-secutiry-option/main.py index 31ce8430..0ac8513b 100644 --- a/lambda/check-view-secutiry-option/main.py +++ b/lambda/check-view-secutiry-option/main.py @@ -1,5 +1,21 @@ +""" +Viewセキュリティオプション付与チェック用Lambda関数のエントリーポイント +""" + +from src.error.file_not_found_exception import FileNotFoundException +from src.error.madaca_exception import MeDaCaException +from src.logger.logger import MeDaCaLogger + + def handler(event, context): + logger = MeDaCaLogger.get_logger() try: - print('lambda handle') - except Exception as e: - print('exception') + logger.info('lambda handle') + raise FileNotFoundException('E-02-01', 'ファイル見つかりません') + except MeDaCaException as e: + logger.exception(f'exception: {e.error_id} {e}') + + +# ローカル実行用 +if __name__ == '__main__': + handler({}, {}) diff --git a/lambda/check-view-secutiry-option/src/error/file_not_found_exception.py b/lambda/check-view-secutiry-option/src/error/file_not_found_exception.py new file mode 100644 index 00000000..f3ea4438 --- /dev/null +++ b/lambda/check-view-secutiry-option/src/error/file_not_found_exception.py @@ -0,0 +1,5 @@ +from .madaca_exception import MeDaCaException + + +class FileNotFoundException(MeDaCaException): + pass diff --git a/lambda/check-view-secutiry-option/src/error/madaca_exception.py b/lambda/check-view-secutiry-option/src/error/madaca_exception.py new file mode 100644 index 00000000..b79129ae --- /dev/null +++ b/lambda/check-view-secutiry-option/src/error/madaca_exception.py @@ -0,0 +1,7 @@ +from abc import ABCMeta + + +class MeDaCaException(Exception, metaclass=ABCMeta): + def __init__(self, error_id: str, message) -> None: + super().__init__(message) + self.error_id = error_id diff --git a/lambda/check-view-secutiry-option/src/logger/logger.py b/lambda/check-view-secutiry-option/src/logger/logger.py new file mode 100644 index 00000000..4e71d919 --- /dev/null +++ b/lambda/check-view-secutiry-option/src/logger/logger.py @@ -0,0 +1,36 @@ +import datetime +import logging +from zoneinfo import ZoneInfo + +from ..system_vars.constants import (DEFAULT_TIMEZONE, LOG_DATE_FORMAT, + LOG_FORMAT) +from ..system_vars.environments import LOG_LEVEL + + +class SingletonLogger: + # インスタンス生成 + @staticmethod + def __internal_new__() -> logging.Logger: + # logger設定 + formatter = logging.Formatter( + LOG_FORMAT, + LOG_DATE_FORMAT + ) + formatter.converter = lambda: datetime.datetime.now(ZoneInfo(DEFAULT_TIMEZONE)).timetuple() + level = logging.getLevelName(LOG_LEVEL) + + logging.basicConfig(level=level, format=LOG_FORMAT, datefmt=LOG_DATE_FORMAT) + # logger.setLevel(level) + logger = logging.getLogger() + return logger + + +class MeDaCaLogger: + __unique_instance: logging.Logger = None + + @staticmethod + def get_logger(): + # インスタンス未生成の場合、唯一のインスタンスを生成する + if not MeDaCaLogger.__unique_instance: + MeDaCaLogger.__unique_instance = SingletonLogger.__internal_new__() + return MeDaCaLogger.__unique_instance diff --git a/lambda/check-view-secutiry-option/src/system_vars/constants.py b/lambda/check-view-secutiry-option/src/system_vars/constants.py new file mode 100644 index 00000000..23f2fbc7 --- /dev/null +++ b/lambda/check-view-secutiry-option/src/system_vars/constants.py @@ -0,0 +1,3 @@ +LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' +LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' +DEFAULT_TIMEZONE = "Asia/Tokyo" diff --git a/lambda/check-view-secutiry-option/src/system_vars/environments.py b/lambda/check-view-secutiry-option/src/system_vars/environments.py new file mode 100644 index 00000000..c1d95246 --- /dev/null +++ b/lambda/check-view-secutiry-option/src/system_vars/environments.py @@ -0,0 +1,3 @@ +import os + +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') From 77ad303506831a1d1435c2ed6fb1ca9d7cd60c49 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 1 Jul 2022 14:57:43 +0900 Subject: [PATCH 03/23] =?UTF-8?q?feat:=20=E5=A4=96=E3=81=AE=E3=83=A2?= =?UTF-8?q?=E3=82=B8=E3=83=A5=E3=83=BC=E3=83=AB=E3=80=81=E5=90=8C=E9=9A=8E?= =?UTF-8?q?=E5=B1=A4=E3=81=AB=E7=84=A1=E3=81=84=E3=81=A8=E3=81=A0=E3=82=81?= =?UTF-8?q?=E3=81=A3=E3=81=BD=E3=81=84.=E3=81=BE=E3=81=A0=E5=8B=95?= =?UTF-8?q?=E3=81=8B=E3=81=AA=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/check-view-secutiry-option/Dockerfile | 3 +-- .../system_vars => check-view-option}/constants.py | 0 .../environments.py | 0 .../exceptions.py} | 4 ++++ .../{ => check-view-option}/main.py | 10 +++++----- .../medaca_logger.py} | 14 +++++++------- .../src/error/file_not_found_exception.py | 5 ----- 7 files changed, 17 insertions(+), 19 deletions(-) rename lambda/check-view-secutiry-option/{src/system_vars => check-view-option}/constants.py (100%) rename lambda/check-view-secutiry-option/{src/system_vars => check-view-option}/environments.py (100%) rename lambda/check-view-secutiry-option/{src/error/madaca_exception.py => check-view-option/exceptions.py} (77%) rename lambda/check-view-secutiry-option/{ => check-view-option}/main.py (57%) rename lambda/check-view-secutiry-option/{src/logger/logger.py => check-view-option/medaca_logger.py} (75%) delete mode 100644 lambda/check-view-secutiry-option/src/error/file_not_found_exception.py diff --git a/lambda/check-view-secutiry-option/Dockerfile b/lambda/check-view-secutiry-option/Dockerfile index 637e1e0c..681c462a 100644 --- a/lambda/check-view-secutiry-option/Dockerfile +++ b/lambda/check-view-secutiry-option/Dockerfile @@ -8,8 +8,7 @@ COPY Pipfile Pipfile.lock ${WORKDIR} RUN pip install pipenv --no-cache-dir && \ pipenv install --system --deploy && \ pip uninstall -y pipenv virtualenv-clone virtualenv -COPY ./src $WORKDIRsrc -COPY main.py $WORKDIR +COPY check-view-option ./ ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] CMD [ "main.handler" ] diff --git a/lambda/check-view-secutiry-option/src/system_vars/constants.py b/lambda/check-view-secutiry-option/check-view-option/constants.py similarity index 100% rename from lambda/check-view-secutiry-option/src/system_vars/constants.py rename to lambda/check-view-secutiry-option/check-view-option/constants.py diff --git a/lambda/check-view-secutiry-option/src/system_vars/environments.py b/lambda/check-view-secutiry-option/check-view-option/environments.py similarity index 100% rename from lambda/check-view-secutiry-option/src/system_vars/environments.py rename to lambda/check-view-secutiry-option/check-view-option/environments.py diff --git a/lambda/check-view-secutiry-option/src/error/madaca_exception.py b/lambda/check-view-secutiry-option/check-view-option/exceptions.py similarity index 77% rename from lambda/check-view-secutiry-option/src/error/madaca_exception.py rename to lambda/check-view-secutiry-option/check-view-option/exceptions.py index b79129ae..131a0126 100644 --- a/lambda/check-view-secutiry-option/src/error/madaca_exception.py +++ b/lambda/check-view-secutiry-option/check-view-option/exceptions.py @@ -5,3 +5,7 @@ class MeDaCaException(Exception, metaclass=ABCMeta): def __init__(self, error_id: str, message) -> None: super().__init__(message) self.error_id = error_id + + +class FileNotFoundException(MeDaCaException): + pass diff --git a/lambda/check-view-secutiry-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py similarity index 57% rename from lambda/check-view-secutiry-option/main.py rename to lambda/check-view-secutiry-option/check-view-option/main.py index 0ac8513b..fa6e8156 100644 --- a/lambda/check-view-secutiry-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -2,9 +2,8 @@ Viewセキュリティオプション付与チェック用Lambda関数のエントリーポイント """ -from src.error.file_not_found_exception import FileNotFoundException -from src.error.madaca_exception import MeDaCaException -from src.logger.logger import MeDaCaLogger +from exceptions import FileNotFoundException, MeDaCaException +from medaca_logger import MeDaCaLogger def handler(event, context): @@ -12,8 +11,9 @@ def handler(event, context): try: logger.info('lambda handle') raise FileNotFoundException('E-02-01', 'ファイル見つかりません') - except MeDaCaException as e: - logger.exception(f'exception: {e.error_id} {e}') + except Exception as e: + logger.exception(f'exception: {e}') + # logger.exception(f'exception: {e.error_id} {e}') # ローカル実行用 diff --git a/lambda/check-view-secutiry-option/src/logger/logger.py b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py similarity index 75% rename from lambda/check-view-secutiry-option/src/logger/logger.py rename to lambda/check-view-secutiry-option/check-view-option/medaca_logger.py index 4e71d919..701ecb5e 100644 --- a/lambda/check-view-secutiry-option/src/logger/logger.py +++ b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py @@ -2,9 +2,8 @@ import datetime import logging from zoneinfo import ZoneInfo -from ..system_vars.constants import (DEFAULT_TIMEZONE, LOG_DATE_FORMAT, - LOG_FORMAT) -from ..system_vars.environments import LOG_LEVEL +from constants import DEFAULT_TIMEZONE, LOG_DATE_FORMAT, LOG_FORMAT +from environments import LOG_LEVEL class SingletonLogger: @@ -12,16 +11,17 @@ class SingletonLogger: @staticmethod def __internal_new__() -> logging.Logger: # logger設定 + logger = logging.getLogger() formatter = logging.Formatter( LOG_FORMAT, LOG_DATE_FORMAT ) formatter.converter = lambda: datetime.datetime.now(ZoneInfo(DEFAULT_TIMEZONE)).timetuple() + for handler in logger.handlers: + handler.setFormatter(formatter) level = logging.getLevelName(LOG_LEVEL) - - logging.basicConfig(level=level, format=LOG_FORMAT, datefmt=LOG_DATE_FORMAT) - # logger.setLevel(level) - logger = logging.getLogger() + print(level) + logger.setLevel(level) return logger diff --git a/lambda/check-view-secutiry-option/src/error/file_not_found_exception.py b/lambda/check-view-secutiry-option/src/error/file_not_found_exception.py deleted file mode 100644 index f3ea4438..00000000 --- a/lambda/check-view-secutiry-option/src/error/file_not_found_exception.py +++ /dev/null @@ -1,5 +0,0 @@ -from .madaca_exception import MeDaCaException - - -class FileNotFoundException(MeDaCaException): - pass From af34d6a3d5fba7766e03736489478e8037c472ab Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 1 Jul 2022 14:57:43 +0900 Subject: [PATCH 04/23] =?UTF-8?q?fix:=20=E3=83=AD=E3=82=B0=E5=87=BA?= =?UTF-8?q?=E3=81=9B=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/check-view-secutiry-option/check-view-option/main.py | 5 ++--- .../check-view-option/medaca_logger.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index fa6e8156..d39b4ce4 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -11,9 +11,8 @@ def handler(event, context): try: logger.info('lambda handle') raise FileNotFoundException('E-02-01', 'ファイル見つかりません') - except Exception as e: - logger.exception(f'exception: {e}') - # logger.exception(f'exception: {e.error_id} {e}') + except MeDaCaException as e: + logger.exception(f'exception: {e.error_id} {e}') # ローカル実行用 diff --git a/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py index 701ecb5e..5546d07a 100644 --- a/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py +++ b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py @@ -16,7 +16,7 @@ class SingletonLogger: LOG_FORMAT, LOG_DATE_FORMAT ) - formatter.converter = lambda: datetime.datetime.now(ZoneInfo(DEFAULT_TIMEZONE)).timetuple() + formatter.converter = lambda *arg: datetime.datetime.now(ZoneInfo(DEFAULT_TIMEZONE)).timetuple() for handler in logger.handlers: handler.setFormatter(formatter) level = logging.getLevelName(LOG_LEVEL) From 3337a0abff2db606894301179dc36ca7ced6880b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 1 Jul 2022 14:57:43 +0900 Subject: [PATCH 05/23] =?UTF-8?q?feat:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E5=AE=9F=E8=A1=8C=E7=94=A8=E3=81=AE=E3=83=AD=E3=82=B0?= =?UTF-8?q?=E3=83=8F=E3=83=B3=E3=83=89=E3=83=A9=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/medaca_logger.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py index 5546d07a..5ab4eb3e 100644 --- a/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py +++ b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py @@ -1,5 +1,6 @@ import datetime import logging +import sys from zoneinfo import ZoneInfo from constants import DEFAULT_TIMEZONE, LOG_DATE_FORMAT, LOG_FORMAT @@ -17,10 +18,14 @@ class SingletonLogger: LOG_DATE_FORMAT ) formatter.converter = lambda *arg: datetime.datetime.now(ZoneInfo(DEFAULT_TIMEZONE)).timetuple() + # ローカル環境で動かす場合、標準出力ハンドラーを追加する + # AWS Lambda上では`LambdaLoggerHandler`がデフォルトでセットされている + if len(sys.argv) == 2 and sys.argv[1] == 'local': + localHandler = logging.StreamHandler() + logger.addHandler(localHandler) for handler in logger.handlers: handler.setFormatter(formatter) level = logging.getLevelName(LOG_LEVEL) - print(level) logger.setLevel(level) return logger From 3ca1a8d7b6e49805286debe90f1a08b8dacb6bd2 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Sat, 2 Jul 2022 16:48:33 +0900 Subject: [PATCH 06/23] =?UTF-8?q?feat:=20=E8=A8=AD=E5=AE=9A=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=83=90=E3=82=B1=E3=83=83=E3=83=88?= =?UTF-8?q?=E3=81=8B=E3=82=89=E3=81=AE=E5=8F=96=E5=BE=97=E5=87=A6=E7=90=86?= =?UTF-8?q?=E8=BF=BD=E8=A8=98=E3=80=82JSON=E3=81=AE=E3=83=91=E3=83=BC?= =?UTF-8?q?=E3=82=B9=E3=81=AF=E3=81=93=E3=82=8C=E3=81=8B=E3=82=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- .../check-view-option/aws/s3.py | 31 +++++++++++++++++++ .../check-view-option/constants.py | 20 ++++++++++++ .../check-view-option/environments.py | 18 ++++++++++- .../check-view-option/main.py | 19 +++++++++--- .../check-view-option/medaca_logger.py | 31 +++++++++++++++---- lambda/sap-data-decrypt/datadecrypt/main.py | 23 +++++++++----- .../view_check/check_target_schemas.json | 3 ++ 8 files changed, 128 insertions(+), 20 deletions(-) create mode 100644 lambda/check-view-secutiry-option/check-view-option/aws/s3.py create mode 100644 s3/config/view_check/check_target_schemas.json diff --git a/.gitignore b/.gitignore index 65e6a105..a85e0c0b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* -__pycache__/ \ No newline at end of file +__pycache__/ +.env \ No newline at end of file diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py new file mode 100644 index 00000000..6ace3816 --- /dev/null +++ b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py @@ -0,0 +1,31 @@ +import boto3 +import environments +import exceptions +from botocore.exceptions import ClientError +from constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY + + +class S3Resource: + + def __init__(self, bucket_name: str) -> None: + self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) + self.__s3_bucket = self.__s3_resource.Bucket(bucket_name) + + def get_object(self, object_key: str): + s3_object = self.__s3_bucket.Object(object_key) + response = s3_object.get() + return response[S3_RESPONSE_BODY].read() + + +class ConfigBucket: + __s3_resource: S3Resource = None + + def __init__(self) -> None: + self.__s3_resource = S3Resource(environments.CONFIG_BUCKET_NAME) + + def read_check_target_schema_names(self): + try: + return self.__s3_resource.get_object(environments.CHECK_TARGET_SCHEMA_NAMES_PATH) + except ClientError as error: + if error.response['Error']['Code'] == 'NoSuchKey': + raise exceptions.FileNotFoundException('E-02-01', f'チェック対象スキーマ名ファイルの読み込みに失敗しました　エラー内容：{error}') diff --git a/lambda/check-view-secutiry-option/check-view-option/constants.py b/lambda/check-view-secutiry-option/check-view-option/constants.py index 23f2fbc7..1672c5a4 100644 --- a/lambda/check-view-secutiry-option/check-view-option/constants.py +++ b/lambda/check-view-secutiry-option/check-view-option/constants.py @@ -1,3 +1,23 @@ +# logger LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' DEFAULT_TIMEZONE = "Asia/Tokyo" +LOG_LEVEL = 'LOG_LEVEL' +LOG_LEVEL_INFO = 'INFO' + +# environments +CHECK_TARGET_SCHEMA_NAMES_PATH = 'CHECK_TARGET_SCHEMA_NAMES_PATH' +CONFIG_BUCKET_NAME = 'CONFIG_BUCKET_NAME' +LOG_LEVEL = 'LOG_LEVEL' +MBJ_NOTICE_TOPIC = 'MBJ_NOTICE_TOPIC' +NDS_NOTICE_TOPIC = 'NDS_NOTICE_TOPIC' +NOTICE_MAIL_BODY_TEMPLATE_PATH = 'NOTICE_MAIL_BODY_TEMPLATE_PATH' +NOTICE_MAIL_TITLE_TEMPLATE_PATH = 'NOTICE_MAIL_TITLE_TEMPLATE_PATH' +PARAM_NAME_DB_HOST = 'PARAM_NAME_DB_HOST' +PARAM_NAME_DB_USER_NAME = 'PARAM_NAME_DB_USER_NAME' +PARAM_NAME_DB_USER_PASSWORD = 'PARAM_NAME_DB_USER_PASSWORD' + +# system var +AWS_RESOURCE_S3 = 's3' +S3_RESPONSE_BODY = 'Body' +UTF8 = 'utf-8' diff --git a/lambda/check-view-secutiry-option/check-view-option/environments.py b/lambda/check-view-secutiry-option/check-view-option/environments.py index c1d95246..56edb18c 100644 --- a/lambda/check-view-secutiry-option/check-view-option/environments.py +++ b/lambda/check-view-secutiry-option/check-view-option/environments.py @@ -1,3 +1,19 @@ import os -LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') +from constants import (CHECK_TARGET_SCHEMA_NAMES_PATH, CONFIG_BUCKET_NAME, + LOG_LEVEL, LOG_LEVEL_INFO, MBJ_NOTICE_TOPIC, + NDS_NOTICE_TOPIC, NOTICE_MAIL_BODY_TEMPLATE_PATH, + NOTICE_MAIL_TITLE_TEMPLATE_PATH, PARAM_NAME_DB_HOST, + PARAM_NAME_DB_USER_NAME, PARAM_NAME_DB_USER_PASSWORD) + +LOG_LEVEL = os.environ.get(LOG_LEVEL, LOG_LEVEL_INFO) +CHECK_TARGET_SCHEMA_NAMES_PATH = os.environ[CHECK_TARGET_SCHEMA_NAMES_PATH] +CONFIG_BUCKET_NAME = os.environ[CONFIG_BUCKET_NAME] +MBJ_NOTICE_TOPIC = os.environ[MBJ_NOTICE_TOPIC] +NDS_NOTICE_TOPIC = os.environ[NDS_NOTICE_TOPIC] +NOTICE_MAIL_BODY_TEMPLATE_PATH = os.environ[NOTICE_MAIL_BODY_TEMPLATE_PATH] +NOTICE_MAIL_TITLE_TEMPLATE_PATH = os.environ[NOTICE_MAIL_TITLE_TEMPLATE_PATH] + +PARAM_NAME_DB_HOST = os.environ[PARAM_NAME_DB_HOST] +PARAM_NAME_DB_USER_NAME = os.environ[PARAM_NAME_DB_USER_NAME] +PARAM_NAME_DB_USER_PASSWORD = os.environ[PARAM_NAME_DB_USER_PASSWORD] diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index d39b4ce4..b870a3bf 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -2,17 +2,28 @@ Viewセキュリティオプション付与チェック用Lambda関数のエントリーポイント """ -from exceptions import FileNotFoundException, MeDaCaException +from aws.s3 import ConfigBucket +from exceptions import MeDaCaException from medaca_logger import MeDaCaLogger def handler(event, context): logger = MeDaCaLogger.get_logger() + try: - logger.info('lambda handle') - raise FileNotFoundException('E-02-01', 'ファイル見つかりません') + logger.info('I-01-01', '処理開始　Viewセキュリティオプション付与チェック') + logger.info('I-01-02', 'チェック対象スキーマ名ファイルを読み込み 開始') + config_bucket = ConfigBucket() + check_target_schema_names = config_bucket.read_check_target_schema_names() + print(check_target_schema_names) + except MeDaCaException as e: - logger.exception(f'exception: {e.error_id} {e}') + logger.exception(e.error_id, e) + raise e + except Exception as e: + logger.exception('E-99', f'想定外のエラーが発生しました　エラー内容：{e}') + finally: + logger.info('I-06-01', '処理終了　Viewセキュリティオプション付与チェック') # ローカル実行用 diff --git a/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py index 5ab4eb3e..b46d893e 100644 --- a/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py +++ b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py @@ -8,9 +8,9 @@ from environments import LOG_LEVEL class SingletonLogger: - # インスタンス生成 - @staticmethod - def __internal_new__() -> logging.Logger: + __logger: logging.Logger = None + + def __init__(self) -> None: # logger設定 logger = logging.getLogger() formatter = logging.Formatter( @@ -27,15 +27,34 @@ class SingletonLogger: handler.setFormatter(formatter) level = logging.getLevelName(LOG_LEVEL) logger.setLevel(level) - return logger + + self.__logger = logger + + def debug(self, log_id: str, msg: str): + self._log(logging.DEBUG, log_id, msg) + + def info(self, log_id: str, msg: str): + self._log(logging.INFO, log_id, msg) + + def warning(self, log_id: str, msg: str): + self._log(logging.WARNING, log_id, msg) + + def error(self, log_id: str, msg: str): + self._log(logging.ERROR, log_id, msg) + + def exception(self, log_id: str, msg: str): + self._log(logging.ERROR, log_id, msg, exc_info=True) + + def _log(self, log_level: int, log_id: str, msg: str, exc_info=False): + self.__logger.log(log_level, f'{log_id} {msg}', exc_info=exc_info) class MeDaCaLogger: __unique_instance: logging.Logger = None @staticmethod - def get_logger(): + def get_logger() -> SingletonLogger: # インスタンス未生成の場合、唯一のインスタンスを生成する if not MeDaCaLogger.__unique_instance: - MeDaCaLogger.__unique_instance = SingletonLogger.__internal_new__() + MeDaCaLogger.__unique_instance = SingletonLogger() return MeDaCaLogger.__unique_instance diff --git a/lambda/sap-data-decrypt/datadecrypt/main.py b/lambda/sap-data-decrypt/datadecrypt/main.py index ec4d24ae..f1f2de59 100644 --- a/lambda/sap-data-decrypt/datadecrypt/main.py +++ b/lambda/sap-data-decrypt/datadecrypt/main.py @@ -1,12 +1,12 @@ -import logging -import os -import boto3 -import gnupg import datetime import logging +import os +import traceback from abc import * from zoneinfo import ZoneInfo -import traceback + +import boto3 +import gnupg # 環境変数 SECRET_KEY_FILE_BUCKET_NAME = os.environ["SECRET_KEY_FILE_BUCKET_NAME"] @@ -42,8 +42,12 @@ sns_client = boto3.client('sns') # logger設定 logger = logging.getLogger() + + def custome_time(*arg): return datetime.datetime.now(ZoneInfo("Asia/Tokyo")).timetuple() + + formatter = logging.Formatter( '[%(levelname)s]\t%(asctime)s\t%(message)s\n', '%Y-%m-%d %H:%M:%S' @@ -136,7 +140,8 @@ def handler(event, context): 'Key': s3_event.file_path } backup_file_key = f'{s3_event.data_source_name}/{execute_date}/{s3_event.file_name}' - logger.info(f'I-07-04 PGP暗号化ファイル移動　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{SAP_DATA_BACKUP_BUCKET_NAME}/{backup_file_key}') + logger.info( + f'I-07-04 PGP暗号化ファイル移動　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{SAP_DATA_BACKUP_BUCKET_NAME}/{backup_file_key}') backup_file_obj = s3_resource.Object(SAP_DATA_BACKUP_BUCKET_NAME, backup_file_key) backup_file_obj.copy(copy_source) s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path) @@ -172,7 +177,8 @@ def create_status_file(s3_event, extension) -> None: result_error_key = s3_event.data_source_name + DIRECTORY_RECV + result_error_file_name result_error_obj = s3_resource.Object(s3_event.bucket_name, result_error_key) result_error_obj.put(Body='') - logger.error(f'E-ERR-01 recvディレクトリにエラーファイルを作成しました　ファイル名：{result_error_file_name}　出力先：{s3_event.bucket_name}/{result_error_key}') + logger.error( + f'E-ERR-01 recvディレクトリにエラーファイルを作成しました　ファイル名：{result_error_file_name}　出力先：{s3_event.bucket_name}/{result_error_key}') except Exception as e: logger.error(f'E-96 エラーステータスファイルの作成に失敗しました　エラー内容：{e}') traceback.print_exc() @@ -191,7 +197,8 @@ def move_encrypt_file(s3_event) -> None: error_obj = s3_resource.Object(s3_event.bucket_name, error_key) error_obj.copy(copy_source) s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path) - logger.error(f'E-ERR-02 recv_errorディレクトリにファイルを移動しました　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{s3_event.bucket_name}/{error_key}') + logger.error( + f'E-ERR-02 recv_errorディレクトリにファイルを移動しました　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{s3_event.bucket_name}/{error_key}') except Exception as e: logger.error(f'E-97 PGP暗号化ファイルの移動に失敗しました　エラー内容：{e}') traceback.print_exc() diff --git a/s3/config/view_check/check_target_schemas.json b/s3/config/view_check/check_target_schemas.json new file mode 100644 index 00000000..4a7df05b --- /dev/null +++ b/s3/config/view_check/check_target_schemas.json @@ -0,0 +1,3 @@ +{ + "check_target_schemas": ["custom01", "custom02", "custom03"] +} From b33a3dae1928ec643c44dcd52e2d9c8611e8fb22 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Sat, 2 Jul 2022 16:51:07 +0900 Subject: [PATCH 07/23] =?UTF-8?q?style:=20=E3=83=95=E3=82=A9=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=83=E3=83=88=E3=81=8C=E9=81=A9=E7=94=A8=E3=81=95?= =?UTF-8?q?=E3=82=8C=E3=81=9F=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3=20?= =?UTF-8?q?=E6=9C=AC=E5=BD=93=E3=81=AF=E7=9B=B4=E3=81=97=E3=81=9F=E3=81=84?= =?UTF-8?q?=E3=81=91=E3=81=A9=E5=BD=B1=E9=9F=BF=E3=81=8C=E3=81=82=E3=82=8B?= =?UTF-8?q?=E3=81=A8=E3=81=84=E3=81=91=E3=81=AA=E3=81=84=E3=81=AE=E3=81=A7?= =?UTF-8?q?=E3=82=82=E3=81=A8=E3=81=AB=E6=88=BB=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/sap-data-decrypt/datadecrypt/main.py | 23 +++++++-------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/lambda/sap-data-decrypt/datadecrypt/main.py b/lambda/sap-data-decrypt/datadecrypt/main.py index f1f2de59..ec4d24ae 100644 --- a/lambda/sap-data-decrypt/datadecrypt/main.py +++ b/lambda/sap-data-decrypt/datadecrypt/main.py @@ -1,12 +1,12 @@ -import datetime import logging import os -import traceback -from abc import * -from zoneinfo import ZoneInfo - import boto3 import gnupg +import datetime +import logging +from abc import * +from zoneinfo import ZoneInfo +import traceback # 環境変数 SECRET_KEY_FILE_BUCKET_NAME = os.environ["SECRET_KEY_FILE_BUCKET_NAME"] @@ -42,12 +42,8 @@ sns_client = boto3.client('sns') # logger設定 logger = logging.getLogger() - - def custome_time(*arg): return datetime.datetime.now(ZoneInfo("Asia/Tokyo")).timetuple() - - formatter = logging.Formatter( '[%(levelname)s]\t%(asctime)s\t%(message)s\n', '%Y-%m-%d %H:%M:%S' @@ -140,8 +136,7 @@ def handler(event, context): 'Key': s3_event.file_path } backup_file_key = f'{s3_event.data_source_name}/{execute_date}/{s3_event.file_name}' - logger.info( - f'I-07-04 PGP暗号化ファイル移動　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{SAP_DATA_BACKUP_BUCKET_NAME}/{backup_file_key}') + logger.info(f'I-07-04 PGP暗号化ファイル移動　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{SAP_DATA_BACKUP_BUCKET_NAME}/{backup_file_key}') backup_file_obj = s3_resource.Object(SAP_DATA_BACKUP_BUCKET_NAME, backup_file_key) backup_file_obj.copy(copy_source) s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path) @@ -177,8 +172,7 @@ def create_status_file(s3_event, extension) -> None: result_error_key = s3_event.data_source_name + DIRECTORY_RECV + result_error_file_name result_error_obj = s3_resource.Object(s3_event.bucket_name, result_error_key) result_error_obj.put(Body='') - logger.error( - f'E-ERR-01 recvディレクトリにエラーファイルを作成しました　ファイル名：{result_error_file_name}　出力先：{s3_event.bucket_name}/{result_error_key}') + logger.error(f'E-ERR-01 recvディレクトリにエラーファイルを作成しました　ファイル名：{result_error_file_name}　出力先：{s3_event.bucket_name}/{result_error_key}') except Exception as e: logger.error(f'E-96 エラーステータスファイルの作成に失敗しました　エラー内容：{e}') traceback.print_exc() @@ -197,8 +191,7 @@ def move_encrypt_file(s3_event) -> None: error_obj = s3_resource.Object(s3_event.bucket_name, error_key) error_obj.copy(copy_source) s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path) - logger.error( - f'E-ERR-02 recv_errorディレクトリにファイルを移動しました　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{s3_event.bucket_name}/{error_key}') + logger.error(f'E-ERR-02 recv_errorディレクトリにファイルを移動しました　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{s3_event.bucket_name}/{error_key}') except Exception as e: logger.error(f'E-97 PGP暗号化ファイルの移動に失敗しました　エラー内容：{e}') traceback.print_exc() From c19cf90a03bf9ca25b99e64417eb15ba0deaabd9 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Sat, 2 Jul 2022 16:58:29 +0900 Subject: [PATCH 08/23] =?UTF-8?q?refactor:=20=E3=83=AA=E3=83=86=E3=83=A9?= =?UTF-8?q?=E3=83=AB=E5=80=A4=E3=82=92=E5=AE=9A=E6=95=B0=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/constants.py | 4 ++-- .../check-view-option/environments.py | 5 ++++- .../check-view-option/medaca_logger.py | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lambda/check-view-secutiry-option/check-view-option/constants.py b/lambda/check-view-secutiry-option/check-view-option/constants.py index 1672c5a4..2ef6a079 100644 --- a/lambda/check-view-secutiry-option/check-view-option/constants.py +++ b/lambda/check-view-secutiry-option/check-view-option/constants.py @@ -1,8 +1,6 @@ # logger LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' -DEFAULT_TIMEZONE = "Asia/Tokyo" -LOG_LEVEL = 'LOG_LEVEL' LOG_LEVEL_INFO = 'INFO' # environments @@ -16,8 +14,10 @@ NOTICE_MAIL_TITLE_TEMPLATE_PATH = 'NOTICE_MAIL_TITLE_TEMPLATE_PATH' PARAM_NAME_DB_HOST = 'PARAM_NAME_DB_HOST' PARAM_NAME_DB_USER_NAME = 'PARAM_NAME_DB_USER_NAME' PARAM_NAME_DB_USER_PASSWORD = 'PARAM_NAME_DB_USER_PASSWORD' +TZ = 'TZ' # system var AWS_RESOURCE_S3 = 's3' S3_RESPONSE_BODY = 'Body' UTF8 = 'utf-8' +LAUNCH_ON_LOCAL = 'local' diff --git a/lambda/check-view-secutiry-option/check-view-option/environments.py b/lambda/check-view-secutiry-option/check-view-option/environments.py index 56edb18c..87156c19 100644 --- a/lambda/check-view-secutiry-option/check-view-option/environments.py +++ b/lambda/check-view-secutiry-option/check-view-option/environments.py @@ -4,7 +4,8 @@ from constants import (CHECK_TARGET_SCHEMA_NAMES_PATH, CONFIG_BUCKET_NAME, LOG_LEVEL, LOG_LEVEL_INFO, MBJ_NOTICE_TOPIC, NDS_NOTICE_TOPIC, NOTICE_MAIL_BODY_TEMPLATE_PATH, NOTICE_MAIL_TITLE_TEMPLATE_PATH, PARAM_NAME_DB_HOST, - PARAM_NAME_DB_USER_NAME, PARAM_NAME_DB_USER_PASSWORD) + PARAM_NAME_DB_USER_NAME, PARAM_NAME_DB_USER_PASSWORD, + TZ) LOG_LEVEL = os.environ.get(LOG_LEVEL, LOG_LEVEL_INFO) CHECK_TARGET_SCHEMA_NAMES_PATH = os.environ[CHECK_TARGET_SCHEMA_NAMES_PATH] @@ -17,3 +18,5 @@ NOTICE_MAIL_TITLE_TEMPLATE_PATH = os.environ[NOTICE_MAIL_TITLE_TEMPLATE_PATH] PARAM_NAME_DB_HOST = os.environ[PARAM_NAME_DB_HOST] PARAM_NAME_DB_USER_NAME = os.environ[PARAM_NAME_DB_USER_NAME] PARAM_NAME_DB_USER_PASSWORD = os.environ[PARAM_NAME_DB_USER_PASSWORD] + +TZ = os.environ[TZ] diff --git a/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py index b46d893e..fac70c1c 100644 --- a/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py +++ b/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py @@ -3,8 +3,8 @@ import logging import sys from zoneinfo import ZoneInfo -from constants import DEFAULT_TIMEZONE, LOG_DATE_FORMAT, LOG_FORMAT -from environments import LOG_LEVEL +from constants import LAUNCH_ON_LOCAL, LOG_DATE_FORMAT, LOG_FORMAT +from environments import LOG_LEVEL, TZ class SingletonLogger: @@ -17,10 +17,10 @@ class SingletonLogger: LOG_FORMAT, LOG_DATE_FORMAT ) - formatter.converter = lambda *arg: datetime.datetime.now(ZoneInfo(DEFAULT_TIMEZONE)).timetuple() + formatter.converter = lambda *arg: datetime.datetime.now(ZoneInfo(TZ)).timetuple() # ローカル環境で動かす場合、標準出力ハンドラーを追加する # AWS Lambda上では`LambdaLoggerHandler`がデフォルトでセットされている - if len(sys.argv) == 2 and sys.argv[1] == 'local': + if len(sys.argv) == 2 and sys.argv[1] == LAUNCH_ON_LOCAL: localHandler = logging.StreamHandler() logger.addHandler(localHandler) for handler in logger.handlers: From 8c8314848c67a07e810ce521645a7cf4fec78aee Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 4 Jul 2022 10:46:53 +0900 Subject: [PATCH 09/23] =?UTF-8?q?feat:=20S3=E3=81=AE=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E3=82=92=E6=B1=8E=E7=94=A8=E7=9A=84=E3=81=AB=E3=80=81SSM?= =?UTF-8?q?=E3=81=AE=E5=8F=96=E5=BE=97=E5=87=A6=E7=90=86=E3=82=82=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/aws/s3.py | 10 +-- .../check-view-option/aws/ssm.py | 31 ++++++++ .../check-view-option/constants.py | 13 +++- .../check-view-option/exceptions.py | 6 ++ .../check-view-option/main.py | 70 +++++++++++++++++-- 5 files changed, 115 insertions(+), 15 deletions(-) create mode 100644 lambda/check-view-secutiry-option/check-view-option/aws/ssm.py diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py index 6ace3816..8d512b71 100644 --- a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py +++ b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py @@ -1,7 +1,5 @@ import boto3 import environments -import exceptions -from botocore.exceptions import ClientError from constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY @@ -23,9 +21,5 @@ class ConfigBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(environments.CONFIG_BUCKET_NAME) - def read_check_target_schema_names(self): - try: - return self.__s3_resource.get_object(environments.CHECK_TARGET_SCHEMA_NAMES_PATH) - except ClientError as error: - if error.response['Error']['Code'] == 'NoSuchKey': - raise exceptions.FileNotFoundException('E-02-01', f'チェック対象スキーマ名ファイルの読み込みに失敗しました　エラー内容：{error}') + def check_target_schema_names(self): + return self.__s3_resource.get_object(environments.CHECK_TARGET_SCHEMA_NAMES_PATH) diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/ssm.py b/lambda/check-view-secutiry-option/check-view-option/aws/ssm.py new file mode 100644 index 00000000..528b4516 --- /dev/null +++ b/lambda/check-view-secutiry-option/check-view-option/aws/ssm.py @@ -0,0 +1,31 @@ +import boto3 +import environments +from constants import (AWS_RESOURCE_SSM, SSM_PARAMETER_RESPONSE, + SSM_PARAMETER_VALUE) + + +class SSMClient: + + def __init__(self) -> None: + self.__ssm_client = boto3.client(AWS_RESOURCE_SSM) + + def get_ssm_params(self, parameter_key: str, with_decryption: bool): + response = self.__ssm_client.get_parameter(Name=parameter_key, WithDecryption=with_decryption) + parameter_value = response[SSM_PARAMETER_RESPONSE][SSM_PARAMETER_VALUE] + return parameter_value + + +class SSMParameterStore: + __ssm_client: SSMClient = None + + def __init__(self) -> None: + self.__ssm_client = SSMClient() + + def db_host(self): + return self.__ssm_client.get_ssm_params(environments.PARAM_NAME_DB_HOST, True) + + def db_user_name(self): + return self.__ssm_client.get_ssm_params(environments.PARAM_NAME_DB_USER_NAME, True) + + def db_user_password(self): + return self.__ssm_client.get_ssm_params(environments.PARAM_NAME_DB_USER_PASSWORD, True) diff --git a/lambda/check-view-secutiry-option/check-view-option/constants.py b/lambda/check-view-secutiry-option/check-view-option/constants.py index 2ef6a079..3336997b 100644 --- a/lambda/check-view-secutiry-option/check-view-option/constants.py +++ b/lambda/check-view-secutiry-option/check-view-option/constants.py @@ -16,8 +16,19 @@ PARAM_NAME_DB_USER_NAME = 'PARAM_NAME_DB_USER_NAME' PARAM_NAME_DB_USER_PASSWORD = 'PARAM_NAME_DB_USER_PASSWORD' TZ = 'TZ' -# system var +# aws AWS_RESOURCE_S3 = 's3' +AWS_RESOURCE_SSM = 'ssm' S3_RESPONSE_BODY = 'Body' +SSM_PARAMETER_RESPONSE = 'Parameter' +SSM_PARAMETER_NAME = 'Name' +SSM_PARAMETER_VALUE = 'Value' +RESPONSE_ERROR = 'Error' +RESPONSE_ERROR_CODE = 'Code' +RESPONSE_CODE_NO_SUCH_KEY = 'NoSuchKey' +RESPONSE_CODE_PARAMETER_NOT_FOUND = 'ParameterNotFound' + +# system var UTF8 = 'utf-8' LAUNCH_ON_LOCAL = 'local' +CHECK_TARGET_SCHEMAS = 'check_target_schemas' diff --git a/lambda/check-view-secutiry-option/check-view-option/exceptions.py b/lambda/check-view-secutiry-option/check-view-option/exceptions.py index 131a0126..d3afb381 100644 --- a/lambda/check-view-secutiry-option/check-view-option/exceptions.py +++ b/lambda/check-view-secutiry-option/check-view-option/exceptions.py @@ -8,4 +8,10 @@ class MeDaCaException(Exception, metaclass=ABCMeta): class FileNotFoundException(MeDaCaException): + """S3のファイルが見つからない場合の例外""" + pass + + +class ParameterNotFoundException(MeDaCaException): + """パラメータストアのキーが見つからない場合の例外""" pass diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index b870a3bf..51586e65 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -2,30 +2,88 @@ Viewセキュリティオプション付与チェック用Lambda関数のエントリーポイント """ +import json + +import botocore + from aws.s3 import ConfigBucket -from exceptions import MeDaCaException +from aws.ssm import SSMParameterStore +from constants import (CHECK_TARGET_SCHEMAS, RESPONSE_CODE_NO_SUCH_KEY, + RESPONSE_CODE_PARAMETER_NOT_FOUND, RESPONSE_ERROR, + RESPONSE_ERROR_CODE) +from exceptions import (FileNotFoundException, MeDaCaException, + ParameterNotFoundException) from medaca_logger import MeDaCaLogger def handler(event, context): logger = MeDaCaLogger.get_logger() - try: logger.info('I-01-01', '処理開始　Viewセキュリティオプション付与チェック') - logger.info('I-01-02', 'チェック対象スキーマ名ファイルを読み込み 開始') - config_bucket = ConfigBucket() - check_target_schema_names = config_bucket.read_check_target_schema_names() - print(check_target_schema_names) + logger.info('I-02-02', 'チェック対象スキーマ名ファイルを読み込み 開始') + check_target_schemas = read_check_target_schemas() + logger.info('I-02-02', f'チェック対象スキーマ名ファイルを読み込み 終了　チェック対象スキーマ名：{check_target_schemas}') + # print(check_target_schemas) + logger.info('I-03-01', 'データベースへの接続開始 開始') + # DB接続のためのパラメータ取得 + db_host, db_user_name, db_user_password = read_db_param_from_parameter_store() + # print(db_host, db_user_name, db_user_password) + logger.info('I-03-01', 'データベースへの接続開始 終了') except MeDaCaException as e: logger.exception(e.error_id, e) raise e except Exception as e: logger.exception('E-99', f'想定外のエラーが発生しました　エラー内容：{e}') + raise e finally: logger.info('I-06-01', '処理終了　Viewセキュリティオプション付与チェック') +def read_check_target_schemas() -> list: + """設定ファイル[チェック対象スキーマ名ファイル]を読み込む + + Raises: + exceptions.FileNotFoundException: ファイルが読み込めなかったエラー + Exception: 想定外のエラー + + Returns: + list: チェック対象のスキーマ名のリスト + """ + try: + config_bucket = ConfigBucket() + check_target_schema_names = config_bucket.check_target_schema_names() + return json.loads(check_target_schema_names)[CHECK_TARGET_SCHEMAS] + except botocore.exceptions.ClientError as e: + if e.response[RESPONSE_ERROR][RESPONSE_ERROR_CODE] == RESPONSE_CODE_NO_SUCH_KEY: + raise FileNotFoundException('E-02-01', f'チェック対象スキーマ名ファイルの読み込みに失敗しました　エラー内容：{e}') + else: + raise Exception(e) + + +def read_db_param_from_parameter_store() -> tuple: + """パラメータストアからDB接続情報を取得する + + Raises: + FileNotFoundException: _description_ + Exception: 想定外のエラー + + Returns: + tuple: DB接続情報 + """ + try: + parameter_store = SSMParameterStore() + db_host = parameter_store.db_host() + db_user_name = parameter_store.db_user_name() + db_user_password = parameter_store.db_user_password() + return db_host, db_user_name, db_user_password + except botocore.exceptions.ClientError as e: + if e.response[RESPONSE_ERROR][RESPONSE_ERROR_CODE] == RESPONSE_CODE_PARAMETER_NOT_FOUND: + raise ParameterNotFoundException('E-03-02', f'パラメータストアの取得に失敗しました　エラー内容：{e}') + else: + raise Exception(e) + + # ローカル実行用 if __name__ == '__main__': handler({}, {}) From 6ac21eee9e4a386eb4b567cf7f82c37c48873722 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 4 Jul 2022 10:48:32 +0900 Subject: [PATCH 10/23] =?UTF-8?q?style:=20=E3=83=89=E3=82=AD=E3=83=A5?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-secutiry-option/check-view-option/exceptions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lambda/check-view-secutiry-option/check-view-option/exceptions.py b/lambda/check-view-secutiry-option/check-view-option/exceptions.py index d3afb381..df870583 100644 --- a/lambda/check-view-secutiry-option/check-view-option/exceptions.py +++ b/lambda/check-view-secutiry-option/check-view-option/exceptions.py @@ -2,6 +2,8 @@ from abc import ABCMeta class MeDaCaException(Exception, metaclass=ABCMeta): + """MeDaCaシステム固有のカスタムエラークラス""" + def __init__(self, error_id: str, message) -> None: super().__init__(message) self.error_id = error_id From 91208ab50b6e55718e8e844be790daba4280d957 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 6 Jul 2022 09:42:41 +0900 Subject: [PATCH 11/23] =?UTF-8?q?feat:=20DB=E6=8E=A5=E7=B6=9A=E3=81=A8View?= =?UTF-8?q?=E3=81=AE=E5=8F=96=E5=BE=97=E5=87=A6=E7=90=86=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/constants.py | 6 +++ .../check-view-option/database.py | 40 +++++++++++++++ .../check-view-option/exceptions.py | 5 ++ .../check-view-option/main.py | 50 +++++++++++++++++-- 4 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 lambda/check-view-secutiry-option/check-view-option/database.py diff --git a/lambda/check-view-secutiry-option/check-view-option/constants.py b/lambda/check-view-secutiry-option/check-view-option/constants.py index 3336997b..a0f93a3a 100644 --- a/lambda/check-view-secutiry-option/check-view-option/constants.py +++ b/lambda/check-view-secutiry-option/check-view-option/constants.py @@ -28,6 +28,12 @@ RESPONSE_ERROR_CODE = 'Code' RESPONSE_CODE_NO_SUCH_KEY = 'NoSuchKey' RESPONSE_CODE_PARAMETER_NOT_FOUND = 'ParameterNotFound' +# sql + +DEFAULT_SCHEMA = 'INFORMATION_SCHEMA' +INFORMATION_SCHEMA_SECURITY_TYPE_INVOKER = 'INVOKER' +CONNECTION_TIMEOUT = 5 + # system var UTF8 = 'utf-8' LAUNCH_ON_LOCAL = 'local' diff --git a/lambda/check-view-secutiry-option/check-view-option/database.py b/lambda/check-view-secutiry-option/check-view-option/database.py new file mode 100644 index 00000000..133ebbb4 --- /dev/null +++ b/lambda/check-view-secutiry-option/check-view-option/database.py @@ -0,0 +1,40 @@ +import contextlib + +import pymysql +from pymysql.constants import CLIENT + +from constants import CONNECTION_TIMEOUT, DEFAULT_SCHEMA + + +class Database: + + __connection: pymysql.Connection = None + __host: str = None + __user: str = None + __password: str = None + __database: str = None + + def __init__(self, host: str, user: str, password: str) -> None: + self.__host = host + self.__user = user + self.__password = password + self.__database = DEFAULT_SCHEMA + + def connect(self): + connection = pymysql.connect(host=self.__host, user=self.__user, passwd=self.__password, + database=self.__database, connect_timeout=CONNECTION_TIMEOUT, + client_flag=CLIENT.MULTI_STATEMENTS) + self.__connection = connection + + @contextlib.contextmanager + def query(self, query: str): + if self.__connection is None: + raise Exception('データベースに接続されていません') + + with self.__connection.cursor() as cursor: + cursor.execute(query) + yield cursor + + def close(self): + self.__connection.close() + self.__connection = None diff --git a/lambda/check-view-secutiry-option/check-view-option/exceptions.py b/lambda/check-view-secutiry-option/check-view-option/exceptions.py index df870583..fc00a279 100644 --- a/lambda/check-view-secutiry-option/check-view-option/exceptions.py +++ b/lambda/check-view-secutiry-option/check-view-option/exceptions.py @@ -17,3 +17,8 @@ class FileNotFoundException(MeDaCaException): class ParameterNotFoundException(MeDaCaException): """パラメータストアのキーが見つからない場合の例外""" pass + + +class DatabaseConnectionException(MeDaCaException): + """データベース接続に失敗した場合の例外""" + pass diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index 51586e65..cd41aaa4 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -8,11 +8,14 @@ import botocore from aws.s3 import ConfigBucket from aws.ssm import SSMParameterStore -from constants import (CHECK_TARGET_SCHEMAS, RESPONSE_CODE_NO_SUCH_KEY, +from constants import (CHECK_TARGET_SCHEMAS, + INFORMATION_SCHEMA_SECURITY_TYPE_INVOKER, + RESPONSE_CODE_NO_SUCH_KEY, RESPONSE_CODE_PARAMETER_NOT_FOUND, RESPONSE_ERROR, RESPONSE_ERROR_CODE) -from exceptions import (FileNotFoundException, MeDaCaException, - ParameterNotFoundException) +from database import Database +from exceptions import (DatabaseConnectionException, FileNotFoundException, + MeDaCaException, ParameterNotFoundException) from medaca_logger import MeDaCaLogger @@ -27,8 +30,16 @@ def handler(event, context): logger.info('I-03-01', 'データベースへの接続開始 開始') # DB接続のためのパラメータ取得 db_host, db_user_name, db_user_password = read_db_param_from_parameter_store() - # print(db_host, db_user_name, db_user_password) + connection = connection_database(db_host, db_user_name, db_user_password) logger.info('I-03-01', 'データベースへの接続開始 終了') + logger.info('I-04-01', 'Viewセキュリティオプション チェック開始') + check_result = check_view_security_option(connection, check_target_schemas) + + if len(check_result) == 0: + logger.info('I-04-02', 'Viewセキュリティオプション 未設定のViewはありません。処理を終了します。') + return + + logger.info('I-04-01', 'Viewセキュリティオプション 未設定のViewがあるため、メール送信処理を開始します。') except MeDaCaException as e: logger.exception(e.error_id, e) @@ -84,6 +95,37 @@ def read_db_param_from_parameter_store() -> tuple: raise Exception(e) +def connection_database(host: str, user_name: str, password: str) -> Database: + try: + database = Database(host, user_name, password) + database.connect() + return database + except Exception as e: + raise DatabaseConnectionException('E-03-02', f'データベースへの接続に失敗しました　エラー内容：{e}') + + +def check_view_security_option(connection: Database, check_target_schemas: list) -> list: + select_view_security_option_sql = f""" + SELECT + TABLE_SCHEMA, + TABLE_NAME + FROM + INFORMATION_SCHEMA.VIEWS + WHERE + TABLE_SCHEMA IN ( + {','.join([f"'{schema_name}'" for schema_name in check_target_schemas])} + ) + AND SECURITY_TYPE <> '{INFORMATION_SCHEMA_SECURITY_TYPE_INVOKER}' + """ + print(select_view_security_option_sql) + try: + with connection.query(select_view_security_option_sql) as cursor: + result = cursor.fetchall() + return result + except Exception as e: + raise DatabaseConnectionException('E-03-02', f'Viewセキュリティオプションチェックに失敗しました　エラー内容：{e}') + + # ローカル実行用 if __name__ == '__main__': handler({}, {}) From 7f6f4659dc80f07038845f7d129ba949a2365911 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 7 Jul 2022 13:28:49 +0900 Subject: [PATCH 12/23] =?UTF-8?q?feat:=20=E3=83=A1=E3=83=BC=E3=83=AB?= =?UTF-8?q?=E6=9C=AC=E6=96=87=E7=94=9F=E6=88=90=E5=87=A6=E7=90=86=E3=82=92?= =?UTF-8?q?=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/aws/__init__.py | 0 .../check-view-option/aws/s3.py | 41 +++++++++++-- .../check-view-option/aws/ssm.py | 3 + .../check-view-option/constants.py | 3 +- .../check-view-option/dto/__init__.py | 0 .../dto/view_secutiry_option.py | 7 +++ .../check-view-option/main.py | 60 ++++++++++++++++--- ...heck_view_secutiry_option_mail_body.config | 6 ++ ...eck_view_secutiry_option_mail_title.config | 1 + 9 files changed, 109 insertions(+), 12 deletions(-) create mode 100644 lambda/check-view-secutiry-option/check-view-option/aws/__init__.py create mode 100644 lambda/check-view-secutiry-option/check-view-option/dto/__init__.py create mode 100644 lambda/check-view-secutiry-option/check-view-option/dto/view_secutiry_option.py create mode 100644 s3/config/view_check/check_view_secutiry_option_mail_body.config create mode 100644 s3/config/view_check/check_view_secutiry_option_mail_title.config diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/__init__.py b/lambda/check-view-secutiry-option/check-view-option/aws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py index 8d512b71..d18b74e5 100644 --- a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py +++ b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py @@ -1,6 +1,6 @@ import boto3 import environments -from constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY +from constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY, UTF8 class S3Resource: @@ -12,14 +12,47 @@ class S3Resource: def get_object(self, object_key: str): s3_object = self.__s3_bucket.Object(object_key) response = s3_object.get() - return response[S3_RESPONSE_BODY].read() + return response[S3_RESPONSE_BODY].read().decode(UTF8) class ConfigBucket: __s3_resource: S3Resource = None + __bucket_name: str + __check_target_schema_names_file_path: str + __notice_mail_title_template_file_path: str + __notice_mail_body_template_file_path: str def __init__(self) -> None: - self.__s3_resource = S3Resource(environments.CONFIG_BUCKET_NAME) + self.__bucket_name = environments.CONFIG_BUCKET_NAME + self.__check_target_schema_names_file_path = environments.CHECK_TARGET_SCHEMA_NAMES_PATH + self.__notice_mail_title_template_file_path = environments.NOTICE_MAIL_TITLE_TEMPLATE_PATH + self.__notice_mail_body_template_file_path = environments.NOTICE_MAIL_BODY_TEMPLATE_PATH + self.__s3_resource = S3Resource(self.__bucket_name) + @property + def bucket_name(self): + return self.__bucket_name + + @property + def check_target_schema_names_file_path(self): + return self.__check_target_schema_names_file_path + + @property + def mail_body_file_path(self): + return self.__notice_mail_body_template_file_path + + @property + def mail_title_file_path(self): + return self.__notice_mail_title_template_file_path + + @property def check_target_schema_names(self): - return self.__s3_resource.get_object(environments.CHECK_TARGET_SCHEMA_NAMES_PATH) + return self.__s3_resource.get_object(self.__check_target_schema_names_file_path) + + @property + def notice_mail_title_template(self): + return self.__s3_resource.get_object(self.__notice_mail_title_template_file_path) + + @property + def notice_mail_body_template(self): + return self.__s3_resource.get_object(self.__notice_mail_body_template_file_path) diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/ssm.py b/lambda/check-view-secutiry-option/check-view-option/aws/ssm.py index 528b4516..7b618b59 100644 --- a/lambda/check-view-secutiry-option/check-view-option/aws/ssm.py +++ b/lambda/check-view-secutiry-option/check-view-option/aws/ssm.py @@ -21,11 +21,14 @@ class SSMParameterStore: def __init__(self) -> None: self.__ssm_client = SSMClient() + @property def db_host(self): return self.__ssm_client.get_ssm_params(environments.PARAM_NAME_DB_HOST, True) + @property def db_user_name(self): return self.__ssm_client.get_ssm_params(environments.PARAM_NAME_DB_USER_NAME, True) + @property def db_user_password(self): return self.__ssm_client.get_ssm_params(environments.PARAM_NAME_DB_USER_PASSWORD, True) diff --git a/lambda/check-view-secutiry-option/check-view-option/constants.py b/lambda/check-view-secutiry-option/check-view-option/constants.py index a0f93a3a..9f6b0735 100644 --- a/lambda/check-view-secutiry-option/check-view-option/constants.py +++ b/lambda/check-view-secutiry-option/check-view-option/constants.py @@ -29,7 +29,6 @@ RESPONSE_CODE_NO_SUCH_KEY = 'NoSuchKey' RESPONSE_CODE_PARAMETER_NOT_FOUND = 'ParameterNotFound' # sql - DEFAULT_SCHEMA = 'INFORMATION_SCHEMA' INFORMATION_SCHEMA_SECURITY_TYPE_INVOKER = 'INVOKER' CONNECTION_TIMEOUT = 5 @@ -38,3 +37,5 @@ CONNECTION_TIMEOUT = 5 UTF8 = 'utf-8' LAUNCH_ON_LOCAL = 'local' CHECK_TARGET_SCHEMAS = 'check_target_schemas' +# メール本文に出力する不足ファイル名一覧のインデント +MAIL_INDENT = '\n　　' diff --git a/lambda/check-view-secutiry-option/check-view-option/dto/__init__.py b/lambda/check-view-secutiry-option/check-view-option/dto/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/lambda/check-view-secutiry-option/check-view-option/dto/view_secutiry_option.py b/lambda/check-view-secutiry-option/check-view-option/dto/view_secutiry_option.py new file mode 100644 index 00000000..f5cdc3dc --- /dev/null +++ b/lambda/check-view-secutiry-option/check-view-option/dto/view_secutiry_option.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + +@dataclass +class ViewSecurityOption: + schema_name: str + table_name: str diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index cd41aaa4..81586cc3 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -9,18 +9,20 @@ import botocore from aws.s3 import ConfigBucket from aws.ssm import SSMParameterStore from constants import (CHECK_TARGET_SCHEMAS, - INFORMATION_SCHEMA_SECURITY_TYPE_INVOKER, + INFORMATION_SCHEMA_SECURITY_TYPE_INVOKER, MAIL_INDENT, RESPONSE_CODE_NO_SUCH_KEY, RESPONSE_CODE_PARAMETER_NOT_FOUND, RESPONSE_ERROR, RESPONSE_ERROR_CODE) from database import Database +from dto.view_secutiry_option import ViewSecurityOption from exceptions import (DatabaseConnectionException, FileNotFoundException, MeDaCaException, ParameterNotFoundException) from medaca_logger import MeDaCaLogger +logger = MeDaCaLogger.get_logger() + def handler(event, context): - logger = MeDaCaLogger.get_logger() try: logger.info('I-01-01', '処理開始　Viewセキュリティオプション付与チェック') logger.info('I-02-02', 'チェック対象スキーマ名ファイルを読み込み 開始') @@ -38,9 +40,13 @@ def handler(event, context): if len(check_result) == 0: logger.info('I-04-02', 'Viewセキュリティオプション 未設定のViewはありません。処理を終了します。') return - logger.info('I-04-01', 'Viewセキュリティオプション 未設定のViewがあるため、メール送信処理を開始します。') + view_security_options = [ViewSecurityOption(*row) for row in check_result] + + mail_title, mail_body = make_notice_mail(view_security_options) + print(mail_title, mail_body) + except MeDaCaException as e: logger.exception(e.error_id, e) raise e @@ -63,7 +69,7 @@ def read_check_target_schemas() -> list: """ try: config_bucket = ConfigBucket() - check_target_schema_names = config_bucket.check_target_schema_names() + check_target_schema_names = config_bucket.check_target_schema_names return json.loads(check_target_schema_names)[CHECK_TARGET_SCHEMAS] except botocore.exceptions.ClientError as e: if e.response[RESPONSE_ERROR][RESPONSE_ERROR_CODE] == RESPONSE_CODE_NO_SUCH_KEY: @@ -84,9 +90,9 @@ def read_db_param_from_parameter_store() -> tuple: """ try: parameter_store = SSMParameterStore() - db_host = parameter_store.db_host() - db_user_name = parameter_store.db_user_name() - db_user_password = parameter_store.db_user_password() + db_host = parameter_store.db_host + db_user_name = parameter_store.db_user_name + db_user_password = parameter_store.db_user_password return db_host, db_user_name, db_user_password except botocore.exceptions.ClientError as e: if e.response[RESPONSE_ERROR][RESPONSE_ERROR_CODE] == RESPONSE_CODE_PARAMETER_NOT_FOUND: @@ -126,6 +132,46 @@ def check_view_security_option(connection: Database, check_target_schemas: list) raise DatabaseConnectionException('E-03-02', f'Viewセキュリティオプションチェックに失敗しました　エラー内容：{e}') +def make_notice_mail(view_security_options: list[ViewSecurityOption]): + config_bucket = ConfigBucket() + logger.info( + 'I-05-02', f'通知メール（タイトル）テンプレートファイル読込　読込元：{config_bucket.bucket_name}/{config_bucket.mail_title_file_path}') + mail_title_template = read_mail_title(config_bucket) + logger.info( + 'I-05-02', f'通知メール（本文）テンプレートファイル読込　読込元：{config_bucket.bucket_name}/{config_bucket.mail_body_file_path}') + mail_body_template = read_mail_body_template(config_bucket) + + mail_message = MAIL_INDENT.join([f'{option.schema_name}.{option.table_name}' for option in view_security_options]) + + mail_body = mail_body_template.format(no_option_views=mail_message) + + return mail_title_template, mail_body + + +def read_mail_title(config_bucket: ConfigBucket): + try: + mail_title = config_bucket.notice_mail_title_template + except botocore.exceptions.ClientError as e: + if e.response[RESPONSE_ERROR][RESPONSE_ERROR_CODE] == RESPONSE_CODE_NO_SUCH_KEY: + raise FileNotFoundException('E-02-01', f'通知メール（タイトル）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') + else: + raise Exception(e) + + return mail_title + + +def read_mail_body_template(config_bucket: ConfigBucket): + try: + mail_body_template = config_bucket.notice_mail_body_template + except botocore.exceptions.ClientError as e: + if e.response[RESPONSE_ERROR][RESPONSE_ERROR_CODE] == RESPONSE_CODE_NO_SUCH_KEY: + raise FileNotFoundException('E-02-01', f'通知メール（本文）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') + else: + raise Exception(e) + + return mail_body_template + + # ローカル実行用 if __name__ == '__main__': handler({}, {}) diff --git a/s3/config/view_check/check_view_secutiry_option_mail_body.config b/s3/config/view_check/check_view_secutiry_option_mail_body.config new file mode 100644 index 00000000..5457eafc --- /dev/null +++ b/s3/config/view_check/check_view_secutiry_option_mail_body.config @@ -0,0 +1,6 @@ +宛先各位 +　customスキーマの以下のviewに「SQL SECURITY INVOKER」オプションが指定されておりません。viewを再作成しオプションを指定してください。 +　　{no_option_views} + +　尚、本メールはシステム自動送信ですので、返信できません。 +　本件に関する問い合わせは、ＩＴ部門ゴザリ様にお願いいたします。 diff --git a/s3/config/view_check/check_view_secutiry_option_mail_title.config b/s3/config/view_check/check_view_secutiry_option_mail_title.config new file mode 100644 index 00000000..a85c0134 --- /dev/null +++ b/s3/config/view_check/check_view_secutiry_option_mail_title.config @@ -0,0 +1 @@ +【MeDaCaシステム通知】view参照制限オプション指定漏れを検出しました \ No newline at end of file From 4f57e1ecf428eee92e571038d417340aaf6ac5ac Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 7 Jul 2022 13:46:04 +0900 Subject: [PATCH 13/23] =?UTF-8?q?style:=20=E3=83=89=E3=82=AD=E3=83=A5?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/exceptions.py | 5 ++ .../check-view-option/main.py | 65 +++++++++++++++++-- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/lambda/check-view-secutiry-option/check-view-option/exceptions.py b/lambda/check-view-secutiry-option/check-view-option/exceptions.py index fc00a279..90cd6efb 100644 --- a/lambda/check-view-secutiry-option/check-view-option/exceptions.py +++ b/lambda/check-view-secutiry-option/check-view-option/exceptions.py @@ -22,3 +22,8 @@ class ParameterNotFoundException(MeDaCaException): class DatabaseConnectionException(MeDaCaException): """データベース接続に失敗した場合の例外""" pass + + +class QueryExecutionException(MeDaCaException): + """クエリ実行に失敗した場合の例外""" + pass diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index 81586cc3..f626b0b8 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -16,7 +16,8 @@ from constants import (CHECK_TARGET_SCHEMAS, from database import Database from dto.view_secutiry_option import ViewSecurityOption from exceptions import (DatabaseConnectionException, FileNotFoundException, - MeDaCaException, ParameterNotFoundException) + MeDaCaException, ParameterNotFoundException, + QueryExecutionException) from medaca_logger import MeDaCaLogger logger = MeDaCaLogger.get_logger() @@ -35,7 +36,7 @@ def handler(event, context): connection = connection_database(db_host, db_user_name, db_user_password) logger.info('I-03-01', 'データベースへの接続開始 終了') logger.info('I-04-01', 'Viewセキュリティオプション チェック開始') - check_result = check_view_security_option(connection, check_target_schemas) + check_result = fetch_view_security_options(connection, check_target_schemas) if len(check_result) == 0: logger.info('I-04-02', 'Viewセキュリティオプション 未設定のViewはありません。処理を終了します。') @@ -61,7 +62,7 @@ def read_check_target_schemas() -> list: """設定ファイル[チェック対象スキーマ名ファイル]を読み込む Raises: - exceptions.FileNotFoundException: ファイルが読み込めなかったエラー + FileNotFoundException: ファイルが読み込めなかったエラー Exception: 想定外のエラー Returns: @@ -82,7 +83,7 @@ def read_db_param_from_parameter_store() -> tuple: """パラメータストアからDB接続情報を取得する Raises: - FileNotFoundException: _description_ + ParameterNotFoundException: 指定されたパラメータが存在しないエラー Exception: 想定外のエラー Returns: @@ -102,6 +103,19 @@ def read_db_param_from_parameter_store() -> tuple: def connection_database(host: str, user_name: str, password: str) -> Database: + """データベース接続 + + Args: + host (str): DBホスト + user_name (str): DBユーザー名 + password (str): DBパスワード + + Raises: + DatabaseConnectionException: データベースへの接続に失敗したエラー + + Returns: + Database: データベース操作クラス + """ try: database = Database(host, user_name, password) database.connect() @@ -110,7 +124,20 @@ def connection_database(host: str, user_name: str, password: str) -> Database: raise DatabaseConnectionException('E-03-02', f'データベースへの接続に失敗しました　エラー内容：{e}') -def check_view_security_option(connection: Database, check_target_schemas: list) -> list: +def fetch_view_security_options(connection: Database, check_target_schemas: list) -> tuple: + """SECURITY INVOKERのついていないViewの一覧を取得する + + Args: + connection (Database): 接続済みDB操作クラス + check_target_schemas (str): チェック対象のスキーマ一覧 + + Raises: + QueryExecutionException: クエリ実行エラー + + Returns: + tuple: クエリ実行結果 + """ + select_view_security_option_sql = f""" SELECT TABLE_SCHEMA, @@ -123,13 +150,12 @@ def check_view_security_option(connection: Database, check_target_schemas: list) ) AND SECURITY_TYPE <> '{INFORMATION_SCHEMA_SECURITY_TYPE_INVOKER}' """ - print(select_view_security_option_sql) try: with connection.query(select_view_security_option_sql) as cursor: result = cursor.fetchall() return result except Exception as e: - raise DatabaseConnectionException('E-03-02', f'Viewセキュリティオプションチェックに失敗しました　エラー内容：{e}') + raise QueryExecutionException('E-03-02', f'Viewセキュリティオプションチェックに失敗しました　エラー内容：{e}') def make_notice_mail(view_security_options: list[ViewSecurityOption]): @@ -149,6 +175,19 @@ def make_notice_mail(view_security_options: list[ViewSecurityOption]): def read_mail_title(config_bucket: ConfigBucket): + """メールタイトルを読み込む + + Args: + config_bucket (ConfigBucket): 設定ファイル保管バケット操作クラス + + Raises: + FileNotFoundException: ファイルが読み込めなかったエラー + Exception: 想定外のエラー + + Returns: + str: メールタイトル + """ + try: mail_title = config_bucket.notice_mail_title_template except botocore.exceptions.ClientError as e: @@ -161,6 +200,18 @@ def read_mail_title(config_bucket: ConfigBucket): def read_mail_body_template(config_bucket: ConfigBucket): + """メール本文を読み込む + + Args: + config_bucket (ConfigBucket): 設定ファイル保管バケット操作クラス + + Raises: + FileNotFoundException: ファイルが読み込めなかったエラー + Exception: 想定外のエラー + + Returns: + str: メール本文 + """ try: mail_body_template = config_bucket.notice_mail_body_template except botocore.exceptions.ClientError as e: From 4f1c4ea86a26da94f08d9db8eae906f13f74f503 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 7 Jul 2022 14:29:41 +0900 Subject: [PATCH 14/23] =?UTF-8?q?feat:=20MBJ=E3=81=A8NDS=E3=81=B8=E3=81=AE?= =?UTF-8?q?=E9=80=9A=E7=9F=A5=E5=87=A6=E7=90=86=E3=82=92=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/aws/sns.py | 31 ++++++++++++ .../check-view-option/constants.py | 2 + .../check-view-option/environments.py | 4 +- .../check-view-option/exceptions.py | 5 ++ .../check-view-option/main.py | 50 +++++++++++++++++-- 5 files changed, 86 insertions(+), 6 deletions(-) create mode 100644 lambda/check-view-secutiry-option/check-view-option/aws/sns.py diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/sns.py b/lambda/check-view-secutiry-option/check-view-option/aws/sns.py new file mode 100644 index 00000000..d81e7307 --- /dev/null +++ b/lambda/check-view-secutiry-option/check-view-option/aws/sns.py @@ -0,0 +1,31 @@ +import boto3 +import environments +from constants import AWS_RESOURCE_SNS + + +class SNSClient: + + def __init__(self) -> None: + self.__sns_client = boto3.client(AWS_RESOURCE_SNS) + + def publish(self, sns_topic_arn: str, subject: str, message: str) -> None: + publish_params = { + 'TopicArn': sns_topic_arn, + 'Subject': subject.rstrip('\n'), + 'Message': message + } + self.__sns_client.publish(**publish_params) + + +class SNSNotifier: + __sns_client: SNSClient = None + + def __init__(self) -> None: + self.__sns_client = SNSClient() + + def publish_to_mbj(self, subject: str, message: str): + self.__sns_client.publish(environments.MBJ_NOTICE_TOPIC, subject, message) + + def publish_to_nds(self, error_id: str, exception: Exception): + error_message = f'{error_id} のエラーが発生しました。ご確認ください\n詳細：{exception}' + self.__sns_client.publish(environments.NDS_NOTICE_TOPIC, environments.NDS_NOTICE_TITLE, error_message) diff --git a/lambda/check-view-secutiry-option/check-view-option/constants.py b/lambda/check-view-secutiry-option/check-view-option/constants.py index 9f6b0735..ac6b2bfa 100644 --- a/lambda/check-view-secutiry-option/check-view-option/constants.py +++ b/lambda/check-view-secutiry-option/check-view-option/constants.py @@ -9,6 +9,7 @@ CONFIG_BUCKET_NAME = 'CONFIG_BUCKET_NAME' LOG_LEVEL = 'LOG_LEVEL' MBJ_NOTICE_TOPIC = 'MBJ_NOTICE_TOPIC' NDS_NOTICE_TOPIC = 'NDS_NOTICE_TOPIC' +NDS_NOTICE_TITLE = 'NDS_NOTICE_TITLE' NOTICE_MAIL_BODY_TEMPLATE_PATH = 'NOTICE_MAIL_BODY_TEMPLATE_PATH' NOTICE_MAIL_TITLE_TEMPLATE_PATH = 'NOTICE_MAIL_TITLE_TEMPLATE_PATH' PARAM_NAME_DB_HOST = 'PARAM_NAME_DB_HOST' @@ -19,6 +20,7 @@ TZ = 'TZ' # aws AWS_RESOURCE_S3 = 's3' AWS_RESOURCE_SSM = 'ssm' +AWS_RESOURCE_SNS = 'sns' S3_RESPONSE_BODY = 'Body' SSM_PARAMETER_RESPONSE = 'Parameter' SSM_PARAMETER_NAME = 'Name' diff --git a/lambda/check-view-secutiry-option/check-view-option/environments.py b/lambda/check-view-secutiry-option/check-view-option/environments.py index 87156c19..3a292a68 100644 --- a/lambda/check-view-secutiry-option/check-view-option/environments.py +++ b/lambda/check-view-secutiry-option/check-view-option/environments.py @@ -2,7 +2,8 @@ import os from constants import (CHECK_TARGET_SCHEMA_NAMES_PATH, CONFIG_BUCKET_NAME, LOG_LEVEL, LOG_LEVEL_INFO, MBJ_NOTICE_TOPIC, - NDS_NOTICE_TOPIC, NOTICE_MAIL_BODY_TEMPLATE_PATH, + NDS_NOTICE_TITLE, NDS_NOTICE_TOPIC, + NOTICE_MAIL_BODY_TEMPLATE_PATH, NOTICE_MAIL_TITLE_TEMPLATE_PATH, PARAM_NAME_DB_HOST, PARAM_NAME_DB_USER_NAME, PARAM_NAME_DB_USER_PASSWORD, TZ) @@ -12,6 +13,7 @@ CHECK_TARGET_SCHEMA_NAMES_PATH = os.environ[CHECK_TARGET_SCHEMA_NAMES_PATH] CONFIG_BUCKET_NAME = os.environ[CONFIG_BUCKET_NAME] MBJ_NOTICE_TOPIC = os.environ[MBJ_NOTICE_TOPIC] NDS_NOTICE_TOPIC = os.environ[NDS_NOTICE_TOPIC] +NDS_NOTICE_TITLE = os.environ[NDS_NOTICE_TITLE] NOTICE_MAIL_BODY_TEMPLATE_PATH = os.environ[NOTICE_MAIL_BODY_TEMPLATE_PATH] NOTICE_MAIL_TITLE_TEMPLATE_PATH = os.environ[NOTICE_MAIL_TITLE_TEMPLATE_PATH] diff --git a/lambda/check-view-secutiry-option/check-view-option/exceptions.py b/lambda/check-view-secutiry-option/check-view-option/exceptions.py index 90cd6efb..aebd28c3 100644 --- a/lambda/check-view-secutiry-option/check-view-option/exceptions.py +++ b/lambda/check-view-secutiry-option/check-view-option/exceptions.py @@ -27,3 +27,8 @@ class DatabaseConnectionException(MeDaCaException): class QueryExecutionException(MeDaCaException): """クエリ実行に失敗した場合の例外""" pass + + +class SNSPublishException(MeDaCaException): + """AmazonSNSへの通知に失敗した場合の例外""" + pass diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index f626b0b8..0226d3e3 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -7,6 +7,7 @@ import json import botocore from aws.s3 import ConfigBucket +from aws.sns import SNSNotifier from aws.ssm import SSMParameterStore from constants import (CHECK_TARGET_SCHEMAS, INFORMATION_SCHEMA_SECURITY_TYPE_INVOKER, MAIL_INDENT, @@ -15,9 +16,10 @@ from constants import (CHECK_TARGET_SCHEMAS, RESPONSE_ERROR_CODE) from database import Database from dto.view_secutiry_option import ViewSecurityOption +from environments import MBJ_NOTICE_TOPIC from exceptions import (DatabaseConnectionException, FileNotFoundException, MeDaCaException, ParameterNotFoundException, - QueryExecutionException) + QueryExecutionException, SNSPublishException) from medaca_logger import MeDaCaLogger logger = MeDaCaLogger.get_logger() @@ -46,13 +48,18 @@ def handler(event, context): view_security_options = [ViewSecurityOption(*row) for row in check_result] mail_title, mail_body = make_notice_mail(view_security_options) - print(mail_title, mail_body) + + logger.info('I-05-06', f'メール送信指示をします　送信先トピック：{MBJ_NOTICE_TOPIC}') + notice_to_mbj(mail_title, mail_body) + logger.info('I-05-07', 'メール送信指示をしました') except MeDaCaException as e: logger.exception(e.error_id, e) + notice_to_nds(e.error_id, e) raise e except Exception as e: logger.exception('E-99', f'想定外のエラーが発生しました　エラー内容：{e}') + notice_to_nds('E-99', e) raise e finally: logger.info('I-06-01', '処理終了　Viewセキュリティオプション付与チェック') @@ -158,7 +165,7 @@ def fetch_view_security_options(connection: Database, check_target_schemas: list raise QueryExecutionException('E-03-02', f'Viewセキュリティオプションチェックに失敗しました　エラー内容：{e}') -def make_notice_mail(view_security_options: list[ViewSecurityOption]): +def make_notice_mail(view_security_options: list[ViewSecurityOption]) -> tuple[str]: config_bucket = ConfigBucket() logger.info( 'I-05-02', f'通知メール（タイトル）テンプレートファイル読込　読込元：{config_bucket.bucket_name}/{config_bucket.mail_title_file_path}') @@ -174,7 +181,7 @@ def make_notice_mail(view_security_options: list[ViewSecurityOption]): return mail_title_template, mail_body -def read_mail_title(config_bucket: ConfigBucket): +def read_mail_title(config_bucket: ConfigBucket) -> str: """メールタイトルを読み込む Args: @@ -199,7 +206,7 @@ def read_mail_title(config_bucket: ConfigBucket): return mail_title -def read_mail_body_template(config_bucket: ConfigBucket): +def read_mail_body_template(config_bucket: ConfigBucket) -> str: """メール本文を読み込む Args: @@ -223,6 +230,39 @@ def read_mail_body_template(config_bucket: ConfigBucket): return mail_body_template +def notice_to_mbj(mail_title: str, mail_body: str) -> None: + """MBJへ通知を行います + + Args: + mail_title (str): メールタイトル + mail_body (str): メール本文 + + Raises: + SNSPublishException: SNSでの通知失敗した場合のエラー + """ + try: + notifier = SNSNotifier() + notifier.publish_to_mbj(mail_title, mail_body) + except Exception as e: + raise SNSPublishException('E-98', f'通知の送信指示に失敗しました　エラー内容：{e}') + + +def notice_to_nds(error_id: str, error_message: str) -> None: + """NDSに処理以上通知を行う + + Args: + error_id (str): エラーID + error_message (str): エラーメッセージ + Raises: + SNSPublishException: SNSでの通知失敗した場合のエラー + """ + try: + notifier = SNSNotifier() + notifier.publish_to_nds(error_id, error_message) + except Exception as e: + raise SNSPublishException('E-98', f'通知の送信指示に失敗しました　エラー内容：{e}') + + # ローカル実行用 if __name__ == '__main__': handler({}, {}) From 28d10dc054f5d8d6b36da92ce8da43626b342d70 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 7 Jul 2022 15:12:07 +0900 Subject: [PATCH 15/23] =?UTF-8?q?fix:=20=E3=83=AD=E3=82=B0=E3=81=AEID?= =?UTF-8?q?=E3=82=92=E8=A8=AD=E8=A8=88=E3=81=A8=E3=81=82=E3=82=8F=E3=81=9B?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/aws/s3.py | 31 +------- ...y_option.py => no_security_option_view.py} | 2 +- .../check-view-option/main.py | 77 +++++++++++-------- 3 files changed, 48 insertions(+), 62 deletions(-) rename lambda/check-view-secutiry-option/check-view-option/dto/{view_secutiry_option.py => no_security_option_view.py} (75%) diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py index d18b74e5..7074b663 100644 --- a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py +++ b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py @@ -18,41 +18,18 @@ class S3Resource: class ConfigBucket: __s3_resource: S3Resource = None __bucket_name: str - __check_target_schema_names_file_path: str - __notice_mail_title_template_file_path: str - __notice_mail_body_template_file_path: str def __init__(self) -> None: - self.__bucket_name = environments.CONFIG_BUCKET_NAME - self.__check_target_schema_names_file_path = environments.CHECK_TARGET_SCHEMA_NAMES_PATH - self.__notice_mail_title_template_file_path = environments.NOTICE_MAIL_TITLE_TEMPLATE_PATH - self.__notice_mail_body_template_file_path = environments.NOTICE_MAIL_BODY_TEMPLATE_PATH - self.__s3_resource = S3Resource(self.__bucket_name) - - @property - def bucket_name(self): - return self.__bucket_name - - @property - def check_target_schema_names_file_path(self): - return self.__check_target_schema_names_file_path - - @property - def mail_body_file_path(self): - return self.__notice_mail_body_template_file_path - - @property - def mail_title_file_path(self): - return self.__notice_mail_title_template_file_path + self.__s3_resource = S3Resource(environments.CONFIG_BUCKET_NAME) @property def check_target_schema_names(self): - return self.__s3_resource.get_object(self.__check_target_schema_names_file_path) + return self.__s3_resource.get_object(environments.CHECK_TARGET_SCHEMA_NAMES_PATH) @property def notice_mail_title_template(self): - return self.__s3_resource.get_object(self.__notice_mail_title_template_file_path) + return self.__s3_resource.get_object(environments.NOTICE_MAIL_TITLE_TEMPLATE_PATH) @property def notice_mail_body_template(self): - return self.__s3_resource.get_object(self.__notice_mail_body_template_file_path) + return self.__s3_resource.get_object(environments.NOTICE_MAIL_BODY_TEMPLATE_PATH) diff --git a/lambda/check-view-secutiry-option/check-view-option/dto/view_secutiry_option.py b/lambda/check-view-secutiry-option/check-view-option/dto/no_security_option_view.py similarity index 75% rename from lambda/check-view-secutiry-option/check-view-option/dto/view_secutiry_option.py rename to lambda/check-view-secutiry-option/check-view-option/dto/no_security_option_view.py index f5cdc3dc..3a58e873 100644 --- a/lambda/check-view-secutiry-option/check-view-option/dto/view_secutiry_option.py +++ b/lambda/check-view-secutiry-option/check-view-option/dto/no_security_option_view.py @@ -2,6 +2,6 @@ from dataclasses import dataclass @dataclass -class ViewSecurityOption: +class NoSecurityOptionView: schema_name: str table_name: str diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index 0226d3e3..ff2361d8 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -15,8 +15,10 @@ from constants import (CHECK_TARGET_SCHEMAS, RESPONSE_CODE_PARAMETER_NOT_FOUND, RESPONSE_ERROR, RESPONSE_ERROR_CODE) from database import Database -from dto.view_secutiry_option import ViewSecurityOption -from environments import MBJ_NOTICE_TOPIC +from dto.no_security_option_view import NoSecurityOptionView +from environments import (CONFIG_BUCKET_NAME, MBJ_NOTICE_TOPIC, + NDS_NOTICE_TOPIC, NOTICE_MAIL_BODY_TEMPLATE_PATH, + NOTICE_MAIL_TITLE_TEMPLATE_PATH) from exceptions import (DatabaseConnectionException, FileNotFoundException, MeDaCaException, ParameterNotFoundException, QueryExecutionException, SNSPublishException) @@ -28,7 +30,7 @@ logger = MeDaCaLogger.get_logger() def handler(event, context): try: logger.info('I-01-01', '処理開始　Viewセキュリティオプション付与チェック') - logger.info('I-02-02', 'チェック対象スキーマ名ファイルを読み込み 開始') + logger.info('I-02-01', 'チェック対象スキーマ名ファイルを読み込み 開始') check_target_schemas = read_check_target_schemas() logger.info('I-02-02', f'チェック対象スキーマ名ファイルを読み込み 終了　チェック対象スキーマ名：{check_target_schemas}') # print(check_target_schemas) @@ -36,7 +38,7 @@ def handler(event, context): # DB接続のためのパラメータ取得 db_host, db_user_name, db_user_password = read_db_param_from_parameter_store() connection = connection_database(db_host, db_user_name, db_user_password) - logger.info('I-03-01', 'データベースへの接続開始 終了') + logger.info('I-03-02', 'データベースへの接続開始 成功') logger.info('I-04-01', 'Viewセキュリティオプション チェック開始') check_result = fetch_view_security_options(connection, check_target_schemas) @@ -45,9 +47,18 @@ def handler(event, context): return logger.info('I-04-01', 'Viewセキュリティオプション 未設定のViewがあるため、メール送信処理を開始します。') - view_security_options = [ViewSecurityOption(*row) for row in check_result] - - mail_title, mail_body = make_notice_mail(view_security_options) + no_security_option_views = [NoSecurityOptionView(*row) for row in check_result] + logger.info( + 'I-05-02', f'通知メール（タイトル）テンプレートファイル読込　読込元：{CONFIG_BUCKET_NAME}/{NOTICE_MAIL_TITLE_TEMPLATE_PATH}') + mail_title = read_mail_title() + logger.info( + 'I-05-03', '通知メール（タイトル）テンプレートファイルを読み込みました') + logger.info( + 'I-05-04', f'通知メール（本文）テンプレートファイル読込　読込元：{CONFIG_BUCKET_NAME}/{NOTICE_MAIL_BODY_TEMPLATE_PATH}') + mail_body_template = read_mail_body_template() + logger.info( + 'I-05-05', '通知メール(本文）テンプレートファイルを読み込みました') + mail_body = make_notice_mail_body(no_security_option_views, mail_body_template) logger.info('I-05-06', f'メール送信指示をします　送信先トピック：{MBJ_NOTICE_TOPIC}') notice_to_mbj(mail_title, mail_body) @@ -55,10 +66,12 @@ def handler(event, context): except MeDaCaException as e: logger.exception(e.error_id, e) + logger.error(f'処理異常通知の送信指示をしました　通知先トピック：{NDS_NOTICE_TOPIC}') notice_to_nds(e.error_id, e) raise e except Exception as e: logger.exception('E-99', f'想定外のエラーが発生しました　エラー内容：{e}') + logger.error('E-ERR-01', f'処理異常通知の送信指示をしました　通知先トピック：{NDS_NOTICE_TOPIC}') notice_to_nds('E-99', e) raise e finally: @@ -104,7 +117,7 @@ def read_db_param_from_parameter_store() -> tuple: return db_host, db_user_name, db_user_password except botocore.exceptions.ClientError as e: if e.response[RESPONSE_ERROR][RESPONSE_ERROR_CODE] == RESPONSE_CODE_PARAMETER_NOT_FOUND: - raise ParameterNotFoundException('E-03-02', f'パラメータストアの取得に失敗しました　エラー内容：{e}') + raise ParameterNotFoundException('E-03-01', f'パラメータストアの取得に失敗しました　エラー内容：{e}') else: raise Exception(e) @@ -162,30 +175,27 @@ def fetch_view_security_options(connection: Database, check_target_schemas: list result = cursor.fetchall() return result except Exception as e: - raise QueryExecutionException('E-03-02', f'Viewセキュリティオプションチェックに失敗しました　エラー内容：{e}') + raise QueryExecutionException('E-04-01', f'Viewセキュリティオプションチェックに失敗しました　エラー内容：{e}') -def make_notice_mail(view_security_options: list[ViewSecurityOption]) -> tuple[str]: - config_bucket = ConfigBucket() - logger.info( - 'I-05-02', f'通知メール（タイトル）テンプレートファイル読込　読込元：{config_bucket.bucket_name}/{config_bucket.mail_title_file_path}') - mail_title_template = read_mail_title(config_bucket) - logger.info( - 'I-05-02', f'通知メール（本文）テンプレートファイル読込　読込元：{config_bucket.bucket_name}/{config_bucket.mail_body_file_path}') - mail_body_template = read_mail_body_template(config_bucket) - - mail_message = MAIL_INDENT.join([f'{option.schema_name}.{option.table_name}' for option in view_security_options]) - - mail_body = mail_body_template.format(no_option_views=mail_message) - - return mail_title_template, mail_body - - -def read_mail_title(config_bucket: ConfigBucket) -> str: - """メールタイトルを読み込む +def make_notice_mail_body(no_security_option_views: list[NoSecurityOptionView], mail_body_template: str) -> tuple[str]: + """メール本文を生成します Args: - config_bucket (ConfigBucket): 設定ファイル保管バケット操作クラス + view_security_options (list[NoSecurityOptionView]): チェック対象のビュー一覧 + mail_body_template (str): メール本文のテンプレート + + Returns: + tuple[str]: メール本文 + """ + mail_message = MAIL_INDENT.join( + [f'{option.schema_name}.{option.table_name}' for option in no_security_option_views]) + mail_body = mail_body_template.format(no_option_views=mail_message) + return mail_body + + +def read_mail_title() -> str: + """メールタイトルを読み込む Raises: FileNotFoundException: ファイルが読み込めなかったエラー @@ -196,22 +206,20 @@ def read_mail_title(config_bucket: ConfigBucket) -> str: """ try: + config_bucket = ConfigBucket() mail_title = config_bucket.notice_mail_title_template except botocore.exceptions.ClientError as e: if e.response[RESPONSE_ERROR][RESPONSE_ERROR_CODE] == RESPONSE_CODE_NO_SUCH_KEY: - raise FileNotFoundException('E-02-01', f'通知メール（タイトル）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') + raise FileNotFoundException('E-05-01', f'通知メール（タイトル）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') else: raise Exception(e) return mail_title -def read_mail_body_template(config_bucket: ConfigBucket) -> str: +def read_mail_body_template() -> str: """メール本文を読み込む - Args: - config_bucket (ConfigBucket): 設定ファイル保管バケット操作クラス - Raises: FileNotFoundException: ファイルが読み込めなかったエラー Exception: 想定外のエラー @@ -220,10 +228,11 @@ def read_mail_body_template(config_bucket: ConfigBucket) -> str: str: メール本文 """ try: + config_bucket = ConfigBucket() mail_body_template = config_bucket.notice_mail_body_template except botocore.exceptions.ClientError as e: if e.response[RESPONSE_ERROR][RESPONSE_ERROR_CODE] == RESPONSE_CODE_NO_SUCH_KEY: - raise FileNotFoundException('E-02-01', f'通知メール（本文）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') + raise FileNotFoundException('E-05-02', f'通知メール（本文）テンプレートファイルの読み込みに失敗しました　エラー内容：{e}') else: raise Exception(e) From 2109e721be2991e1dabc1f0bec137fc5c8aa895d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 7 Jul 2022 15:16:47 +0900 Subject: [PATCH 16/23] =?UTF-8?q?fix:=20=E3=83=AD=E3=82=B0=E3=81=AEID?= =?UTF-8?q?=E9=96=93=E9=81=95=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/check-view-secutiry-option/check-view-option/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index ff2361d8..504388fb 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -45,7 +45,7 @@ def handler(event, context): if len(check_result) == 0: logger.info('I-04-02', 'Viewセキュリティオプション 未設定のViewはありません。処理を終了します。') return - logger.info('I-04-01', 'Viewセキュリティオプション 未設定のViewがあるため、メール送信処理を開始します。') + logger.info('I-05-01', 'Viewセキュリティオプション 未設定のViewがあるため、メール送信処理を開始します。') no_security_option_views = [NoSecurityOptionView(*row) for row in check_result] logger.info( From 218215cb87ec4375379a9901d8caa4713c788065 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 7 Jul 2022 15:29:09 +0900 Subject: [PATCH 17/23] =?UTF-8?q?refactor:=E4=BD=BF=E3=81=A3=E3=81=A6?= =?UTF-8?q?=E3=81=84=E3=81=AA=E3=81=84=E5=A4=89=E6=95=B0=E3=82=92=E5=89=8A?= =?UTF-8?q?=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/check-view-secutiry-option/check-view-option/aws/s3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py index 7074b663..55c33c81 100644 --- a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py +++ b/lambda/check-view-secutiry-option/check-view-option/aws/s3.py @@ -17,7 +17,6 @@ class S3Resource: class ConfigBucket: __s3_resource: S3Resource = None - __bucket_name: str def __init__(self) -> None: self.__s3_resource = S3Resource(environments.CONFIG_BUCKET_NAME) From fc8ecccf2115e5f801cdc02db4b4162f59f57706 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 8 Jul 2022 15:12:52 +0900 Subject: [PATCH 18/23] =?UTF-8?q?refactor:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/37#comment-1247185 --- .../check-view-secutiry-option/check-view-option/main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index 504388fb..c34dae43 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -29,24 +29,26 @@ logger = MeDaCaLogger.get_logger() def handler(event, context): try: + # ① 処理開始ログを出力する logger.info('I-01-01', '処理開始　Viewセキュリティオプション付与チェック') + # ② 設定ファイル[チェック対象スキーマ名ファイル]を読み込む logger.info('I-02-01', 'チェック対象スキーマ名ファイルを読み込み 開始') check_target_schemas = read_check_target_schemas() logger.info('I-02-02', f'チェック対象スキーマ名ファイルを読み込み 終了　チェック対象スキーマ名：{check_target_schemas}') - # print(check_target_schemas) + # ③ データベースに接続する logger.info('I-03-01', 'データベースへの接続開始 開始') # DB接続のためのパラメータ取得 db_host, db_user_name, db_user_password = read_db_param_from_parameter_store() connection = connection_database(db_host, db_user_name, db_user_password) logger.info('I-03-02', 'データベースへの接続開始 成功') + # ④ Viewのオプションを確認するため、データを取得する logger.info('I-04-01', 'Viewセキュリティオプション チェック開始') check_result = fetch_view_security_options(connection, check_target_schemas) - if len(check_result) == 0: logger.info('I-04-02', 'Viewセキュリティオプション 未設定のViewはありません。処理を終了します。') return logger.info('I-05-01', 'Viewセキュリティオプション 未設定のViewがあるため、メール送信処理を開始します。') - + # ⑤ 取得できたデータをもとに、メール通知する文言を作成する no_security_option_views = [NoSecurityOptionView(*row) for row in check_result] logger.info( 'I-05-02', f'通知メール（タイトル）テンプレートファイル読込　読込元：{CONFIG_BUCKET_NAME}/{NOTICE_MAIL_TITLE_TEMPLATE_PATH}') @@ -75,6 +77,7 @@ def handler(event, context): notice_to_nds('E-99', e) raise e finally: + # ⑥ 処理終了ログを出力する logger.info('I-06-01', '処理終了　Viewセキュリティオプション付与チェック') From 5e1f737be983768b30a520694a450dcf382c81b8 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 8 Jul 2022 18:00:26 +0900 Subject: [PATCH 19/23] =?UTF-8?q?fix:=20typo=E4=BF=AE=E6=AD=A3=20secutiry?= =?UTF-8?q?=E2=86=92security,=20NDS=E5=90=91=E3=81=91=E9=80=9A=E7=9F=A5?= =?UTF-8?q?=E5=87=A6=E7=90=86=E3=81=AE=E3=82=A8=E3=83=A9=E3=83=BCID?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E6=BC=8F=E3=82=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/check-view-secutiry-option/check-view-option/main.py | 2 +- ..._body.config => check_view_security_option_mail_body.config} | 0 ...itle.config => check_view_security_option_mail_title.config} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename s3/config/view_check/{check_view_secutiry_option_mail_body.config => check_view_security_option_mail_body.config} (100%) rename s3/config/view_check/{check_view_secutiry_option_mail_title.config => check_view_security_option_mail_title.config} (100%) diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-secutiry-option/check-view-option/main.py index c34dae43..90dad25f 100644 --- a/lambda/check-view-secutiry-option/check-view-option/main.py +++ b/lambda/check-view-secutiry-option/check-view-option/main.py @@ -68,7 +68,7 @@ def handler(event, context): except MeDaCaException as e: logger.exception(e.error_id, e) - logger.error(f'処理異常通知の送信指示をしました　通知先トピック：{NDS_NOTICE_TOPIC}') + logger.error('E-ERR-01', f'処理異常通知の送信指示をしました　通知先トピック：{NDS_NOTICE_TOPIC}') notice_to_nds(e.error_id, e) raise e except Exception as e: diff --git a/s3/config/view_check/check_view_secutiry_option_mail_body.config b/s3/config/view_check/check_view_security_option_mail_body.config similarity index 100% rename from s3/config/view_check/check_view_secutiry_option_mail_body.config rename to s3/config/view_check/check_view_security_option_mail_body.config diff --git a/s3/config/view_check/check_view_secutiry_option_mail_title.config b/s3/config/view_check/check_view_security_option_mail_title.config similarity index 100% rename from s3/config/view_check/check_view_secutiry_option_mail_title.config rename to s3/config/view_check/check_view_security_option_mail_title.config From bb77ae9368f2a03a2c6568c86ecf13fa73522418 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 11 Jul 2022 08:44:03 +0900 Subject: [PATCH 20/23] =?UTF-8?q?fix:=20typo=E4=BF=AE=E6=AD=A3=20secutiry?= =?UTF-8?q?=E2=86=92security?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Dockerfile | 0 .../Pipfile | 0 .../Pipfile.lock | 0 .../check-view-option/aws/__init__.py | 0 .../check-view-option/aws/s3.py | 0 .../check-view-option/aws/sns.py | 0 .../check-view-option/aws/ssm.py | 0 .../check-view-option/constants.py | 0 .../check-view-option/database.py | 0 .../check-view-option/dto/__init__.py | 0 .../check-view-option/dto/no_security_option_view.py | 0 .../check-view-option/environments.py | 0 .../check-view-option/exceptions.py | 0 .../check-view-option/main.py | 0 .../check-view-option/medaca_logger.py | 0 15 files changed, 0 insertions(+), 0 deletions(-) rename lambda/{check-view-secutiry-option => check-view-security-option}/Dockerfile (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/Pipfile (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/Pipfile.lock (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/aws/__init__.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/aws/s3.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/aws/sns.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/aws/ssm.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/constants.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/database.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/dto/__init__.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/dto/no_security_option_view.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/environments.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/exceptions.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/main.py (100%) rename lambda/{check-view-secutiry-option => check-view-security-option}/check-view-option/medaca_logger.py (100%) diff --git a/lambda/check-view-secutiry-option/Dockerfile b/lambda/check-view-security-option/Dockerfile similarity index 100% rename from lambda/check-view-secutiry-option/Dockerfile rename to lambda/check-view-security-option/Dockerfile diff --git a/lambda/check-view-secutiry-option/Pipfile b/lambda/check-view-security-option/Pipfile similarity index 100% rename from lambda/check-view-secutiry-option/Pipfile rename to lambda/check-view-security-option/Pipfile diff --git a/lambda/check-view-secutiry-option/Pipfile.lock b/lambda/check-view-security-option/Pipfile.lock similarity index 100% rename from lambda/check-view-secutiry-option/Pipfile.lock rename to lambda/check-view-security-option/Pipfile.lock diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/__init__.py b/lambda/check-view-security-option/check-view-option/aws/__init__.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/aws/__init__.py rename to lambda/check-view-security-option/check-view-option/aws/__init__.py diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/s3.py b/lambda/check-view-security-option/check-view-option/aws/s3.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/aws/s3.py rename to lambda/check-view-security-option/check-view-option/aws/s3.py diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/sns.py b/lambda/check-view-security-option/check-view-option/aws/sns.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/aws/sns.py rename to lambda/check-view-security-option/check-view-option/aws/sns.py diff --git a/lambda/check-view-secutiry-option/check-view-option/aws/ssm.py b/lambda/check-view-security-option/check-view-option/aws/ssm.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/aws/ssm.py rename to lambda/check-view-security-option/check-view-option/aws/ssm.py diff --git a/lambda/check-view-secutiry-option/check-view-option/constants.py b/lambda/check-view-security-option/check-view-option/constants.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/constants.py rename to lambda/check-view-security-option/check-view-option/constants.py diff --git a/lambda/check-view-secutiry-option/check-view-option/database.py b/lambda/check-view-security-option/check-view-option/database.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/database.py rename to lambda/check-view-security-option/check-view-option/database.py diff --git a/lambda/check-view-secutiry-option/check-view-option/dto/__init__.py b/lambda/check-view-security-option/check-view-option/dto/__init__.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/dto/__init__.py rename to lambda/check-view-security-option/check-view-option/dto/__init__.py diff --git a/lambda/check-view-secutiry-option/check-view-option/dto/no_security_option_view.py b/lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/dto/no_security_option_view.py rename to lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py diff --git a/lambda/check-view-secutiry-option/check-view-option/environments.py b/lambda/check-view-security-option/check-view-option/environments.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/environments.py rename to lambda/check-view-security-option/check-view-option/environments.py diff --git a/lambda/check-view-secutiry-option/check-view-option/exceptions.py b/lambda/check-view-security-option/check-view-option/exceptions.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/exceptions.py rename to lambda/check-view-security-option/check-view-option/exceptions.py diff --git a/lambda/check-view-secutiry-option/check-view-option/main.py b/lambda/check-view-security-option/check-view-option/main.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/main.py rename to lambda/check-view-security-option/check-view-option/main.py diff --git a/lambda/check-view-secutiry-option/check-view-option/medaca_logger.py b/lambda/check-view-security-option/check-view-option/medaca_logger.py similarity index 100% rename from lambda/check-view-secutiry-option/check-view-option/medaca_logger.py rename to lambda/check-view-security-option/check-view-option/medaca_logger.py From bd515960d9e97d957e0d0ddffce32dd911606929 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 11 Jul 2022 09:58:58 +0900 Subject: [PATCH 21/23] =?UTF-8?q?feat:=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E6=8C=87=E6=91=98=E4=BA=8B=E9=A0=85=E4=BF=AE=E6=AD=A3=20https:?= =?UTF-8?q?//nds-tyo.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/3?= =?UTF-8?q?7#comment-1247549=20https://nds-tyo.backlog.com/git/NEWDWH2021/?= =?UTF-8?q?newsdwh2021/pullRequests/37#comment-1247558?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/dto/no_security_option_view.py | 1 + lambda/check-view-security-option/check-view-option/main.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py b/lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py index 3a58e873..a349cc80 100644 --- a/lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py +++ b/lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py @@ -5,3 +5,4 @@ from dataclasses import dataclass class NoSecurityOptionView: schema_name: str table_name: str + security_type: str diff --git a/lambda/check-view-security-option/check-view-option/main.py b/lambda/check-view-security-option/check-view-option/main.py index 90dad25f..9cfd19bb 100644 --- a/lambda/check-view-security-option/check-view-option/main.py +++ b/lambda/check-view-security-option/check-view-option/main.py @@ -44,6 +44,7 @@ def handler(event, context): # ④ Viewのオプションを確認するため、データを取得する logger.info('I-04-01', 'Viewセキュリティオプション チェック開始') check_result = fetch_view_security_options(connection, check_target_schemas) + logger.debug('D-04-01', f'取得データ：{check_result}') if len(check_result) == 0: logger.info('I-04-02', 'Viewセキュリティオプション 未設定のViewはありません。処理を終了します。') return @@ -164,7 +165,8 @@ def fetch_view_security_options(connection: Database, check_target_schemas: list select_view_security_option_sql = f""" SELECT TABLE_SCHEMA, - TABLE_NAME + TABLE_NAME, + SECURITY_TYPE FROM INFORMATION_SCHEMA.VIEWS WHERE From ee0ad723e4f2291c2a129b7872e1173a71f1fbaa Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 11 Jul 2022 11:20:38 +0900 Subject: [PATCH 22/23] =?UTF-8?q?refactor:=20=E5=8F=96=E5=BE=97=E3=82=AB?= =?UTF-8?q?=E3=83=A9=E3=83=A0=E3=81=ABDEFINER=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/check-view-security-option/check-view-option/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/check-view-security-option/check-view-option/main.py b/lambda/check-view-security-option/check-view-option/main.py index 9cfd19bb..c40d8c51 100644 --- a/lambda/check-view-security-option/check-view-option/main.py +++ b/lambda/check-view-security-option/check-view-option/main.py @@ -166,7 +166,7 @@ def fetch_view_security_options(connection: Database, check_target_schemas: list SELECT TABLE_SCHEMA, TABLE_NAME, - SECURITY_TYPE + DEFINER FROM INFORMATION_SCHEMA.VIEWS WHERE From 64b0f951fd109dbe95cc16ddf93332ff078d985b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 11 Jul 2022 11:27:52 +0900 Subject: [PATCH 23/23] =?UTF-8?q?refactor:=E3=82=AF=E3=82=A8=E3=83=AA?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=81=AB=E3=81=A8=E3=82=82=E3=81=AA=E3=81=86?= =?UTF-8?q?DTO=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../check-view-option/dto/no_security_option_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py b/lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py index a349cc80..5a5fb39c 100644 --- a/lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py +++ b/lambda/check-view-security-option/check-view-option/dto/no_security_option_view.py @@ -5,4 +5,4 @@ from dataclasses import dataclass class NoSecurityOptionView: schema_name: str table_name: str - security_type: str + definer: str