From 2b68a9f054d1e2431818402e583b39a5719f32bd Mon Sep 17 00:00:00 2001 From: "maruyama.t" Date: Wed, 13 Mar 2024 07:54:10 +0000 Subject: [PATCH] =?UTF-8?q?Merged=20PR=20824:=20AzureFunctions=E5=AE=9F?= =?UTF-8?q?=E8=A3=853=EF=BC=88CSV=E3=82=92=E3=82=B9=E3=83=88=E3=83=AC?= =?UTF-8?q?=E3=83=BC=E3=82=B8=E3=82=A2=E3=82=AB=E3=82=A6=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=81=AB=E9=85=8D=E7=BD=AE=E3=81=99=E3=82=8B=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3846: AzureFunctions実装3（CSVをストレージアカウントに配置する）](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3846) outputDataを追加 →outputAnalysisLicensesDataに変更（アラートルールなどでログを見るので、何の処理か理解できるように） blobstorageService.tsに以下を追加 - uploadFileAnalysisLicensesCSV （ライセンスCSVを配置する） - createContainerAnalysisを追加 （コンテナーを作成する） 環境変数の追加 ## レビューポイント - 今回追加されたJP-EASTのストレージアカウントのコンテナーが、第一階層のアカウントのものであるかどうかはソース上は特に意識していないが問題ないでしょうか。 ## 動作確認状況 - ローカルで確認（モックでソース上処理が通ることのみ確認のみ） 詳細なテストは別タスクで行う。 ## 補足 - 相談、参考資料などがあれば --- .../src/blobstorage/blobstorage.service.ts | 115 +++++++++++++++--- dictation_function/src/constants/index.ts | 12 ++ .../src/functions/analysisLicenses.ts | 100 ++++++++++++--- .../functions/analysisLicensesManualRetry.ts | 2 +- .../src/test/analysisLicenses.spec.ts | 92 ++++++++++++++ .../blobstorage/blobstorage.service.ts | 2 - 6 files changed, 286 insertions(+), 37 deletions(-) diff --git a/dictation_function/src/blobstorage/blobstorage.service.ts b/dictation_function/src/blobstorage/blobstorage.service.ts index d9b4472..2adfee4 100644 --- a/dictation_function/src/blobstorage/blobstorage.service.ts +++ b/dictation_function/src/blobstorage/blobstorage.service.ts @@ -2,29 +2,52 @@ import { BlobServiceClient, StorageSharedKeyCredential, } from "@azure/storage-blob"; -import { IMPORT_USERS_CONTAINER_NAME } from "../constants"; +import { + IMPORT_USERS_CONTAINER_NAME, + LICENSE_COUNT_ANALYSIS_CONTAINER_NAME, +} from "../constants"; import { InvocationContext } from "@azure/functions"; export class BlobstorageService { private readonly blobServiceClient: BlobServiceClient; private readonly sharedKeyCredential: StorageSharedKeyCredential; - constructor() { - if ( - !process.env.STORAGE_ACCOUNT_NAME_IMPORT || - !process.env.STORAGE_ACCOUNT_KEY_IMPORT || - !process.env.STORAGE_ACCOUNT_ENDPOINT_IMPORT - ) { - throw new Error("Storage account information is missing"); - } - this.sharedKeyCredential = new StorageSharedKeyCredential( - process.env.STORAGE_ACCOUNT_NAME_IMPORT, - process.env.STORAGE_ACCOUNT_KEY_IMPORT - ); - this.blobServiceClient = new BlobServiceClient( - process.env.STORAGE_ACCOUNT_ENDPOINT_IMPORT, - this.sharedKeyCredential - ); + constructor(useAnalysisBlob: boolean = false) { + if (!useAnalysisBlob) { + if ( + !process.env.STORAGE_ACCOUNT_NAME_IMPORT || + !process.env.STORAGE_ACCOUNT_KEY_IMPORT || + !process.env.STORAGE_ACCOUNT_ENDPOINT_IMPORT + ) { + throw new Error("Storage account information is missing"); + } + + this.sharedKeyCredential = new StorageSharedKeyCredential( + process.env.STORAGE_ACCOUNT_NAME_IMPORT, + process.env.STORAGE_ACCOUNT_KEY_IMPORT + ); + this.blobServiceClient = new BlobServiceClient( + process.env.STORAGE_ACCOUNT_ENDPOINT_IMPORT, + this.sharedKeyCredential + ); + } else { + if ( + !process.env.STORAGE_ACCOUNT_NAME_ANALYSIS || + !process.env.STORAGE_ACCOUNT_KEY_ANALYSIS || + !process.env.STORAGE_ACCOUNT_ENDPOINT_ANALYSIS + ) { + throw new Error("Storage account information for analysis is missing"); + } + + this.sharedKeyCredential = new StorageSharedKeyCredential( + process.env.STORAGE_ACCOUNT_NAME_ANALYSIS, + process.env.STORAGE_ACCOUNT_KEY_ANALYSIS + ); + this.blobServiceClient = new BlobServiceClient( + process.env.STORAGE_ACCOUNT_ENDPOINT_ANALYSIS, + this.sharedKeyCredential + ); + } } /** * Lists blobs @@ -181,4 +204,62 @@ export class BlobstorageService { context.log(`[OUT] ${this.isFileExists.name}`); } } + + /** + * uplocad file analysis licenses csv + * @param context + * @param filename + * @param data + * @returns boolean + */ + public async uploadFileAnalysisLicensesCSV( + context: InvocationContext, + filename: string, + data: string + ): Promise { + context.log( + `[IN] ${this.uploadFileAnalysisLicensesCSV.name} | params: { filename: ${filename} }` + ); + try { + const containerClient = this.blobServiceClient.getContainerClient( + LICENSE_COUNT_ANALYSIS_CONTAINER_NAME + ); + const { response } = await containerClient.uploadBlockBlob( + filename, + data, + data.length + ); + if (response.errorCode) { + context.log(`update failed. response errorCode: ${response.errorCode}`); + return false; + } + return true; + } catch (e) { + context.error(e); + throw e; + } finally { + context.log(`[OUT] ${this.uploadFileAnalysisLicensesCSV.name}`); + } + } + /** + * Create container analysis + * @param context + * @returns container + */ + async createContainerAnalysis(context: InvocationContext): Promise { + context.log(`[IN] ${this.createContainerAnalysis.name}`); + + try { + // コンテナ作成 + const containerClient = this.blobServiceClient.getContainerClient( + LICENSE_COUNT_ANALYSIS_CONTAINER_NAME + ); + await containerClient.create(); + } catch (e) { + context.error(e); + throw e; + } finally { + context.log(`[OUT] ${this.createContainerAnalysis.name}`); + } + } } diff --git a/dictation_function/src/constants/index.ts b/dictation_function/src/constants/index.ts index 0c31242..6f94278 100644 --- a/dictation_function/src/constants/index.ts +++ b/dictation_function/src/constants/index.ts @@ -390,3 +390,15 @@ export const LICENSE_COUNT_ANALYSIS_ROLE = { NONE: "None", UNALLOCATED: "Unallocated", }; + +/** + * ライセンス数推移出力機能のファイルの先頭文字列 + * @const {string[]} + */ +export const LICENSE_COUNT_ANALYSIS_FRONT_STRING = "LicenseAggregated"; + +/** + * ライセンス数推移CSV用のコンテナー名 + * @const {string} + */ +export const LICENSE_COUNT_ANALYSIS_CONTAINER_NAME = "analysis-licenses"; diff --git a/dictation_function/src/functions/analysisLicenses.ts b/dictation_function/src/functions/analysisLicenses.ts index 515b3aa..b0caa71 100644 --- a/dictation_function/src/functions/analysisLicenses.ts +++ b/dictation_function/src/functions/analysisLicenses.ts @@ -23,9 +23,8 @@ import { BLOB_STORAGE_REGION_EU, LICENSE_TYPE, LICENSE_COUNT_ANALYSIS_HEADER, + LICENSE_COUNT_ANALYSIS_FRONT_STRING, } from "../constants"; -import * as fs from "fs"; -import * as path from "path"; import { DateWithDayEndTime } from "../common/types/types"; import { initializeDataSource } from "../database/initializeDataSource"; @@ -55,8 +54,11 @@ export async function analysisLicensesProcessing( baseDataFromDeletedAccounts, targetMonthYYYYMM ); - // TODO: 後続処理の呼び出しイメージ（別タスクで追加） - // await outputData(context, blobstorageService, outputCsvData); + await outputAnalysisLicensesData( + context, + blobstorageService, + outputCsvData + ); } catch (e) { context.log("analysisLicensesProcessing failed."); context.error(e); @@ -336,7 +338,7 @@ export async function analysisLicenses( dotenv.config({ path: ".env.local", override: true }); try { const datasource = await initializeDataSource(context); - const blobstorageService = new BlobstorageService(); + const blobstorageService = new BlobstorageService(true); try { // 現在の日付より、先月の年月をYYYYMM形式で取得 @@ -1321,18 +1323,6 @@ export async function transferData( ); } } - // outputDataUSをローカルにCSV出力(テスト用、別タスクで消す) - // outputDataUSの配列をCSV形式に変換 - /*let csvContentUS = ""; - for (let i = 0; i < outputDataUS.length; i++) { - //カンマ区切りの文字列を作成 - - csvContentUS += outputDataUS[i]; - } - // CSVファイルを出力 - const filePathUS = path.join(__dirname, "outputDataUS.csv"); - fs.writeFileSync(filePathUS, csvContentUS); - */ return { outputDataUS: outputDataUS, outputDataEU: outputDataEU, @@ -1775,3 +1765,79 @@ async function createOutputData( context.log("[OUT]createOutputData"); } } + +/** + * 出力データを第一階層のストレージアカウントにCSVファイルとして出力する + * @param context + * @param blobstorageService: BlobstorageService, + * @param outputCsvData: outputDataAnalysisLicensesCSV + */ +export async function outputAnalysisLicensesData( + context: InvocationContext, + blobstorageService: BlobstorageService, + outputCsvData: outputDataAnalysisLicensesCSV +): Promise { + context.log("[IN]outputAnalysisLicensesData"); + try { + let csvContentUS = ""; + let csvContentEU = ""; + let csvContentAU = ""; + // 出力日時を取得 + const outputDateTime = new Date().toISOString(); + // YYYYMMDDHH24MISS形式に変換 + const outputDateTimeYYYYMMDDHH24MISS = outputDateTime.replace(/[-T:]/g, ""); + // 出力ファイル名を作成 + const outputFileNameUS = + LICENSE_COUNT_ANALYSIS_FRONT_STRING + + `_US_${outputDateTimeYYYYMMDDHH24MISS}.csv`; + const outputFileNameEU = + LICENSE_COUNT_ANALYSIS_FRONT_STRING + + `_EU_${outputDateTimeYYYYMMDDHH24MISS}.csv`; + const outputFileNameAU = + LICENSE_COUNT_ANALYSIS_FRONT_STRING + + `_AU_${outputDateTimeYYYYMMDDHH24MISS}.csv`; + + // outputDataUSの配列をCSV形式に変換 + for (let i = 0; i < outputCsvData.outputDataUS.length; i++) { + //カンマ区切りの文字列を作成 + csvContentUS += outputCsvData.outputDataUS[i]; + } + // outputDataEUの配列をCSV形式に変換 + for (let i = 0; i < outputCsvData.outputDataEU.length; i++) { + //カンマ区切りの文字列を作成 + csvContentEU += outputCsvData.outputDataEU[i]; + } + // outputDataAUの配列をCSV形式に変換 + for (let i = 0; i < outputCsvData.outputDataAU.length; i++) { + //カンマ区切りの文字列を作成 + csvContentAU += outputCsvData.outputDataAU[i]; + } + await blobstorageService.createContainerAnalysis(context); + // 出力ファイル名を指定して出力 + const resultUS = await blobstorageService.uploadFileAnalysisLicensesCSV( + context, + outputFileNameUS, + outputCsvData.outputDataUS.join("\r\n") + ); + context.log("resultUS: " + resultUS); + const resultEU = await blobstorageService.uploadFileAnalysisLicensesCSV( + context, + outputFileNameEU, + outputCsvData.outputDataEU.join("\r\n") + ); + const resultAU = await blobstorageService.uploadFileAnalysisLicensesCSV( + context, + outputFileNameAU, + outputCsvData.outputDataAU.join("\r\n") + ); + // 出力結果を返却 + // 3つのリージョンの出力が全て成功した場合にtrueを返却 + return resultUS && resultEU && resultAU; + } catch (e) { + context.log("outputAnalysisLicensesData failed."); + context.error(e); + throw e; + } finally { + context.log("[OUT]outputAnalysisLicensesData"); + } +} diff --git a/dictation_function/src/functions/analysisLicensesManualRetry.ts b/dictation_function/src/functions/analysisLicensesManualRetry.ts index d21c61b..8b9dfaf 100644 --- a/dictation_function/src/functions/analysisLicensesManualRetry.ts +++ b/dictation_function/src/functions/analysisLicensesManualRetry.ts @@ -23,7 +23,7 @@ export async function analysisLicensesManualRetry( dotenv.config({ path: ".env" }); dotenv.config({ path: ".env.local", override: true }); const datasource = await initializeDataSource(context); - const blobstorageService = new BlobstorageService(); + const blobstorageService = new BlobstorageService(true); try { // 現在の日付より、先月の年月をYYYYMM形式で取得 diff --git a/dictation_function/src/test/analysisLicenses.spec.ts b/dictation_function/src/test/analysisLicenses.spec.ts index 4545ca7..0330d68 100644 --- a/dictation_function/src/test/analysisLicenses.spec.ts +++ b/dictation_function/src/test/analysisLicenses.spec.ts @@ -2,6 +2,7 @@ import { DataSource } from "typeorm"; import { getBaseData, getBaseDataFromDeletedAccounts, + outputAnalysisLicensesData, transferData, } from "../functions/analysisLicenses"; import { @@ -27,6 +28,7 @@ import { LICENSE_COUNT_ANALYSIS_LICENSE_TYPE, SWITCH_FROM_TYPE, } from "../constants"; +import { BlobstorageService } from "../blobstorage/blobstorage.service"; describe("analysisLicenses", () => { dotenv.config({ path: ".env" }); dotenv.config({ path: ".env.local", override: true }); @@ -1221,4 +1223,94 @@ describe("analysisLicenses", () => { expect(transferDataResult.outputDataUS[12]).toEqual(`"",`); expect(transferDataResult.outputDataUS[13]).toEqual(`"3"` + "\r\n"); }); + + it("outputDataの確認（Mock）", async () => { + const blobService = new BlobstorageService(); + + if (!source) fail(); + const context = new InvocationContext(); + + const currentDate = new DateWithZeroTime(); + const expiringSoonDate = new ExpirationThresholdDate(currentDate.getTime()); + + // 現在の日付を取得 + const nowDate = new Date(); + + // 先月の日付を取得 + const lastMonth = new Date(nowDate); + lastMonth.setMonth(nowDate.getMonth() - 1); + const lastMonthYYYYMM = `${lastMonth.getFullYear()}${( + lastMonth.getMonth() + 1 + ) + .toString() + .padStart(2, "0")}`; + + // 先々月の日付を取得 + const last2Month = new Date(nowDate); + last2Month.setMonth(nowDate.getMonth() - 2); + const last2MonthYYYYMM = `${last2Month.getFullYear()}${( + last2Month.getMonth() + 1 + ) + .toString() + .padStart(2, "0")}`; + + // tier4とtier5のアカウント＋管理者を作る + const { account: account4, admin: admin4 } = await makeTestAccount( + source, + { tier: 4 }, + { external_id: "external_id_tier4admin" } + ); + const { account: account5_1, admin: admin5_1_1 } = await makeTestAccount( + source, + { + tier: 5, + parent_account_id: account4.id, + }, + { + external_id: "external_id_tier5admin1", + role: "author", + } + ); + + // 所有ライセンス + // usedTrialLicensesAuthorCount 1件 + await createLicense( + source, + 1, + null, + account5_1.id, + "TRIAL", + LICENSE_ALLOCATED_STATUS.ALLOCATED, + admin5_1_1.id, + null, + null, + null, + last2Month + ); + + const result = await getBaseData(context, lastMonthYYYYMM, source); + const result_D = await getBaseDataFromDeletedAccounts( + context, + lastMonthYYYYMM, + source + ); + const transferDataResult = await transferData( + context, + result, + result_D, + lastMonthYYYYMM + ); + const mockUploadFileAnalysisLicensesCSV = jest.fn().mockReturnValue(true); + const mockCreateContainerAnalysis = jest.fn().mockReturnValue(true); + blobService.uploadFileAnalysisLicensesCSV = + mockUploadFileAnalysisLicensesCSV; + blobService.createContainerAnalysis = mockCreateContainerAnalysis; + + const resultOutputData = await outputAnalysisLicensesData( + context, + blobService, + transferDataResult + ); + expect(resultOutputData).toEqual(true); + }); }); diff --git a/dictation_server/src/gateways/blobstorage/blobstorage.service.ts b/dictation_server/src/gateways/blobstorage/blobstorage.service.ts index baaf57e..fcd8301 100644 --- a/dictation_server/src/gateways/blobstorage/blobstorage.service.ts +++ b/dictation_server/src/gateways/blobstorage/blobstorage.service.ts @@ -45,7 +45,6 @@ export class BlobstorageService { this.configService.getOrThrow('STORAGE_ACCOUNT_NAME_IMPORTS'), this.configService.getOrThrow('STORAGE_ACCOUNT_KEY_IMPORTS'), ); - this.blobServiceClientUS = new BlobServiceClient( this.configService.getOrThrow('STORAGE_ACCOUNT_ENDPOINT_US'), this.sharedKeyCredentialUS, @@ -104,7 +103,6 @@ export class BlobstorageService { ); } } - /** * 指定されたコンテナを削除します。（コンテナが存在しない場合、何もせず終了します） * @param context