From f0d71937e336af153dd07716cdd8a2c14439d5e6 Mon Sep 17 00:00:00 2001 From: "maruyama.t" Date: Tue, 27 Feb 2024 06:24:41 +0000 Subject: [PATCH 01/12] =?UTF-8?q?Merged=20PR=20780:=20=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E5=A4=89=E6=8F=9B=E3=83=84=E3=83=BC=E3=83=AB=EF=BC=88?= =?UTF-8?q?=E6=B1=9A=E3=81=84=E3=83=87=E3=83=BC=E3=82=BF=E5=AF=BE=E5=BF=9C?= =?UTF-8?q?=E7=89=88=EF=BC=89=E3=81=AE=E4=BD=9C=E6=88=90=EF=BC=8B=E5=8B=95?= =?UTF-8?q?=E4=BD=9C=E7=A2=BA=E8=AA=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3776: データ変換ツール（汚いデータ対応版）の作成](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3776) 綺麗なデータ対応版のレビュー指摘も合わせて修正。 一旦OMDS様よりいただいた1万件～のデータを処理できることは確認済みです。 実装コストとバグの入れ込みを懸念し、有効期限が"9999/12/31"のデータは最初にデータを積む段階で除外するようにしました。 ## レビューポイント - メールアドレス重複チェックについて、想定通りの重複対象を検索出来ているか。 - step3の1.アカウントとユーザが同じ場合 adminMainとuserEmailが重複していた場合に、重複していたユーザーは削除し、アカウントのみを残す（accountユーザーのroleとauthorIdは削除したuserに設定されていたものとする）処理は妥当か。 →accountのIFにroleとauthorIdを追加し、register側のcreateAccountで登録するようにしています。 ## 動作確認状況 - ローカルで確認（Account_transition_2024.1.19.csvで実施） ４つのJSONファイルができていることを確認。 Countryの場合の付け替えができていることを確認。 adminMainとemailが重複している場合の重複削除ができていることを確認。 ## 補足 - 登録ツールと共通のパラメータで動作するようにしました。 例） POST: localhost:8280/transfer Body: { "inputFilePath": "./data/" } 　変換ツールの使い方としてはAccount_transition.jsonというファイルを見るようにしています。 　 --- .../server/src/common/types/types.ts | 97 ++-- .../server/src/constants/index.ts | 2 +- .../src/features/accounts/accounts.service.ts | 2 + .../features/register/register.controller.ts | 109 ++-- .../src/features/register/register.service.ts | 21 +- .../features/transfer/transfer.controller.ts | 84 ++- .../src/features/transfer/transfer.service.ts | 538 ++++++++++++------ .../src/features/transfer/types/types.ts | 26 +- .../accounts/accounts.repository.service.ts | 3 + .../licenses/licenses.repository.service.ts | 62 +- .../worktypes/worktypes.repository.service.ts | 10 +- 11 files changed, 601 insertions(+), 353 deletions(-) diff --git a/data_migration_tools/server/src/common/types/types.ts b/data_migration_tools/server/src/common/types/types.ts index de92211..69ef466 100644 --- a/data_migration_tools/server/src/common/types/types.ts +++ b/data_migration_tools/server/src/common/types/types.ts @@ -8,8 +8,8 @@ export class csvInputFile { last_name: string; country: string; state: string; - start_date: Date; - expired_date: Date; + start_date: string; + expired_date: string; user_email: string; author_id: string; recording_mode: string; @@ -34,7 +34,7 @@ export class csvInputFile { wt19: string; wt20: string; } -export class AccountsOutputFileStep1 { +export class AccountsFileType { accountId: number; type: string; companyName: string; @@ -43,9 +43,11 @@ export class AccountsOutputFileStep1 { adminName: string; adminMail: string; userId: number; + role: string; + authorId: string; } -export class AccountsOutputFile { +export class AccountsFile { accountId: number; type: number; companyName: string; @@ -54,18 +56,11 @@ export class AccountsOutputFile { adminName: string; adminMail: string; userId: number; + role: string; + authorId: string; } -export class AccountsInputFile { - accountId: number; - type: number; - companyName: string; - country: string; - dealerAccountId?: number; - adminName: string; - adminMail: string; - userId: number; -} -export class UsersOutputFile { + +export class UsersFile { accountId: number; userId: number; name: string; @@ -74,23 +69,7 @@ export class UsersOutputFile { email: string; } -export class UsersInputFile { - accountId: number; - userId: number; - name: string; - role: string; - authorId: string; - email: string; -} - -export class LicensesOutputFile { - expiry_date: string; - account_id: number; - type: string; - status: string; - allocated_user_id?: number; -} -export class LicensesInputFile { +export class LicensesFile { expiry_date: string; account_id: number; type: string; @@ -98,16 +77,12 @@ export class LicensesInputFile { allocated_user_id?: number; } -export class WorktypesOutputFile { - account_id: number; - custom_worktype_id: string; -} -export class WorktypesInputFile { +export class WorktypesFile { account_id: number; custom_worktype_id: string; } -export class CardLicensesInputFile { +export class CardLicensesFile { license_id: number; issue_id: number; card_license_key: string; @@ -118,10 +93,10 @@ export class CardLicensesInputFile { updated_by?: string; } -export function isAccountsInputFileArray(obj: any): obj is AccountsInputFile[] { - return Array.isArray(obj) && obj.every((item) => isAccountsInputFile(item)); +export function isAccountsFileArray(obj: any): obj is AccountsFile[] { + return Array.isArray(obj) && obj.every((item) => isAccountsFile(item)); } -export function isAccountsInputFile(obj: any): obj is AccountsInputFile { +export function isAccountsFile(obj: any): obj is AccountsFile { return ( typeof obj === "object" && obj !== null && @@ -141,14 +116,20 @@ export function isAccountsInputFile(obj: any): obj is AccountsInputFile { "adminMail" in obj && typeof obj.adminMail === "string" && "userId" in obj && - typeof obj.userId === "number" + typeof obj.userId === "number" && + ("role" in obj + ? obj.role === null || typeof obj.role === "string" + : true) && + ("authorId" in obj + ? obj.authorId === null || typeof obj.authorId === "string" + : true) ); } -export function isUsersInputFileArray(obj: any): obj is UsersInputFile[] { - return Array.isArray(obj) && obj.every((item) => isUsersInputFile(item)); +export function isUsersFileArray(obj: any): obj is UsersFile[] { + return Array.isArray(obj) && obj.every((item) => isUsersFile(item)); } -export function isUsersInputFile(obj: any): obj is UsersInputFile { +export function isUsersFile(obj: any): obj is UsersFile { return ( typeof obj === "object" && obj !== null && @@ -167,10 +148,10 @@ export function isUsersInputFile(obj: any): obj is UsersInputFile { ); } -export function isLicensesInputFileArray(obj: any): obj is LicensesInputFile[] { - return Array.isArray(obj) && obj.every((item) => isLicensesInputFile(item)); +export function isLicensesFileArray(obj: any): obj is LicensesFile[] { + return Array.isArray(obj) && obj.every((item) => isLicensesFile(item)); } -export function isLicensesInputFile(obj: any): obj is LicensesInputFile { +export function isLicensesFile(obj: any): obj is LicensesFile { return ( typeof obj === "object" && obj !== null && @@ -187,12 +168,10 @@ export function isLicensesInputFile(obj: any): obj is LicensesInputFile { ); } -export function isWorktypesInputFileArray( - obj: any -): obj is WorktypesInputFile[] { - return Array.isArray(obj) && obj.every((item) => isWorktypesInputFile(item)); +export function isWorktypesFileArray(obj: any): obj is WorktypesFile[] { + return Array.isArray(obj) && obj.every((item) => isWorktypesFile(item)); } -export function isWorktypesInputFile(obj: any): obj is WorktypesInputFile { +export function isWorktypesFile(obj: any): obj is WorktypesFile { return ( typeof obj === "object" && obj !== null && @@ -203,16 +182,10 @@ export function isWorktypesInputFile(obj: any): obj is WorktypesInputFile { ); } -export function isCardLicensesInputFileArray( - obj: any -): obj is CardLicensesInputFile[] { - return ( - Array.isArray(obj) && obj.every((item) => isCardLicensesInputFile(item)) - ); +export function isCardLicensesFileArray(obj: any): obj is CardLicensesFile[] { + return Array.isArray(obj) && obj.every((item) => isCardLicensesFile(item)); } -export function isCardLicensesInputFile( - obj: any -): obj is CardLicensesInputFile { +export function isCardLicensesFile(obj: any): obj is CardLicensesFile { return ( typeof obj === "object" && obj !== null && diff --git a/data_migration_tools/server/src/constants/index.ts b/data_migration_tools/server/src/constants/index.ts index 71bd022..74e6e3a 100644 --- a/data_migration_tools/server/src/constants/index.ts +++ b/data_migration_tools/server/src/constants/index.ts @@ -351,7 +351,7 @@ export const COUNTRY_LIST = [ { value: "BG", label: "Bulgaria" }, { value: "HR", label: "Croatia" }, { value: "CY", label: "Cyprus" }, - { value: "CZ", label: "Czech Republic" }, + { value: "CZ", label: "Czech" }, { value: "DK", label: "Denmark" }, { value: "EE", label: "Estonia" }, { value: "FI", label: "Finland" }, diff --git a/data_migration_tools/server/src/features/accounts/accounts.service.ts b/data_migration_tools/server/src/features/accounts/accounts.service.ts index 4a7dd65..2954cbd 100644 --- a/data_migration_tools/server/src/features/accounts/accounts.service.ts +++ b/data_migration_tools/server/src/features/accounts/accounts.service.ts @@ -37,6 +37,7 @@ export class AccountsService { password: string, username: string, role: string, + authorId: string, acceptedEulaVersion: string, acceptedPrivacyNoticeVersion: string, acceptedDpaVersion: string, @@ -103,6 +104,7 @@ export class AccountsService { type, externalUser.sub, role, + authorId, accountId, userId, acceptedEulaVersion, diff --git a/data_migration_tools/server/src/features/register/register.controller.ts b/data_migration_tools/server/src/features/register/register.controller.ts index c55d548..cb36d2a 100644 --- a/data_migration_tools/server/src/features/register/register.controller.ts +++ b/data_migration_tools/server/src/features/register/register.controller.ts @@ -17,11 +17,11 @@ import { AccountsService } from "../accounts/accounts.service"; import { UsersService } from "../users/users.service"; import { makeContext } from "../../common/log"; import { - isAccountsInputFileArray, - isUsersInputFileArray, - isLicensesInputFileArray, - isWorktypesInputFileArray, - isCardLicensesInputFileArray, + isAccountsFileArray, + isUsersFileArray, + isLicensesFileArray, + isWorktypesFileArray, + isCardLicensesFileArray, } from "../../common/types/types"; import { makePassword } from "../../common/password/password"; import { @@ -73,13 +73,24 @@ export class RegisterController { const cardLicensesFileFullPath = inputFilePath + "cardLicenses.json"; // ファイル存在チェックと読み込み - if ( - !fs.existsSync(accouncsFileFullPath) || - !fs.existsSync(usersFileFullPath) || - !fs.existsSync(licensesFileFullPath) || - !fs.existsSync(worktypesFileFullPath) || - !fs.existsSync(cardLicensesFileFullPath) - ) { + // どのファイルがないのかわからないのでそれぞれに存在しない場合はエラーを出す + if (!fs.existsSync(accouncsFileFullPath)) { + this.logger.error(`file not exists from ${inputFilePath}`); + throw new Error(`file not exists from ${inputFilePath}`); + } + if (!fs.existsSync(usersFileFullPath)) { + this.logger.error(`file not exists from ${inputFilePath}`); + throw new Error(`file not exists from ${inputFilePath}`); + } + if (!fs.existsSync(licensesFileFullPath)) { + this.logger.error(`file not exists from ${inputFilePath}`); + throw new Error(`file not exists from ${inputFilePath}`); + } + if (!fs.existsSync(worktypesFileFullPath)) { + this.logger.error(`file not exists from ${inputFilePath}`); + throw new Error(`file not exists from ${inputFilePath}`); + } + if (!fs.existsSync(cardLicensesFileFullPath)) { this.logger.error(`file not exists from ${inputFilePath}`); throw new Error(`file not exists from ${inputFilePath}`); } @@ -90,34 +101,50 @@ export class RegisterController { ); // 型ガード（account） - if (!isAccountsInputFileArray(accountsObject)) { - throw new Error("input file is not accountsInputFiles"); + if (!isAccountsFileArray(accountsObject)) { + throw new Error("input file is not AccountsFiles"); } - for (const accountsInputFile of accountsObject) { + for (const AccountsFile of accountsObject) { // ランダムなパスワードを生成する const ramdomPassword = makePassword(); + // roleの設定 + // roleの値がnullなら"none"、null以外ならroleの値、 + // また、roleの値が"author"なら"author"を設定 + let role: string; + let authorId: string; + if (AccountsFile.role === null) { + role = USER_ROLES.NONE; + authorId = null; + } else if (AccountsFile.role === USER_ROLES.AUTHOR) { + role = USER_ROLES.AUTHOR; + authorId = AccountsFile.authorId; + } else { + // ありえないが、roleの値が"none"または"author"の文字列以外の場合はエラーを返す + throw new Error("Invalid role value"); + } await this.accountsService.createAccount( context, - accountsInputFile.companyName, - accountsInputFile.country, - accountsInputFile.dealerAccountId, - accountsInputFile.adminMail, + AccountsFile.companyName, + AccountsFile.country, + AccountsFile.dealerAccountId, + AccountsFile.adminMail, ramdomPassword, - accountsInputFile.adminName, - "none", + AccountsFile.adminName, + role, + authorId, null, null, null, - accountsInputFile.type, - accountsInputFile.accountId, - accountsInputFile.userId + AccountsFile.type, + AccountsFile.accountId, + AccountsFile.userId ); // ratelimit対応のためsleepを行う await sleep(MIGRATION_DATA_REGISTER_INTERVAL_MILLISEC); } - // const accountsInputFiles = accountsObject as AccountsInputFile[]; + // const AccountsFiles = accountsObject as AccountsFile[]; // ユーザの登録用ファイル読み込み const usersObject = JSON.parse( @@ -125,24 +152,24 @@ export class RegisterController { ); // 型ガード（user） - if (!isUsersInputFileArray(usersObject)) { - throw new Error("input file is not usersInputFiles"); + if (!isUsersFileArray(usersObject)) { + throw new Error("input file is not UsersFiles"); } - for (const usersInputFile of usersObject) { - this.logger.log(usersInputFile.name); + for (const UsersFile of usersObject) { + this.logger.log(UsersFile.name); await this.usersService.createUser( context, - usersInputFile.name, - usersInputFile.role === USER_ROLES.AUTHOR + UsersFile.name, + UsersFile.role === USER_ROLES.AUTHOR ? USER_ROLES.AUTHOR : USER_ROLES.NONE, - usersInputFile.email, + UsersFile.email, true, true, - usersInputFile.accountId, - usersInputFile.userId, - usersInputFile.authorId, + UsersFile.accountId, + UsersFile.userId, + UsersFile.authorId, false, null, true @@ -157,8 +184,8 @@ export class RegisterController { ); // 型ガード（license） - if (!isLicensesInputFileArray(licensesObject)) { - throw new Error("input file is not licensesInputFiles"); + if (!isLicensesFileArray(licensesObject)) { + throw new Error("input file is not LicensesFiles"); } // ワークタイプの登録用ファイル読み込み @@ -167,8 +194,8 @@ export class RegisterController { ); // 型ガード（Worktypes） - if (!isWorktypesInputFileArray(worktypesObject)) { - throw new Error("input file is not WorktypesInputFiles"); + if (!isWorktypesFileArray(worktypesObject)) { + throw new Error("input file is not WorktypesFiles"); } // カードライセンスの登録用ファイル読み込み @@ -177,8 +204,8 @@ export class RegisterController { ); // 型ガード（cardLicenses） - if (!isCardLicensesInputFileArray(cardLicensesObject)) { - throw new Error("input file is not cardLicensesInputFiles"); + if (!isCardLicensesFileArray(cardLicensesObject)) { + throw new Error("input file is not cardLicensesFiles"); } // ライセンス・ワークタイプ・カードライセンスの登録 diff --git a/data_migration_tools/server/src/features/register/register.service.ts b/data_migration_tools/server/src/features/register/register.service.ts index 6e42dc1..9af654f 100644 --- a/data_migration_tools/server/src/features/register/register.service.ts +++ b/data_migration_tools/server/src/features/register/register.service.ts @@ -1,9 +1,9 @@ import { HttpException, HttpStatus, Injectable, Logger } from "@nestjs/common"; import { Context } from "../../common/log"; import { - LicensesInputFile, - WorktypesInputFile, - CardLicensesInputFile, + LicensesFile, + WorktypesFile, + CardLicensesFile, } from "../../common/types/types"; import { LicensesRepositoryService } from "../../repositories/licenses/licenses.repository.service"; import { WorktypesRepositoryService } from "../../repositories/worktypes/worktypes.repository.service"; @@ -22,9 +22,9 @@ export class RegisterService { */ async registLicenseAndWorktypeData( context: Context, - licensesInputFiles: LicensesInputFile[], - worktypesInputFiles: WorktypesInputFile[], - cardlicensesInputFiles: CardLicensesInputFile[] + LicensesFiles: LicensesFile[], + WorktypesFiles: WorktypesFile[], + cardLicensesFiles: CardLicensesFile[] ): Promise { // パラメータ内容が長大なのでログには出さない this.logger.log( @@ -35,20 +35,17 @@ export class RegisterService { try { this.logger.log("Licenses register start"); - await this.licensesRepository.insertLicenses(context, licensesInputFiles); + await this.licensesRepository.insertLicenses(context, LicensesFiles); this.logger.log("Licenses register end"); this.logger.log("Worktypes register start"); - await this.worktypesRepository.createWorktype( - context, - worktypesInputFiles - ); + await this.worktypesRepository.createWorktype(context, WorktypesFiles); this.logger.log("Worktypes register end"); this.logger.log("CardLicenses register start"); await this.licensesRepository.insertCardLicenses( context, - cardlicensesInputFiles + cardLicensesFiles ); this.logger.log("CardLicenses register end"); } catch (e) { diff --git a/data_migration_tools/server/src/features/transfer/transfer.controller.ts b/data_migration_tools/server/src/features/transfer/transfer.controller.ts index 8725a4c..7916f27 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.controller.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.controller.ts @@ -15,16 +15,7 @@ import { TransferService } from "./transfer.service"; import { makeContext } from "../../common/log"; import { csvInputFile } from "../../common/types/types"; import { makeErrorResponse } from "src/common/errors/makeErrorResponse"; -import { - COUNTRY_LIST, - MIGRATION_TYPE, - TIERS, - WORKTYPE_MAX_COUNT, - RECORDING_MODE, - LICENSE_ALLOCATED_STATUS, - USER_ROLES, - AUTO_INCREMENT_START, -} from "../../constants"; +import { AUTO_INCREMENT_START } from "../../constants"; @ApiTags("transfer") @Controller("transfer") export class TransferController { @@ -57,16 +48,16 @@ export class TransferController { ); try { // 読み込みファイルのフルパス - const csvFileFullPath = inputFilePath + ".csv"; + const accouncsFileFullPath = inputFilePath + "Account_transition.csv"; // ファイル存在チェックと読み込み - if (!fs.existsSync(csvFileFullPath)) { + if (!fs.existsSync(accouncsFileFullPath)) { this.logger.error(`file not exists from ${inputFilePath}`); throw new Error(`file not exists from ${inputFilePath}`); } // CSVファイルを全行読み込む - const inputFile = fs.readFileSync(csvFileFullPath, "utf-8"); + const inputFile = fs.readFileSync(accouncsFileFullPath, "utf-8"); // レコードごとに分割 const csvInputFileLines = inputFile.split("\n"); @@ -77,11 +68,34 @@ export class TransferController { // 項目ごとに切り分ける let csvInputFile: csvInputFile[] = []; csvInputFileLines.forEach((line) => { + // 項目にカンマが入っている場合を考慮して、ダブルクォーテーションで囲まれた部分を一つの項目として扱う + const regExp = /"[^"]*"/g; + const matchList = line.match(regExp); + if (matchList) { + matchList.forEach((match) => { + const replaced = match.replace(/,/g, " "); + line = line.replace(match, replaced); + }); + } const data = line.split(","); - // ダブルクォーテーションは削除 + // ダブルクォーテーションを削除 data.forEach((value, index) => { data[index] = value.replace(/"/g, ""); }); + // "\r"を削除 + data[data.length - 1] = data[data.length - 1].replace(/\r/g, ""); + // dataの要素数が34(csvInputFileの要素数)より多い場合、フォーマット不一致エラー(移行元はworktypeの数が20より多く設定できるので理論上は存在する) + // worktypeの数の確認を促すエラーを出す + if (data.length > 34) { + this.logger.error( + `[${context.getTrackingId()}] format error.please check the number of elements in worktype. data=${data}` + ); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.BAD_REQUEST + ); + } + csvInputFile.push({ type: data[0], account_id: data[1], @@ -92,8 +106,8 @@ export class TransferController { last_name: data[6], country: data[7], state: data[8], - start_date: new Date(data[9]), - expired_date: new Date(data[10]), + start_date: data[9], + expired_date: data[10], user_email: data[11], author_id: data[12], recording_mode: data[13], @@ -119,6 +133,10 @@ export class TransferController { wt20: data[33], }); }); + // 最後の行がundefinedの場合はその行を削除 + if (csvInputFile[csvInputFile.length - 1].account_id === undefined) { + csvInputFile.pop(); + } // 各データのバリデーションチェック await this.transferService.validateInputData(context, csvInputFile); @@ -132,35 +150,39 @@ export class TransferController { accountIdMap.set(accountId, index + AUTO_INCREMENT_START); }); // CSVファイルの変換 - const transferResponse = await this.transferService.registInputData( + const transferResponseCsv = await this.transferService.transferInputData( context, csvInputFile, accountIdMap ); // countryを除いた階層の再配置 - const accountsOutputFileStep1Lines = - transferResponse.accountsOutputFileStep1Lines; - const accountsOutputFile = await this.transferService.relocateHierarchy( + const AccountsFileTypeLines = transferResponseCsv.accountsFileTypeLines; + const AccountsFile = await this.transferService.relocateHierarchy( context, - accountsOutputFileStep1Lines + AccountsFileTypeLines ); + const UsersFile = transferResponseCsv.usersFileLines; + const LicensesFile = transferResponseCsv.licensesFileLines; // メールアドレスの重複を削除 - // デモライセンスの削除 - // いったんこのままコミットしてテストを実施する + const resultDuplicateEmail = + await this.transferService.removeDuplicateEmail( + context, + AccountsFile, + UsersFile, + LicensesFile + ); - // transferResponseを４つのJSONファイルの出力する(出力先はinputと同じにする) + // transferResponseCsvを４つのJSONファイルの出力する(出力先はinputと同じにする) const outputFilePath = body.inputFilePath; - const usersOutputFile = transferResponse.usersOutputFileLines; - const licensesOutputFile = transferResponse.licensesOutputFileLines; - const worktypesOutputFile = transferResponse.worktypesOutputFileLines; + const WorktypesFile = transferResponseCsv.worktypesFileLines; this.transferService.outputJsonFile( context, outputFilePath, - accountsOutputFile, - usersOutputFile, - licensesOutputFile, - worktypesOutputFile + resultDuplicateEmail.accountsFileLines, + resultDuplicateEmail.usersFileLines, + resultDuplicateEmail.licensesFileLines, + WorktypesFile ); return {}; } catch (e) { diff --git a/data_migration_tools/server/src/features/transfer/transfer.service.ts b/data_migration_tools/server/src/features/transfer/transfer.service.ts index 31b75ed..c1b0757 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.service.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.service.ts @@ -1,12 +1,12 @@ import { HttpException, HttpStatus, Injectable, Logger } from "@nestjs/common"; import { Context } from "../../common/log"; import { - AccountsOutputFileStep1, - UsersOutputFile, - LicensesOutputFile, - WorktypesOutputFile, + AccountsFileType, + UsersFile, + LicensesFile, + WorktypesFile, csvInputFile, - AccountsOutputFile, + AccountsFile, } from "../../common/types/types"; import { COUNTRY_LIST, @@ -18,8 +18,12 @@ import { USER_ROLES, SWITCH_FROM_TYPE, } from "src/constants"; -import { registInputDataResponse } from "./types/types"; +import { + registInputDataResponse, + removeDuplicateEmailResponse, +} from "./types/types"; import fs from "fs"; +import { makeErrorResponse } from "src/common/error/makeErrorResponse"; @Injectable() export class TransferService { @@ -27,33 +31,33 @@ export class TransferService { private readonly logger = new Logger(TransferService.name); /** - * Regist Data + * Transfer Input Data * @param OutputFilePath: string * @param csvInputFile: csvInputFile[] */ - async registInputData( + async transferInputData( context: Context, csvInputFile: csvInputFile[], accountIdMap: Map ): Promise { // パラメータ内容が長大なのでログには出さない this.logger.log( - `[IN] [${context.getTrackingId()}] ${this.registInputData.name}` + `[IN] [${context.getTrackingId()}] ${this.transferInputData.name}` ); try { - let accountsOutputFileStep1Lines: AccountsOutputFileStep1[] = []; - let usersOutputFileLines: UsersOutputFile[] = []; - let licensesOutputFileLines: LicensesOutputFile[] = []; - let worktypesOutputFileLines: WorktypesOutputFile[] = []; + let accountsFileTypeLines: AccountsFileType[] = []; + let usersFileLines: UsersFile[] = []; + let licensesFileLines: LicensesFile[] = []; + let worktypesFileLines: WorktypesFile[] = []; let userIdIndex = 0; + // authorIdとuserIdの対応関係を保持するMapを定義 + const authorIdToUserIdMap: Map = new Map(); // csvInputFileを一行読み込みする csvInputFile.forEach((line) => { // typeが"USER"以外の場合、アカウントデータの作成を行う if (line.type !== MIGRATION_TYPE.USER) { - // userIdのインクリメント - userIdIndex = userIdIndex + 1; // line.countryの値を読み込みCOUNTRY_LISTのlabelからvalueに変換する const country = COUNTRY_LIST.find( (country) => country.label === line.country @@ -71,8 +75,16 @@ export class TransferService { if (line.parent_id) { parentAccountId = accountIdMap.get(line.parent_id); } - // AccountsOutputFile配列にPush - accountsOutputFileStep1Lines.push({ + // 万が一parent_idが入力されているのに存在しなかった場合は、nullを設定する。 + if (parentAccountId === undefined) { + parentAccountId = null; + } + + // userIdIndexをインクリメントする + userIdIndex++; + + // AccountsFile配列にPush + accountsFileTypeLines.push({ // accountIdはaccountIdMapから取得する accountId: accountIdMap.get(line.account_id), type: line.type, @@ -82,150 +94,176 @@ export class TransferService { adminName: adminName, adminMail: line.email, userId: userIdIndex, + role: null, + authorId: null, }); } else { - // typeが"USER"の場合、ユーザデータの作成を行う - // userIdのインクリメント - userIdIndex = userIdIndex + 1; - // nameの変換 - // もしline.last_nameとline.first_nameが存在しない場合、line.emailをnameにする - // 存在する場合は、last_name + " " + first_name - let name = line.email; - if (line.last_name && line.first_name) { - name = `${line.last_name} ${line.first_name}`; - } - // roleの変換 - // authorIdが設定されてる場合はauthor、されていない場合は移行しないので次の行に進む - if (line.author_id) { - usersOutputFileLines.push({ - accountId: accountIdMap.get(line.account_id), - userId: userIdIndex, - name: name, - role: USER_ROLES.AUTHOR, - authorId: line.author_id, - email: line.user_email, - }); - } else { - return; - } - // ライセンスのデータの作成を行う - // authorIdが設定されてる場合、statusは"allocated"、allocated_user_idは対象のユーザID - // されていない場合、statusは"reusable"、allocated_user_idはnull - licensesOutputFileLines.push({ - expiry_date: line.expired_date.toISOString(), - account_id: accountIdMap.get(line.account_id), - type: SWITCH_FROM_TYPE.NONE, - status: line.author_id - ? LICENSE_ALLOCATED_STATUS.ALLOCATED - : LICENSE_ALLOCATED_STATUS.REUSABLE, - allocated_user_id: line.author_id ? userIdIndex : null, - }); - // WorktypesOutputFileの作成 - // wt1~wt20まで読み込み、account単位で作成する - // 作成したWorktypesOutputFileを配列にPush - for (let i = 1; i <= WORKTYPE_MAX_COUNT; i++) { - const wt = `wt${i}`; - if (line[wt]) { - // 既に存在する場合は、作成しない - if ( - worktypesOutputFileLines.find( - (worktype) => - worktype.account_id === accountIdMap.get(line.account_id) && - worktype.custom_worktype_id === line[wt].toString() - ) - ) { - continue; + // typeが"USER"の場合 + if (line.type == MIGRATION_TYPE.USER) { + // line.author_idが存在する場合のみユーザーデータを作成する + if (line.author_id) { + // userIdIndexをインクリメントする + userIdIndex++; + + // nameの変換 + // もしline.last_nameとline.first_nameが存在しない場合、line.emailをnameにする + // 存在する場合は、last_name + " " + first_name + let name = line.user_email; + if (line.last_name && line.first_name) { + name = `${line.last_name} ${line.first_name}`; + } + // UsersFileの作成 + usersFileLines.push({ + accountId: accountIdMap.get(line.account_id), + userId: userIdIndex, + name: name, + role: USER_ROLES.AUTHOR, + authorId: line.author_id, + email: line.user_email, + }); + // authorIdとuserIdの対応関係をマッピング + authorIdToUserIdMap.set(line.author_id, userIdIndex); + } + + // ライセンスのデータの作成を行う + // line.expired_dateが9999/12/31 23:59:59.997のデータの場合はデモライセンスなので登録しない + if (line.expired_date !== "9999/12/31 23:59:59.997") { + // authorIdが設定されてる場合、statusは"allocated"、allocated_user_idは対象のユーザID + // されていない場合、statusは"reusable"、allocated_user_idはnull + let status: string; + let allocated_user_id: number | null; + if (line.author_id) { + status = LICENSE_ALLOCATED_STATUS.ALLOCATED; + allocated_user_id = + authorIdToUserIdMap.get(line.author_id) ?? null; // authorIdに対応するuserIdを取得 + } else { + status = LICENSE_ALLOCATED_STATUS.REUSABLE; + allocated_user_id = null; + } + // LicensesFileの作成 + licensesFileLines.push({ + expiry_date: line.expired_date, + account_id: accountIdMap.get(line.account_id), + type: SWITCH_FROM_TYPE.NONE, + status: status, + allocated_user_id: allocated_user_id, + }); + } + // WorktypesFileの作成 + // wt1~wt20まで読み込み、account単位で作成する + // 作成したWorktypesFileを配列にPush + for (let i = 1; i <= WORKTYPE_MAX_COUNT; i++) { + const wt = `wt${i}`; + if (line[wt]) { + // account_idで同一のcustom_worktype_idが存在しない場合は、作成する + if ( + !worktypesFileLines.find( + (worktype) => + worktype.account_id === + accountIdMap.get(line.account_id) && + worktype.custom_worktype_id === line[wt] + ) + ) { + worktypesFileLines.push({ + account_id: accountIdMap.get(line.account_id), + custom_worktype_id: line[wt], + }); + } else { + continue; + } } } } } - // つぎの行に進む }); return { - accountsOutputFileStep1Lines, - usersOutputFileLines, - licensesOutputFileLines, - worktypesOutputFileLines, + accountsFileTypeLines, + usersFileLines, + licensesFileLines, + worktypesFileLines, }; } catch (e) { this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); } finally { this.logger.log( - `[OUT] [${context.getTrackingId()}] ${this.registInputData.name}` + `[OUT] [${context.getTrackingId()}] ${this.transferInputData.name}` ); } } /** * 階層の付け替えを行う - * @param accountsOutputFileStep1: AccountsOutputFileStep1[] - * @returns AccountsOutputFile[] + * @param accountsFileType: AccountsFileType[] + * @returns AccountsFile[] */ async relocateHierarchy( context: Context, - accountsOutputFileStep1: AccountsOutputFileStep1[] - ): Promise { + accountsFileType: AccountsFileType[] + ): Promise { // パラメータ内容が長大なのでログには出さない this.logger.log( `[IN] [${context.getTrackingId()}] ${this.relocateHierarchy.name}` ); try { - // dealerAccountIdを検索し、typeがCountryの場合 - accountsOutputFileStep1.forEach((account) => { - if (account.type === MIGRATION_TYPE.COUNTRY) { - console.log(account); - // そのacccountIdをdealerAccountIdにもつアカウント(Distributor)を検索する - const distributor = accountsOutputFileStep1.find( - (distributor) => distributor.accountId === account.dealerAccountId - ); - console.log(distributor); - if (distributor) { - // DistributorのdealerAccountIdをBC（Countryの親）に付け替える - distributor.dealerAccountId = account.dealerAccountId; - } - } - }); - // typeがCountryのアカウントを取り除く - accountsOutputFileStep1 = accountsOutputFileStep1.filter( - (account) => account.type !== MIGRATION_TYPE.COUNTRY - ); + const relocatedAccounts: AccountsFile[] = []; + const countryRecords: Map = new Map(); - // typeをtierに変換し、AccountsOutputFileに変換する - let accountsOutputFile: AccountsOutputFile[] = []; - accountsOutputFileStep1.forEach((account) => { - let tier = 0; - switch (account.type) { - case MIGRATION_TYPE.ADMINISTRATOR: - tier = TIERS.TIER1; - break; - case MIGRATION_TYPE.BC: - tier = TIERS.TIER2; - break; - case MIGRATION_TYPE.DISTRIBUTOR: - tier = TIERS.TIER3; - break; - case MIGRATION_TYPE.DEALER: - tier = TIERS.TIER4; - break; - case MIGRATION_TYPE.CUSTOMER: - tier = TIERS.TIER5; - break; + // accountsFileTypeをループ + accountsFileType.forEach((account) => { + // Countryの場合はDistributorのアカウントIDと新たな親アカウントID（BC）の組み合わせをMapに登録 + if (account.type === MIGRATION_TYPE.COUNTRY) { + // 配下のDistributorアカウントを取得 + const distributor = accountsFileType.find( + (distributor) => + distributor.dealerAccountId === account.accountId && + distributor.type === MIGRATION_TYPE.DISTRIBUTOR + ); + if (distributor) { + countryRecords.set(distributor.accountId, account.dealerAccountId); + } + } else { + // Country以外のアカウントの場合は、そのまま登録 + countryRecords.set(account.accountId, account.dealerAccountId); } - accountsOutputFile.push({ - accountId: account.accountId, - type: tier, - companyName: account.companyName, - country: account.country, - dealerAccountId: account.dealerAccountId, - adminName: account.adminName, - adminMail: account.adminMail, - userId: account.userId, - }); }); - return accountsOutputFile; + + // AccountsFileTypeのループを行い、階層情報の置換と新たな配列へのpushを行う + accountsFileType.forEach((account) => { + //　Countryのレコードは除外する + if (account.type !== MIGRATION_TYPE.COUNTRY) { + const dealerAccountId = + countryRecords.get(account.dealerAccountId) ?? + account.dealerAccountId; + const type = this.getAccountType(account.type); + const newAccount: AccountsFile = { + accountId: account.accountId, + type: type, + companyName: account.companyName, + country: account.country, + dealerAccountId: dealerAccountId, + adminName: account.adminName, + adminMail: account.adminMail, + userId: account.userId, + role: account.role, + authorId: account.authorId, + }; + + relocatedAccounts.push(newAccount); + } + }); + + return relocatedAccounts; } catch (e) { this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); } finally { this.logger.log( `[OUT] [${context.getTrackingId()}] ${this.relocateHierarchy.name}` @@ -233,21 +271,38 @@ export class TransferService { } } + // メソッド: アカウントタイプを数値に変換するヘルパー関数 + private getAccountType(type: string): number { + switch (type) { + case MIGRATION_TYPE.ADMINISTRATOR: + return TIERS.TIER1; + case MIGRATION_TYPE.BC: + return TIERS.TIER2; + case MIGRATION_TYPE.DISTRIBUTOR: + return TIERS.TIER3; + case MIGRATION_TYPE.DEALER: + return TIERS.TIER4; + case MIGRATION_TYPE.CUSTOMER: + return TIERS.TIER5; + default: + return 0; + } + } /** * JSONファイルの出力 * @param outputFilePath: string - * @param accountsOutputFile: AccountsOutputFile[] - * @param usersOutputFile: UsersOutputFile[] - * @param licensesOutputFile: LicensesOutputFile[] - * @param worktypesOutputFile: WorktypesOutputFile[] + * @param accountsFile: AccountsFile[] + * @param usersFile: UsersFile[] + * @param licensesFile: LicensesFile[] + * @param worktypesFile: WorktypesFile[] */ async outputJsonFile( context: Context, outputFilePath: string, - accountsOutputFile: AccountsOutputFile[], - usersOutputFile: UsersOutputFile[], - licensesOutputFile: LicensesOutputFile[], - worktypesOutputFile: WorktypesOutputFile[] + accountsFile: AccountsFile[], + usersFile: UsersFile[], + licensesFile: LicensesFile[], + worktypesFile: WorktypesFile[] ): Promise { // パラメータ内容が長大なのでログには出さない this.logger.log( @@ -256,29 +311,24 @@ export class TransferService { try { // JSONファイルの出力を行う - // accountsOutputFile配列の出力 - const accountsOutputFileJson = JSON.stringify(accountsOutputFile); - fs.writeFileSync( - `${outputFilePath}_accounts.json`, - accountsOutputFileJson - ); - // usersOutputFile - const usersOutputFileJson = JSON.stringify(usersOutputFile); - fs.writeFileSync(`${outputFilePath}_users.json`, usersOutputFileJson); - // licensesOutputFile - const licensesOutputFileJson = JSON.stringify(licensesOutputFile); - fs.writeFileSync( - `${outputFilePath}_licenses.json`, - licensesOutputFileJson - ); - // worktypesOutputFile - const worktypesOutputFileJson = JSON.stringify(worktypesOutputFile); - fs.writeFileSync( - `${outputFilePath}_worktypes.json`, - worktypesOutputFileJson - ); + // AccountsFile配列の出力 + const accountsFileJson = JSON.stringify(accountsFile); + fs.writeFileSync(`${outputFilePath}accounts.json`, accountsFileJson); + // UsersFile + const usersFileJson = JSON.stringify(usersFile); + fs.writeFileSync(`${outputFilePath}users.json`, usersFileJson); + // LicensesFile + const licensesFileJson = JSON.stringify(licensesFile); + fs.writeFileSync(`${outputFilePath}licenses.json`, licensesFileJson); + // WorktypesFile + const worktypesFileJson = JSON.stringify(worktypesFile); + fs.writeFileSync(`${outputFilePath}worktypes.json`, worktypesFileJson); } catch (e) { this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); } finally { this.logger.log( `[OUT] [${context.getTrackingId()}] ${this.outputJsonFile.name}` @@ -300,6 +350,8 @@ export class TransferService { ); try { + // アカウントに対するworktypeのMap配列を作成する + const accountWorktypeMap = new Map(); // csvInputFileのバリデーションチェックを行う csvInputFile.forEach((line, index) => { // typeのバリデーションチェック @@ -320,7 +372,6 @@ export class TransferService { // countryのバリデーションチェック if (line.country) { if (!COUNTRY_LIST.find((country) => country.label === line.country)) { - console.log(line.country); throw new HttpException( `country is invalid. index=${index} country=${line.country}`, HttpStatus.BAD_REQUEST @@ -329,7 +380,8 @@ export class TransferService { } // mailのバリデーションチェック // メールアドレスの形式が正しいかどうかのチェック - const mailRegExp = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/; + const mailRegExp = + /^[a-zA-Z0-9!#$%&'_`/=~+\-?^{|}.]+@[a-zA-Z0-9!#$%&'_`/=~+\-?^{|}.]*\.[a-zA-Z0-9!#$%&'_`/=~+\-?^{|}.]*[a-zA-Z]$/; if (line.email) { if (!mailRegExp.test(line.email)) { throw new HttpException( @@ -360,13 +412,181 @@ export class TransferService { ); } } + // worktypeの1アカウント20件上限チェック + for (let i = 1; i <= WORKTYPE_MAX_COUNT; i++) { + const wt = `wt${i}`; + if (line[wt]) { + if (accountWorktypeMap.has(line.account_id)) { + const worktypes = accountWorktypeMap.get(line.account_id); + // 重複している場合はPushしない + if (worktypes?.includes(line[wt])) { + continue; + } else { + worktypes?.push(line[wt]); + } + // 20件を超えたらエラー + if (worktypes?.length > WORKTYPE_MAX_COUNT) { + throw new HttpException( + `worktype is over. index=${index} account_id=${line.account_id}`, + HttpStatus.BAD_REQUEST + ); + } + } else { + accountWorktypeMap.set(line.account_id, [line[wt]]); + } + } + } }); } catch (e) { this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); } finally { this.logger.log( `[OUT] [${context.getTrackingId()}] ${this.validateInputData.name}` ); } } + + /** + * removeDuplicateEmail + * @param accountsFileLines: AccountsFile[] + * @param usersFileLines: UsersFile[] + * @param licensesFileLines: LicensesFile[] + * @returns registInputDataResponse + */ + async removeDuplicateEmail( + context: Context, + accountsFileLines: AccountsFile[], + usersFileLines: UsersFile[], + licensesFileLines: LicensesFile[] + ): Promise { + // パラメータ内容が長大なのでログには出さない + this.logger.log( + `[IN] [${context.getTrackingId()}] ${this.removeDuplicateEmail.name}` + ); + + try { + const newAccountsFileLines: AccountsFile[] = []; + const newUsersFileLines: UsersFile[] = []; + const newLicensesFileLines: LicensesFile[] = [...licensesFileLines]; // licensesFileLinesを新規配列にコピー + + // accountsFileLinesの行ループ + accountsFileLines.forEach((account) => { + const duplicateAdminMail = newAccountsFileLines.find( + (a) => a.adminMail === account.adminMail + ); + + if (duplicateAdminMail) { + // 重複がある場合はどちらが取込対象か判断できないのでファイルを出力し、エラーにする + const errorFileJson = JSON.stringify(account); + fs.writeFileSync(`duplicate_error.json`, errorFileJson); + throw new HttpException( + `adminMail is duplicate. adminMail=${account.adminMail}`, + HttpStatus.BAD_REQUEST + ); + } else { + // 重複がない場合 + newAccountsFileLines.push(account); + } + }); + + // usersFileLinesの行ループ + usersFileLines.forEach((user) => { + const duplicateUserEmail = newUsersFileLines.find( + (u) => u.email === user.email + ); + + if (duplicateUserEmail) { + // 重複がある場合 + const index = newLicensesFileLines.findIndex( + (license) => + license.account_id === user.accountId && + license.allocated_user_id === duplicateUserEmail.userId + ); + if (index !== -1) { + // ライセンスの割り当てを解除 + newLicensesFileLines[index].status = + LICENSE_ALLOCATED_STATUS.REUSABLE; + newLicensesFileLines[index].allocated_user_id = null; + } + } else { + // 重複がない場合 + newUsersFileLines.push(user); + } + // newAccountsFileLinesとの突合せ + const duplicateAdminUserEmail = newAccountsFileLines.find( + (a) => a.adminMail === user.email + ); + // 重複がある場合 + if (duplicateAdminUserEmail) { + // 同一アカウント内での重複の場合 + const isDuplicateInSameAccount = + duplicateAdminUserEmail.accountId === user.accountId; + + if (isDuplicateInSameAccount) { + // アカウント管理者にauthorロールを付与する + duplicateAdminUserEmail.role = USER_ROLES.AUTHOR; + duplicateAdminUserEmail.authorId = user.authorId; + + // アカウントにライセンスが割り当てられているか確認する + const isAllocatedLicense = newLicensesFileLines.some( + (license) => + license.account_id === duplicateAdminUserEmail.accountId && + license.allocated_user_id === duplicateAdminUserEmail.userId + ); + // 割り当てられていなければアカウントに割り当てる + if (!isAllocatedLicense) { + const index = newLicensesFileLines.findIndex( + (license) => + license.account_id === user.accountId && + license.allocated_user_id === user.userId + ); + if (index !== -1) { + newLicensesFileLines[index].allocated_user_id = + duplicateAdminUserEmail.userId; + } + } + } + // ユーザーから割り当て解除する + const index = newLicensesFileLines.findIndex( + (license) => + license.account_id === user.accountId && + license.allocated_user_id === user.userId + ); + if (index !== -1) { + // ライセンスの割り当てを解除 + newLicensesFileLines[index].status = + LICENSE_ALLOCATED_STATUS.REUSABLE; + newLicensesFileLines[index].allocated_user_id = null; + } + // ユーザーの削除 + const userIndex = newUsersFileLines.findIndex( + (u) => u.userId === user.userId + ); + if (userIndex !== -1) { + newUsersFileLines.splice(userIndex, 1); + } + } + }); + + return { + accountsFileLines: newAccountsFileLines, + usersFileLines: newUsersFileLines, + licensesFileLines: newLicensesFileLines, + }; + } catch (e) { + this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); + } finally { + this.logger.log( + `[OUT] [${context.getTrackingId()}] ${this.removeDuplicateEmail.name}` + ); + } + } } diff --git a/data_migration_tools/server/src/features/transfer/types/types.ts b/data_migration_tools/server/src/features/transfer/types/types.ts index 63eb3ab..d4dd969 100644 --- a/data_migration_tools/server/src/features/transfer/types/types.ts +++ b/data_migration_tools/server/src/features/transfer/types/types.ts @@ -1,9 +1,10 @@ import { ApiProperty } from "@nestjs/swagger"; import { - AccountsOutputFileStep1, - LicensesOutputFile, - UsersOutputFile, - WorktypesOutputFile, + AccountsFile, + AccountsFileType, + LicensesFile, + UsersFile, + WorktypesFile, } from "src/common/types/types"; export class transferRequest { @@ -15,11 +16,20 @@ export class transferResponse {} export class registInputDataResponse { @ApiProperty() - accountsOutputFileStep1Lines: AccountsOutputFileStep1[]; + accountsFileTypeLines: AccountsFileType[]; @ApiProperty() - usersOutputFileLines: UsersOutputFile[]; + usersFileLines: UsersFile[]; @ApiProperty() - licensesOutputFileLines: LicensesOutputFile[]; + licensesFileLines: LicensesFile[]; @ApiProperty() - worktypesOutputFileLines: WorktypesOutputFile[]; + worktypesFileLines: WorktypesFile[]; +} + +export class removeDuplicateEmailResponse { + @ApiProperty() + accountsFileLines: AccountsFile[]; + @ApiProperty() + usersFileLines: UsersFile[]; + @ApiProperty() + licensesFileLines: LicensesFile[]; } diff --git a/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts b/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts index b38f6ba..96d5030 100644 --- a/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts +++ b/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts @@ -28,6 +28,7 @@ export class AccountsRepositoryService { * @param tier * @param adminExternalUserId * @param adminUserRole + * @param adminUserAuthId * @param accountId * @param userId * @param adminUserAcceptedEulaVersion @@ -43,6 +44,7 @@ export class AccountsRepositoryService { tier: number, adminExternalUserId: string, adminUserRole: string, + adminUserAuthId: string, accountId: number, userId: number, adminUserAcceptedEulaVersion?: string, @@ -75,6 +77,7 @@ export class AccountsRepositoryService { user.account_id = persistedAccount.id; user.external_id = adminExternalUserId; user.role = adminUserRole; + user.author_id = adminUserAuthId; user.accepted_eula_version = adminUserAcceptedEulaVersion ?? null; user.accepted_privacy_notice_version = adminUserAcceptedPrivacyNoticeVersion ?? null; diff --git a/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts b/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts index 643b7f4..df1c190 100644 --- a/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts +++ b/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts @@ -1,5 +1,5 @@ import { Injectable, Logger } from "@nestjs/common"; -import { DataSource, In } from "typeorm"; +import { DataSource } from "typeorm"; import { License, LicenseAllocationHistory, @@ -8,15 +8,10 @@ import { } from "./entity/license.entity"; import { insertEntity, insertEntities } from "../../common/repository"; import { Context } from "../../common/log"; -import { - LicensesInputFile, - CardLicensesInputFile, -} from "../../common/types/types"; -import {AUTO_INCREMENT_START} from "../../constants/index" -import { - LICENSE_ALLOCATED_STATUS, - LICENSE_TYPE, -} from "../../constants"; + +import { AUTO_INCREMENT_START } from "../../constants/index"; +import { LICENSE_ALLOCATED_STATUS, LICENSE_TYPE } from "../../constants"; +import { CardLicensesFile, LicensesFile } from "src/common/types/types"; @Injectable() export class LicensesRepositoryService { //クエリログにコメントを出力するかどうか @@ -27,27 +22,27 @@ export class LicensesRepositoryService { /** * ライセンスを登録する * @context Context - * @param licensesInputFiles + * @param LicensesFiles */ async insertLicenses( context: Context, - licensesInputFiles: LicensesInputFile[] + LicensesFiles: LicensesFile[] ): Promise<{}> { const nowDate = new Date(); return await this.dataSource.transaction(async (entityManager) => { const licenseRepo = entityManager.getRepository(License); let newLicenses: License[] = []; - licensesInputFiles.forEach((licensesInputFile) => { + LicensesFiles.forEach((LicensesFile) => { const license = new License(); - license.account_id = licensesInputFile.account_id; - license.status = licensesInputFile.status; - license.type = licensesInputFile.type; - license.expiry_date = licensesInputFile.expiry_date - ? new Date(licensesInputFile.expiry_date) + license.account_id = LicensesFile.account_id; + license.status = LicensesFile.status; + license.type = LicensesFile.type; + license.expiry_date = LicensesFile.expiry_date + ? new Date(LicensesFile.expiry_date) : null; - if (licensesInputFile.allocated_user_id) { - license.allocated_user_id = licensesInputFile.allocated_user_id; + if (LicensesFile.allocated_user_id) { + license.allocated_user_id = LicensesFile.allocated_user_id; } newLicenses.push(license); }); @@ -96,11 +91,11 @@ export class LicensesRepositoryService { /** * カードライセンスを登録する * @context Context - * @param cardLicensesInputFiles + * @param cardLicensesFiles */ async insertCardLicenses( context: Context, - cardLicensesInputFiles: CardLicensesInputFile[] + cardLicensesFiles: CardLicensesFile[] ): Promise<{}> { return await this.dataSource.transaction(async (entityManager) => { const cardLicenseRepo = entityManager.getRepository(CardLicense); @@ -110,7 +105,7 @@ export class LicensesRepositoryService { const licenses: License[] = []; // ライセンステーブルを作成する（BULK INSERT) - for (let i = 0; i < cardLicensesInputFiles.length; i++) { + for (let i = 0; i < cardLicensesFiles.length; i++) { const license = new License(); license.account_id = AUTO_INCREMENT_START; // 最初に登場するアカウント（第一アカウント） license.status = LICENSE_ALLOCATED_STATUS.UNALLOCATED; @@ -139,23 +134,22 @@ export class LicensesRepositoryService { const newCardLicenses: CardLicense[] = []; // カードライセンステーブルを作成する（BULK INSERT) - for (let i = 0; i < cardLicensesInputFiles.length; i++) { + for (let i = 0; i < cardLicensesFiles.length; i++) { const cardLicense = new CardLicense(); cardLicense.license_id = savedLicenses[i].id; // Licenseテーブルの自動採番されたIDを挿入 cardLicense.issue_id = savedCardLicensesIssue.id; // CardLicenseIssueテーブルの自動採番されたIDを挿入 - cardLicense.card_license_key = - cardLicensesInputFiles[i].card_license_key; - cardLicense.activated_at = cardLicensesInputFiles[i].activated_at - ? new Date(cardLicensesInputFiles[i].activated_at) + cardLicense.card_license_key = cardLicensesFiles[i].card_license_key; + cardLicense.activated_at = cardLicensesFiles[i].activated_at + ? new Date(cardLicensesFiles[i].activated_at) : null; - cardLicense.created_at = cardLicensesInputFiles[i].created_at - ? new Date(cardLicensesInputFiles[i].created_at) + cardLicense.created_at = cardLicensesFiles[i].created_at + ? new Date(cardLicensesFiles[i].created_at) : null; - cardLicense.created_by = cardLicensesInputFiles[i].created_by; - cardLicense.updated_at = cardLicensesInputFiles[i].updated_at - ? new Date(cardLicensesInputFiles[i].updated_at) + cardLicense.created_by = cardLicensesFiles[i].created_by; + cardLicense.updated_at = cardLicensesFiles[i].updated_at + ? new Date(cardLicensesFiles[i].updated_at) : null; - cardLicense.updated_by = cardLicensesInputFiles[i].updated_by; + cardLicense.updated_by = cardLicensesFiles[i].updated_by; newCardLicenses.push(cardLicense); } diff --git a/data_migration_tools/server/src/repositories/worktypes/worktypes.repository.service.ts b/data_migration_tools/server/src/repositories/worktypes/worktypes.repository.service.ts index 9f7f2ff..3245b63 100644 --- a/data_migration_tools/server/src/repositories/worktypes/worktypes.repository.service.ts +++ b/data_migration_tools/server/src/repositories/worktypes/worktypes.repository.service.ts @@ -13,7 +13,7 @@ import { import { OptionItem } from "./entity/option_item.entity"; import { insertEntities, insertEntity } from "../../common/repository"; import { Context } from "../../common/log"; -import { WorktypesInputFile } from "../../common/types/types"; +import { WorktypesFile } from "../../common/types/types"; @Injectable() export class WorktypesRepositoryService { @@ -30,15 +30,15 @@ export class WorktypesRepositoryService { */ async createWorktype( context: Context, - worktypesInputFiles: WorktypesInputFile[] + WorktypesFiles: WorktypesFile[] ): Promise { await this.dataSource.transaction(async (entityManager) => { const worktypeRepo = entityManager.getRepository(Worktype); const optionItemRepo = entityManager.getRepository(OptionItem); - for (const worktypesInputFile of worktypesInputFiles) { - const accountId = worktypesInputFile.account_id; - const worktypeId = worktypesInputFile.custom_worktype_id; + for (const WorktypesFile of WorktypesFiles) { + const accountId = WorktypesFile.account_id; + const worktypeId = WorktypesFile.custom_worktype_id; const description = null; const duplicatedWorktype = await worktypeRepo.findOne({ From f6d39a4c26d7fb0d32a0b43cf32ed28e4270f05c Mon Sep 17 00:00:00 2001 From: "maruyama.t" Date: Tue, 27 Feb 2024 23:55:44 +0000 Subject: [PATCH 02/12] =?UTF-8?q?Merged=20PR=20788:=20[2=E5=9B=9E=E7=9B=AE?= =?UTF-8?q?=E5=AE=9F=E8=A1=8C]=E5=AE=9F=E6=96=BD=E5=BE=8C=E3=81=AE?= =?UTF-8?q?=E5=8B=95=E4=BD=9C=E7=A2=BA=E8=AA=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3577: [2回目実行]実施後の動作確認](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3577) accountがCountryの場合に配下のDistributorの親アカウントIDを付け替える処理について、Typeの付け替えができていなかったのを修正。 ライセンスの有効期限が"9999/12/31 23:59:59.997"でフォーマットチェックをしているが、移行元は"9999/12/31 23:59:59"なので移行元に合わせた。 dealerAccountIdが設定されているが、そのdealerが存在しない場合もデータを作ってしまっている。 →該当レコードはエラーファイルを出力する。 ## レビューポイント - エラーファイルの出力処理だが簡素すぎるか？ JSONで出力する意味はないが、これまでの動作確認で動作担保できているのでJSONで出しています。 ## 動作確認状況 - ローカルで確認 正常の場合データ変換が行われることを確認。 dealerAccountIdが設定されているが、そのdealerが存在しない場合もデータでテストした場合、error.jsonが作られることを確認。 ## 補足 - 相談、参考資料などがあれば --- .../src/features/transfer/transfer.service.ts | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/data_migration_tools/server/src/features/transfer/transfer.service.ts b/data_migration_tools/server/src/features/transfer/transfer.service.ts index c1b0757..cf8dd53 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.service.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.service.ts @@ -50,7 +50,7 @@ export class TransferService { let usersFileLines: UsersFile[] = []; let licensesFileLines: LicensesFile[] = []; let worktypesFileLines: WorktypesFile[] = []; - + let errorArray: string[] = []; let userIdIndex = 0; // authorIdとuserIdの対応関係を保持するMapを定義 const authorIdToUserIdMap: Map = new Map(); @@ -75,9 +75,11 @@ export class TransferService { if (line.parent_id) { parentAccountId = accountIdMap.get(line.parent_id); } - // 万が一parent_idが入力されているのに存在しなかった場合は、nullを設定する。 + // 万が一parent_idが入力されているのに存在しなかった場合は、エラー配列に追加する if (parentAccountId === undefined) { - parentAccountId = null; + errorArray.push( + `parent_id is invalid. parent_id=${line.parent_id}` + ); } // userIdIndexをインクリメントする @@ -126,8 +128,8 @@ export class TransferService { } // ライセンスのデータの作成を行う - // line.expired_dateが9999/12/31 23:59:59.997のデータの場合はデモライセンスなので登録しない - if (line.expired_date !== "9999/12/31 23:59:59.997") { + // line.expired_dateが"9999/12/31 23:59:59"のデータの場合はデモライセンスなので登録しない + if (line.expired_date !== "9999/12/31 23:59:59") { // authorIdが設定されてる場合、statusは"allocated"、allocated_user_idは対象のユーザID // されていない場合、statusは"reusable"、allocated_user_idはnull let status: string; @@ -176,6 +178,15 @@ export class TransferService { } } }); + // エラー配列に値が存在する場合はエラーファイルを出力する + if (errorArray.length > 0) { + const errorFileJson = JSON.stringify(errorArray); + fs.writeFileSync(`error.json`, errorFileJson); + throw new HttpException( + `errorArray is invalid. errorArray=${errorArray}`, + HttpStatus.BAD_REQUEST + ); + } return { accountsFileTypeLines, usersFileLines, @@ -211,24 +222,23 @@ export class TransferService { try { const relocatedAccounts: AccountsFile[] = []; - const countryRecords: Map = new Map(); + const dealerRecords: Map = new Map(); // accountsFileTypeをループ accountsFileType.forEach((account) => { - // Countryの場合はDistributorのアカウントIDと新たな親アカウントID（BC）の組み合わせをMapに登録 - if (account.type === MIGRATION_TYPE.COUNTRY) { - // 配下のDistributorアカウントを取得 - const distributor = accountsFileType.find( - (distributor) => - distributor.dealerAccountId === account.accountId && - distributor.type === MIGRATION_TYPE.DISTRIBUTOR + // Distributorの場合はdealerを検索し、COUNTRYかチェックする + if (account.type === MIGRATION_TYPE.DISTRIBUTOR) { + const distributorParent = accountsFileType.find( + (a) => a.accountId === account.dealerAccountId ); - if (distributor) { - countryRecords.set(distributor.accountId, account.dealerAccountId); + if (distributorParent.type === MIGRATION_TYPE.COUNTRY) { + dealerRecords.set( + account.accountId, + distributorParent.dealerAccountId // Countryの親、BCのIDを設定 + ); } } else { - // Country以外のアカウントの場合は、そのまま登録 - countryRecords.set(account.accountId, account.dealerAccountId); + dealerRecords.set(account.accountId, account.dealerAccountId); } }); @@ -237,7 +247,7 @@ export class TransferService { //　Countryのレコードは除外する if (account.type !== MIGRATION_TYPE.COUNTRY) { const dealerAccountId = - countryRecords.get(account.dealerAccountId) ?? + dealerRecords.get(account.dealerAccountId) ?? account.dealerAccountId; const type = this.getAccountType(account.type); const newAccount: AccountsFile = { From 0be9c26f09ff57a380e41b2b1c7bbf3e00fbcc5a Mon Sep 17 00:00:00 2001 From: masaaki Date: Wed, 28 Feb 2024 05:31:13 +0000 Subject: [PATCH 03/12] =?UTF-8?q?Merged=20PR=20781:=20=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E6=A4=9C=E8=A8=BC=E3=83=84=E3=83=BC=E3=83=AB=E4=BD=9C?= =?UTF-8?q?=E6=88=90=EF=BC=8B=E5=8B=95=E4=BD=9C=E7=A2=BA=E8=AA=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3573: データ検証ツール作成＋動作確認](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3573) - データ検証ツールを作成しました ## レビューポイント - 特にレビューしてほしい箇所 　詳細情報の突き合わせについて、ラフスケッチと対応しているか第三者目線でも確認してほしいです 　verification.serviceのcompareCardLicenses、compareLicenses、compareAccountsになります。 ## UIの変更 - 無し ## 動作確認状況 - ローカルで確認 ## 補足 - 無し --- data_migration_tools/server/package-lock.json | 66 ++ data_migration_tools/server/package.json | 3 +- data_migration_tools/server/src/app.module.ts | 7 + .../server/src/common/types/types.ts | 83 +++ .../features/transfer/transfer.controller.ts | 17 +- .../src/features/verification/types/types.ts | 9 + .../verification/verification.controller.ts | 148 ++++ .../verification/verification.module.ts | 17 + .../verification/verification.service.ts | 695 ++++++++++++++++++ .../accounts/accounts.repository.service.ts | 17 + .../licenses/licenses.repository.service.ts | 30 + .../users/users.repository.service.ts | 16 + 12 files changed, 1106 insertions(+), 2 deletions(-) create mode 100644 data_migration_tools/server/src/features/verification/types/types.ts create mode 100644 data_migration_tools/server/src/features/verification/verification.controller.ts create mode 100644 data_migration_tools/server/src/features/verification/verification.module.ts create mode 100644 data_migration_tools/server/src/features/verification/verification.service.ts diff --git a/data_migration_tools/server/package-lock.json b/data_migration_tools/server/package-lock.json index c1c2143..1a8aff6 100644 --- a/data_migration_tools/server/package-lock.json +++ b/data_migration_tools/server/package-lock.json @@ -26,6 +26,7 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", + "csv": "^6.3.6", "multer": "^1.4.5-lts.1", "mysql2": "^2.3.3", "reflect-metadata": "^0.1.13", @@ -4049,6 +4050,35 @@ "node": ">= 8" } }, + "node_modules/csv": { + "version": "6.3.6", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz", + "integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==", + "dependencies": { + "csv-generate": "^4.3.1", + "csv-parse": "^5.5.3", + "csv-stringify": "^6.4.5", + "stream-transform": "^3.3.0" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz", + "integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w==" + }, + "node_modules/csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==" + }, + "node_modules/csv-stringify": { + "version": "6.4.5", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz", + "integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ==" + }, "node_modules/date-fns": { "version": "2.30.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", @@ -8654,6 +8684,11 @@ "npm": ">=6" } }, + "node_modules/stream-transform": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz", + "integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA==" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -13027,6 +13062,32 @@ "which": "^2.0.1" } }, + "csv": { + "version": "6.3.6", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz", + "integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==", + "requires": { + "csv-generate": "^4.3.1", + "csv-parse": "^5.5.3", + "csv-stringify": "^6.4.5", + "stream-transform": "^3.3.0" + } + }, + "csv-generate": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz", + "integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w==" + }, + "csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==" + }, + "csv-stringify": { + "version": "6.4.5", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz", + "integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ==" + }, "date-fns": { "version": "2.30.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", @@ -16557,6 +16618,11 @@ "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==" }, + "stream-transform": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz", + "integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA==" + }, "streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", diff --git a/data_migration_tools/server/package.json b/data_migration_tools/server/package.json index a2299cb..a0502b7 100644 --- a/data_migration_tools/server/package.json +++ b/data_migration_tools/server/package.json @@ -45,7 +45,8 @@ "reflect-metadata": "^0.1.13", "rxjs": "^7.8.0", "swagger-cli": "^4.0.4", - "typeorm": "^0.3.20" + "typeorm": "^0.3.20", + "csv": "^6.3.6" }, "devDependencies": { "@types/express": "^4.17.17", diff --git a/data_migration_tools/server/src/app.module.ts b/data_migration_tools/server/src/app.module.ts index 6935bba..bcf32fe 100644 --- a/data_migration_tools/server/src/app.module.ts +++ b/data_migration_tools/server/src/app.module.ts @@ -27,6 +27,10 @@ import { DeleteService } from "./features/delete/delete.service"; import { TransferModule } from "./features/transfer/transfer.module"; import { TransferController } from "./features/transfer/transfer.controller"; import { TransferService } from "./features/transfer/transfer.service"; +import { VerificationController } from "./features/verification/verification.controller"; +import { VerificationService } from "./features/verification/verification.service"; +import { VerificationModule } from "./features/verification/verification.module"; + @Module({ imports: [ ServeStaticModule.forRoot({ @@ -41,6 +45,7 @@ import { TransferService } from "./features/transfer/transfer.service"; UsersModule, TransferModule, RegisterModule, + VerificationModule, AccountsRepositoryModule, UsersRepositoryModule, SortCriteriaRepositoryModule, @@ -70,6 +75,7 @@ import { TransferService } from "./features/transfer/transfer.service"; UsersController, DeleteController, TransferController, + VerificationController, ], providers: [ RegisterService, @@ -77,6 +83,7 @@ import { TransferService } from "./features/transfer/transfer.service"; UsersService, DeleteService, TransferService, + VerificationService, ], }) export class AppModule { diff --git a/data_migration_tools/server/src/common/types/types.ts b/data_migration_tools/server/src/common/types/types.ts index 69ef466..529a4ad 100644 --- a/data_migration_tools/server/src/common/types/types.ts +++ b/data_migration_tools/server/src/common/types/types.ts @@ -34,6 +34,11 @@ export class csvInputFile { wt19: string; wt20: string; } + +export class csvInputFileWithRow extends csvInputFile { + row: number; +} + export class AccountsFileType { accountId: number; type: string; @@ -93,6 +98,22 @@ export class CardLicensesFile { updated_by?: string; } +export class AccountsMappingFile { + accountIdText: string; + accountIdNumber: number; +} + +export class VerificationResultDetails { + input: string; + inputRow: number; + diffTargetTable: string; + columnName: string; + fileData: string; + databaseData: string; + reason: string; +} + + export function isAccountsFileArray(obj: any): obj is AccountsFile[] { return Array.isArray(obj) && obj.every((item) => isAccountsFile(item)); } @@ -202,3 +223,65 @@ export function isCardLicensesFile(obj: any): obj is CardLicensesFile { (obj.updated_by === null || typeof obj.updated_by === "string") ); } + +export function isAccountsMappingFileArray( + obj: any +): obj is AccountsMappingFile[] { + return Array.isArray(obj) && obj.every((item) => isAccountsMappingFile(item)); +} +export function isAccountsMappingFile(obj: any): obj is AccountsMappingFile { + return ( + typeof obj === "object" && + obj !== null && + "accountIdText" in obj && + "accountIdNumber" in obj && + typeof obj.accountIdText === "string" && + typeof obj.accountIdNumber === "number" + ); +} + +export function isCsvInputFileForValidateArray(obj: any): obj is csvInputFile[] { + return ( + Array.isArray(obj) && obj.every((item) => isCsvInputFileForValidate(item)) + ); +} + +export function isCsvInputFileForValidate(obj: any): obj is csvInputFile { + return ( + typeof obj === "object" && + "type" in obj && + "account_id" in obj && + "parent_id" in obj && + "email" in obj && + "company_name" in obj && + "first_name" in obj && + "last_name" in obj && + "country" in obj && + "state" in obj && + "start_date" in obj && + "expired_date" in obj && + "user_email" in obj && + "author_id" in obj && + "recording_mode" in obj && + "wt1" in obj && + "wt2" in obj && + "wt3" in obj && + "wt4" in obj && + "wt5" in obj && + "wt6" in obj && + "wt7" in obj && + "wt8" in obj && + "wt9" in obj && + "wt10" in obj && + "wt11" in obj && + "wt12" in obj && + "wt13" in obj && + "wt14" in obj && + "wt15" in obj && + "wt16" in obj && + "wt17" in obj && + "wt18" in obj && + "wt19" in obj && + "wt20" in obj + ); +} diff --git a/data_migration_tools/server/src/features/transfer/transfer.controller.ts b/data_migration_tools/server/src/features/transfer/transfer.controller.ts index 7916f27..ee38969 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.controller.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.controller.ts @@ -13,7 +13,7 @@ import { Request } from "express"; import { transferRequest, transferResponse } from "./types/types"; import { TransferService } from "./transfer.service"; import { makeContext } from "../../common/log"; -import { csvInputFile } from "../../common/types/types"; +import { csvInputFile, AccountsMappingFile } from "../../common/types/types"; import { makeErrorResponse } from "src/common/errors/makeErrorResponse"; import { AUTO_INCREMENT_START } from "../../constants"; @ApiTags("transfer") @@ -149,6 +149,21 @@ export class TransferController { accountIdListArray.forEach((accountId, index) => { accountIdMap.set(accountId, index + AUTO_INCREMENT_START); }); + + // アカウントID numberとstring対応表の出力 + const accountsMappingFiles: AccountsMappingFile[] = []; + accountIdMap.forEach((value, key) => { + const accountsMappingFile = new AccountsMappingFile(); + accountsMappingFile.accountIdNumber = value; + accountsMappingFile.accountIdText = key + accountsMappingFiles.push(accountsMappingFile) + }); + + fs.writeFileSync( + `${inputFilePath}account_map.json`, + JSON.stringify(accountsMappingFiles) + ); + // CSVファイルの変換 const transferResponseCsv = await this.transferService.transferInputData( context, diff --git a/data_migration_tools/server/src/features/verification/types/types.ts b/data_migration_tools/server/src/features/verification/types/types.ts new file mode 100644 index 0000000..2eda873 --- /dev/null +++ b/data_migration_tools/server/src/features/verification/types/types.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class VerificationRequest { + @ApiProperty() + inputFilePath: string; +} + +export class VerificationResponse {} + diff --git a/data_migration_tools/server/src/features/verification/verification.controller.ts b/data_migration_tools/server/src/features/verification/verification.controller.ts new file mode 100644 index 0000000..31cde89 --- /dev/null +++ b/data_migration_tools/server/src/features/verification/verification.controller.ts @@ -0,0 +1,148 @@ +import { + Body, + Controller, + HttpStatus, + Post, + Req, + Logger, + HttpException, +} from "@nestjs/common"; +import { makeErrorResponse } from "../../common/error/makeErrorResponse"; +import fs from "fs"; +import { ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger"; +import { Request } from "express"; +import { VerificationRequest, VerificationResponse } from "./types/types"; +import { VerificationService } from "./verification.service"; +import { makeContext } from "../../common/log"; +import { + csvInputFileWithRow, + isAccountsMappingFileArray, + isCardLicensesFileArray, + isCsvInputFileForValidateArray, +} from "../../common/types/types"; +import * as csv from "csv"; + +@ApiTags("verification") +@Controller("verification") +export class VerificationController { + private readonly logger = new Logger(VerificationController.name); + constructor(private readonly verificationService: VerificationService) {} + + @Post() + @ApiResponse({ + status: HttpStatus.OK, + type: VerificationResponse, + description: "成功時のレスポンス", + }) + @ApiResponse({ + status: HttpStatus.INTERNAL_SERVER_ERROR, + description: "想定外のサーバーエラー", + }) + @ApiOperation({ operationId: "dataVerification" }) + async dataVerification( + @Body() body: VerificationRequest, + @Req() req: Request + ): Promise { + const context = makeContext("iko", "varification"); + + const inputFilePath = body.inputFilePath; + + this.logger.log( + `[IN] [${context.getTrackingId()}] ${ + this.dataVerification.name + } | params: { inputFilePath: ${inputFilePath}};` + ); + + try { + // 読み込みファイルのフルパス + const accountTransitionFileFullPath = + inputFilePath + "Account_transition.csv"; + const accountMapFileFullPath = inputFilePath + "account_map.json"; + const cardLicensesFileFullPath = inputFilePath + "cardLicenses.json"; + + // ファイル存在チェックと読み込み + if (!fs.existsSync(accountTransitionFileFullPath)) { + this.logger.error( + `file not exists from ${accountTransitionFileFullPath}` + ); + throw new Error( + `file not exists from ${accountTransitionFileFullPath}` + ); + } + + if (!fs.existsSync(accountMapFileFullPath)) { + this.logger.error(`file not exists from ${accountMapFileFullPath}`); + throw new Error(`file not exists from ${accountMapFileFullPath}`); + } + + if (!fs.existsSync(cardLicensesFileFullPath)) { + this.logger.error(`file not exists from ${cardLicensesFileFullPath}`); + throw new Error(`file not exists from ${cardLicensesFileFullPath}`); + } + + // カードライセンスの登録用ファイル読み込み + const cardLicensesObject = JSON.parse( + fs.readFileSync(cardLicensesFileFullPath, "utf8") + ); + + // 型ガード（cardLicenses） + if (!isCardLicensesFileArray(cardLicensesObject)) { + throw new Error("input file is not cardLicensesInputFiles"); + } + + // アカウントIDマッピング用ファイル読み込み + const accountsMapObject = JSON.parse( + fs.readFileSync(accountMapFileFullPath, "utf8") + ); + + // 型ガード（accountsMapingFile） + if (!isAccountsMappingFileArray(accountsMapObject)) { + throw new Error("input file is not accountsMapingFile"); + } + + // 移行用csvファイルの読み込み（csv parse） + fs.createReadStream(accountTransitionFileFullPath).pipe( + csv.parse({ columns: true, delimiter: "," }, (err, csvInputFiles) => { + // 型ガード（csvInputFile） + if (!isCsvInputFileForValidateArray(csvInputFiles)) { + throw new Error("input file is not csvInputFile"); + } + + const csvInputFileswithRows: csvInputFileWithRow[] = []; + let rowCount = 2; // csvの何行目かを表す変数。ヘッダ行があるので2から開始 + for (const csvInputFile of csvInputFiles) { + const csvInputFileswithRow: csvInputFileWithRow = { + ...csvInputFile, + row: rowCount + }; + csvInputFileswithRows.push(csvInputFileswithRow); + rowCount = rowCount + 1; + } + this.verificationService.varificationData( + context, + inputFilePath, + csvInputFileswithRows, + accountsMapObject, + cardLicensesObject + ); + }) + ); + + return {}; + } catch (e) { + this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); + } finally { + this.logger.log( + `[OUT] [${context.getTrackingId()}] ${this.dataVerification.name}` + ); + } + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/data_migration_tools/server/src/features/verification/verification.module.ts b/data_migration_tools/server/src/features/verification/verification.module.ts new file mode 100644 index 0000000..51b9892 --- /dev/null +++ b/data_migration_tools/server/src/features/verification/verification.module.ts @@ -0,0 +1,17 @@ +import { Module } from "@nestjs/common"; +import { VerificationController } from "./verification.controller"; +import { VerificationService } from "./verification.service"; +import { LicensesRepositoryModule } from "../../repositories/licenses/licenses.repository.module"; +import { AccountsRepositoryModule } from "../../repositories/accounts/accounts.repository.module"; +import { UsersRepositoryModule } from "../../repositories//users/users.repository.module"; + +@Module({ + imports: [ + LicensesRepositoryModule, + AccountsRepositoryModule, + UsersRepositoryModule, + ], + controllers: [VerificationController], + providers: [VerificationService], +}) +export class VerificationModule {} diff --git a/data_migration_tools/server/src/features/verification/verification.service.ts b/data_migration_tools/server/src/features/verification/verification.service.ts new file mode 100644 index 0000000..1de9596 --- /dev/null +++ b/data_migration_tools/server/src/features/verification/verification.service.ts @@ -0,0 +1,695 @@ +import { HttpException, HttpStatus, Injectable, Logger } from "@nestjs/common"; +import { Context } from "../../common/log"; +import { + AccountsMappingFile, + CardLicensesFile, + csvInputFileWithRow, + VerificationResultDetails, +} from "../../common/types/types"; +import { + AUTO_INCREMENT_START, + MIGRATION_TYPE, + COUNTRY_LIST, +} from "../../constants/index"; + +import { makeErrorResponse } from "../../common/error/makeErrorResponse"; +import { LicensesRepositoryService } from "../../repositories/licenses/licenses.repository.service"; +import { + License, + CardLicense, +} from "../../repositories/licenses/entity/license.entity"; +import { AccountsRepositoryService } from "../../repositories/accounts/accounts.repository.service"; +import { UsersRepositoryService } from "../../repositories//users/users.repository.service"; +import { Account } from "src/repositories/accounts/entity/account.entity"; +import fs from "fs"; + +@Injectable() +export class VerificationService { + constructor( + private readonly AccountsRepository: AccountsRepositoryService, + private readonly UsersRepository: UsersRepositoryService, + private readonly licensesRepository: LicensesRepositoryService + ) {} + private readonly logger = new Logger(VerificationService.name); + + /** + * Verification Data + * @param inputFilePath: string + */ + async varificationData( + context: Context, + inputFilePath: string, + csvInputFiles: csvInputFileWithRow[], + accountsMappingInputFiles: AccountsMappingFile[], + cardlicensesInputFiles: CardLicensesFile[] + ): Promise { + // パラメータ内容が長大なのでログには出さない + this.logger.log( + `[IN] [${context.getTrackingId()}] ${this.varificationData.name}` + ); + + // this.logger.log(csvInputFiles); + try { + // 件数情報の取得 + this.logger.log(`入力ファイルから件数情報を取得する`); + + const accountCountFromFile = csvInputFiles.filter( + (item) => item.type !== "USER" && item.type !== "Country" + ).length; + const cardLicensesCountFromFile = cardlicensesInputFiles.length; + + const licensesCountFromFile = + csvInputFiles.filter( + (item) => + item.type === "USER" && item.expired_date !== "9999/12/31 23:59:59" + ).length + cardLicensesCountFromFile; + + // 管理ユーザ数のカウント + const administratorCountFromFile = accountCountFromFile; + // 一般ユーザ数のカウント + const normaluserCountFromFile = csvInputFiles.filter( + (item) => item.type === "USER" && item.user_email.length !== 0 + ).length; + + // ユーザ重複数のカウント + let mailAdresses: string[] = []; + csvInputFiles.forEach((item) => { + // メールアドレスの要素を配列に追加（入力データとして管理者とユーザの両方に入ることはない） + if (item.email.length !== 0) { + mailAdresses.push(item.email); + } + if (item.user_email.length !== 0) { + mailAdresses.push(item.user_email); + } + }); + + // 重複する要素を抽出 + const duplicates: { [key: string]: number } = {}; + mailAdresses.forEach((str) => { + duplicates[str] = (duplicates[str] || 0) + 1; + }); + + // 重複する要素と件数を表示 + let duplicateCount = 0; + Object.keys(duplicates).forEach((key) => { + const count = duplicates[key]; + if (count > 1) { + // 重複件数をカウント + duplicateCount = duplicateCount + (count - 1); + //console.log(`${key}が${count}件`); + } + }); + const userCountFromFile = + administratorCountFromFile + normaluserCountFromFile - duplicateCount; + + this.logger.log(`accountCountFromFile=${accountCountFromFile}`); + this.logger.log(`cardLicensesCountFromFile=${cardLicensesCountFromFile}`); + this.logger.log(`licensesCountFromFile=${licensesCountFromFile}`); + this.logger.log(`userCountFromFile=${userCountFromFile}`); + + // DBから情報を取得する + this.logger.log(`DBの情報を取得する`); + + const accounts = await this.AccountsRepository.getAllAccounts(context); + const users = await this.UsersRepository.getAllUsers(context); + const licenses = await this.licensesRepository.getAllLicenses(context); + const cardLicenses = await this.licensesRepository.getAllCardLicense( + context + ); + + // DB件数のカウント + this.logger.log(`DBの情報から件数を取得する`); + const accountsCountFromDB = accounts.length; + const usersCountFromDB = users.length; + const licensesCountFromDB = licenses.length; + const cardLicensesCountFromDB = cardLicenses.length; + + this.logger.log(`accountsCountFromDB=${accountsCountFromDB}`); + this.logger.log(`usersCountFromDB=${usersCountFromDB}`); + this.logger.log(`licensesCountFromDB=${licensesCountFromDB}`); + this.logger.log(`cardLicensesCountFromDB=${cardLicensesCountFromDB}`); + + // エラー情報の定義 + const VerificationResultDetails: VerificationResultDetails[] = []; + + // カードライセンス関連の情報突き合わせ + this.logger.log(`カードライセンス関連の情報突き合わせ`); + const isCardDetailNoError = compareCardLicenses( + VerificationResultDetails, + cardlicensesInputFiles, + cardLicenses, + licenses + ); + + // ライセンス関連の情報突き合わせ + this.logger.log(`ライセンス関連の情報突き合わせ`); + const isLicensesDetailNoError = compareLicenses( + VerificationResultDetails, + csvInputFiles.filter( + (item) => + item.type === "USER" && item.expired_date !== "9999/12/31 23:59:59" + ), + licenses.filter((item) => item.expiry_date !== null), + accountsMappingInputFiles + ); + + // アカウント情報の突き合わせ + this.logger.log(`アカウント関連の情報突き合わせ`); + const isAccountsDetailNoError = compareAccounts( + VerificationResultDetails, + csvInputFiles.filter( + (item) => item.type !== "USER" && item.type !== "Country" + ), + csvInputFiles.filter((item) => item.type === "Country"), + accounts, + accountsMappingInputFiles + ); + + // 結果の判定と出力 + this.logger.log(`結果の判定と出力`); + const isAccountCountNoDifference = + accountCountFromFile === accountsCountFromDB; + const isUsersCountNoDifference = userCountFromFile === usersCountFromDB; + const isLicensesCountNoDifference = + licensesCountFromFile === licensesCountFromDB; + const isCardLicensesCountNoDifference = + cardLicensesCountFromFile === cardLicensesCountFromDB; + const isNoDetailError = VerificationResultDetails.length === 0; + + const isSummaryNoError = + isAccountCountNoDifference && + isUsersCountNoDifference && + isLicensesCountNoDifference && + isCardLicensesCountNoDifference && + isNoDetailError; + + const summaryString = ` +サマリファイル： + +比較結果：${isSummaryNoError ? "OK" : "NG"} + +件数： +アカウント：${ + isAccountCountNoDifference ? "OK" : "NG" + }（csv件数:${accountCountFromFile}/DB件数:${accountsCountFromDB}） +ライセンス：${ + isLicensesCountNoDifference ? "OK" : "NG" + }（csv件数:${licensesCountFromFile}/DB件数:${licensesCountFromDB}） +カードライセンス：${ + isCardLicensesCountNoDifference ? "OK" : "NG" + }（csv件数:${cardLicensesCountFromFile}/DB件数:${cardLicensesCountFromDB}） +ユーザー：${ + isUsersCountNoDifference ? "OK" : "NG" + }（csv件数:${userCountFromFile}/DB件数:${usersCountFromDB}） + +項目比較： +アカウント：${isAccountsDetailNoError ? "OK" : "NG"} +カードライセンス：${isCardDetailNoError ? "OK" : "NG"} +ライセンス：${isLicensesDetailNoError ? "OK" : "NG"} +`; + + // サマリファイルの書き込み + fs.writeFileSync(`${inputFilePath}resultsummary.txt`, summaryString); + + // 詳細ファイルの書き込み + // 配列をJSON文字列に変換 + const jsonContent = JSON.stringify(VerificationResultDetails, null, 2); + + // JSONをファイルに書き込み + fs.writeFileSync(`${inputFilePath}resultdetail.json`, jsonContent); + } catch (e) { + this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); + } finally { + this.logger.log( + `[OUT] [${context.getTrackingId()}] ${this.varificationData.name}` + ); + } + } +} + +// dateを任意のフォーマットに変換する +const getFormattedDate = (date: Date | null, format: string) => { + if (!date) { + return null; + } + const symbol = { + M: date.getMonth() + 1, + d: date.getDate(), + h: date.getHours(), + m: date.getMinutes(), + s: date.getSeconds(), + }; + + const formatted = format.replace(/(M+|d+|h+|m+|s+)/g, (v) => + ( + (v.length > 1 ? "0" : "") + symbol[v.slice(-1) as keyof typeof symbol] + ).slice(-2) + ); + + return formatted.replace(/(y+)/g, (v) => + date.getFullYear().toString().slice(-v.length) + ); +}; + +// 親の階層がcountryの場合、countryの親を返却する +function transrateCountryHierarchy( + countriesFromFile: csvInputFileWithRow[], + targetParentAccountIdString: string +): string { + for (const countryFromFile of countriesFromFile) { + if (countryFromFile.account_id === targetParentAccountIdString) { + return countryFromFile.parent_id; + } + } + return targetParentAccountIdString; +} + +// アカウントID（number）を対応するアカウントID（string）に変換する +function findAccountIdText( + accountsMappings: AccountsMappingFile[], + targetAccountIdNumber: number +): string { + if (targetAccountIdNumber == null) { + return ""; + } + for (const accountsMapping of accountsMappings) { + if (accountsMapping.accountIdNumber === targetAccountIdNumber) { + return accountsMapping.accountIdText; + } + } + return `NO_MATCHED_ACCOUNTID_${targetAccountIdNumber}`; // マッチするものが見つからない場合 +} + +// 階層（number）を対応する階層（string）に変換する +function getMigrationTypeByNumber(numberValue: number): string { + switch (numberValue) { + case 1: + return MIGRATION_TYPE.ADMINISTRATOR; + case 2: + return MIGRATION_TYPE.BC; + case 3: + return MIGRATION_TYPE.DISTRIBUTOR; + case 4: + return MIGRATION_TYPE.DEALER; + case 5: + return MIGRATION_TYPE.CUSTOMER; + default: + return `NO_MATCHED_TIER_${numberValue}`; + } +} + +// 国（省略版）を対応する国（非省略版）に変換する +function getCountryLabelByValue(value: string): string { + const country = COUNTRY_LIST.find((country) => country.value === value); + return country ? country.label : `NO_MATCHED_COUNTRY_${value}`; +} + +// カードライセンス情報の突き合わせを行い、エラー時はエラー情報配列に情報追加する +function compareCardLicenses( + VerificationResultDetails: VerificationResultDetails[], + cardlicensesInputFiles: CardLicensesFile[], + cardLicenses: CardLicense[], + licenses: License[] +): boolean { + let isNoError = true; + + let row = 1; // カードライセンスファイルの行数 + for (const cardlicensesInputFile of cardlicensesInputFiles) { + const filterdCardLicenses = cardLicenses.filter( + (cardLicenses) => + cardLicenses.card_license_key === cardlicensesInputFile.card_license_key + ); + + if (filterdCardLicenses.length === 0) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "cardLicenses", + columnName: "card_license_key", + fileData: cardlicensesInputFile.card_license_key, + databaseData: "-", + reason: "レコード無し", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + /* issue_idは自動採番のため比較しない + if (cardlicensesInputFile.issue_id !== filterdCardLicenses[0].issue_id) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + diffTargetTable: "cardLicenses", + columnName: "issue_id", + fileData: cardlicensesInputFile.issue_id.toString(), + databaseData: filterdCardLicenses[0].issue_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + */ + + const formattedActivated = getFormattedDate( + filterdCardLicenses[0].activated_at, + `yyyy/MM/dd hh:mm:ss` + ); + if (cardlicensesInputFile.activated_at !== formattedActivated) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "cardLicenses", + columnName: "activated_at", + fileData: cardlicensesInputFile.activated_at, + databaseData: formattedActivated, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + + const filterdLicenses = licenses.filter( + (licenses) => licenses.id === filterdCardLicenses[0].license_id + ); + if (filterdLicenses.length === 0) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "id", + fileData: filterdCardLicenses[0].license_id.toString(), + databaseData: "-", + reason: "紐つくライセンスのレコード無し", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + if (filterdLicenses[0].expiry_date !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "expiry_date", + fileData: null, + databaseData: getFormattedDate( + filterdLicenses[0].expiry_date, + `yyyy/MM/dd hh:mm:ss` + ), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + + if (filterdLicenses[0].account_id !== AUTO_INCREMENT_START) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "account_id", + fileData: AUTO_INCREMENT_START.toString(), + databaseData: filterdLicenses[0].account_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + + if (filterdLicenses[0].type !== "CARD") { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "type", + fileData: "CARD", + databaseData: filterdLicenses[0].type, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + + if (filterdLicenses[0].status !== "Unallocated") { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "status", + fileData: "Unallocated", + databaseData: filterdLicenses[0].status, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if (filterdLicenses[0].allocated_user_id !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "allocated_user_id", + fileData: null, + databaseData: filterdLicenses[0].allocated_user_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if (filterdLicenses[0].order_id !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "order_id", + fileData: null, + databaseData: filterdLicenses[0].order_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if (filterdLicenses[0].deleted_at !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "deleted_at", + fileData: null, + databaseData: getFormattedDate( + filterdLicenses[0].deleted_at, + `yyyy/MM/dd hh:mm:ss` + ), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if (filterdLicenses[0].delete_order_id !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "delete_order_id", + fileData: null, + databaseData: filterdLicenses[0].delete_order_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + row = row + 1; + } + return isNoError; +} + +// ライセンス情報の突き合わせを行い、エラー時はエラー情報配列に情報追加する +function compareLicenses( + VerificationResultDetails: VerificationResultDetails[], + licensesFromFile: csvInputFileWithRow[], + licensesFromDatabase: License[], + accountsMappingInputFiles: AccountsMappingFile[] +): boolean { + let isNoError = true; + for (let i = 0; i < licensesFromFile.length; i++) { + if ( + !licensesFromDatabase[i] || + licensesFromFile[i].account_id !== + findAccountIdText( + accountsMappingInputFiles, + licensesFromDatabase[i].account_id + ) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: licensesFromFile[i].row, + diffTargetTable: "licenses", + columnName: "account_id", + fileData: licensesFromFile[i].account_id, + databaseData: licensesFromDatabase[i] + ? findAccountIdText( + accountsMappingInputFiles, + licensesFromDatabase[i].account_id + ) + `(${licensesFromDatabase[i].account_id})` + : "undifined", + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if ( + !licensesFromDatabase[i] || + licensesFromFile[i].expired_date !== + getFormattedDate( + licensesFromDatabase[i].expiry_date, + `yyyy/MM/dd hh:mm:ss` + ) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: licensesFromFile[i].row, + diffTargetTable: "licenses", + columnName: "expired_date", + fileData: licensesFromFile[i].expired_date, + databaseData: licensesFromDatabase[i] + ? getFormattedDate( + licensesFromDatabase[i].expiry_date, + `yyyy/MM/dd hh:mm:ss` + ) + : "undifined", + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + } + return isNoError; +} + +// アカウント情報の突き合わせを行い、エラー時はエラー情報配列に情報追加する +function compareAccounts( + VerificationResultDetails: VerificationResultDetails[], + accountsFromFile: csvInputFileWithRow[], + countriesFromFile: csvInputFileWithRow[], + accountsFromDatabase: Account[], + accountsMappingInputFiles: AccountsMappingFile[] +): boolean { + let isNoError = true; + for (const accountFromFile of accountsFromFile) { + // DBレコードの存在チェック + const filterdAccounts = accountsFromDatabase.filter( + (accountsFromDatabase) => + findAccountIdText( + accountsMappingInputFiles, + accountsFromDatabase.id + ) === accountFromFile.account_id + ); + + if (filterdAccounts.length === 0) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "account_id", + fileData: accountFromFile.account_id, + databaseData: "-", + reason: "レコード無し", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + // 項目チェック（parent_account_id） + const transratedParentId = transrateCountryHierarchy( + countriesFromFile, + accountFromFile.parent_id + ); + if ( + transratedParentId !== + findAccountIdText( + accountsMappingInputFiles, + filterdAccounts[0].parent_account_id + ) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "parent_account_id", + fileData: + transratedParentId === accountFromFile.parent_id + ? accountFromFile.parent_id + : `${transratedParentId}(${accountFromFile.parent_id})`, + databaseData: + findAccountIdText( + accountsMappingInputFiles, + filterdAccounts[0].parent_account_id + ) + `(${filterdAccounts[0].parent_account_id})`, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + // 項目チェック（tier） + if ( + accountFromFile.type !== getMigrationTypeByNumber(filterdAccounts[0].tier) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "tier", + fileData: accountFromFile.type, + databaseData: getMigrationTypeByNumber(filterdAccounts[0].tier), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + // 項目チェック（country） + if ( + accountFromFile.country !== + getCountryLabelByValue(filterdAccounts[0].country) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "country", + fileData: accountFromFile.country, + databaseData: getCountryLabelByValue(filterdAccounts[0].country), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + // 項目チェック（company_name） + if (accountFromFile.company_name !== filterdAccounts[0].company_name) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "company_name", + fileData: accountFromFile.company_name, + databaseData: filterdAccounts[0].company_name, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + } + return isNoError; +} diff --git a/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts b/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts index 96d5030..d73563a 100644 --- a/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts +++ b/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts @@ -163,4 +163,21 @@ export class AccountsRepositoryService { ); }); } + + /** + * アカウント情報を全件取得する + * @returns Account[] + */ + async getAllAccounts( + context: Context, + ): Promise { + return await this.dataSource.transaction(async (entityManager) => { + const accountsRepo = entityManager.getRepository(Account); + + const accouts = accountsRepo.find({ + comment: `${context.getTrackingId()}_${new Date().toUTCString()}`, + }); + return accouts; + }); + } } diff --git a/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts b/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts index df1c190..7a9f46d 100644 --- a/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts +++ b/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts @@ -166,4 +166,34 @@ export class LicensesRepositoryService { return {}; }); } + + /** + * ライセンス情報を全件取得する + * @returns License[] + */ + async getAllLicenses(context: Context): Promise { + return await this.dataSource.transaction(async (entityManager) => { + const licenseRepo = entityManager.getRepository(License); + + const licenses = licenseRepo.find({ + comment: `${context.getTrackingId()}_${new Date().toUTCString()}`, + }); + return licenses; + }); + } + + /** + * カードライセンス情報を全件取得する + * @returns CardLicense[] + */ + async getAllCardLicense(context: Context): Promise { + return await this.dataSource.transaction(async (entityManager) => { + const cardLicenseRepo = entityManager.getRepository(CardLicense); + + const cardLicenses = cardLicenseRepo.find({ + comment: `${context.getTrackingId()}_${new Date().toUTCString()}`, + }); + return cardLicenses; + }); + } } diff --git a/data_migration_tools/server/src/repositories/users/users.repository.service.ts b/data_migration_tools/server/src/repositories/users/users.repository.service.ts index ebb4dc3..d060de7 100644 --- a/data_migration_tools/server/src/repositories/users/users.repository.service.ts +++ b/data_migration_tools/server/src/repositories/users/users.repository.service.ts @@ -138,4 +138,20 @@ export class UsersRepositoryService { await deleteEntity(usersRepo, { id: userId }, this.isCommentOut, context); }); } + + /** + * ユーザー情報を全件取得する + * @returns User[] + */ + async getAllUsers(context: Context): Promise { + return await this.dataSource.transaction(async (entityManager) => { + const userRepo = entityManager.getRepository(User); + + const users = userRepo.find({ + comment: `${context.getTrackingId()}_${new Date().toUTCString()}`, + }); + return users; + }); + } } + From 6d56255a5a44040fe764b4a863efdf4d061adaa9 Mon Sep 17 00:00:00 2001 From: "maruyama.t" Date: Wed, 28 Feb 2024 09:04:36 +0000 Subject: [PATCH 04/12] =?UTF-8?q?Merged=20PR=20792:=20parent=5Faccount=5Fi?= =?UTF-8?q?d=E3=81=8C=E6=AD=A3=E3=81=97=E3=81=8F=E8=A8=AD=E5=AE=9A?= =?UTF-8?q?=E3=81=95=E3=82=8C=E3=81=AA=E3=81=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3804: parent_account_idが正しく設定されない](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3804) Map配列からaccountidをキーにdealerAccountIdを取る処理で、検索keyが逆になっていたため修正。 ## レビューポイント - とくになし ## 動作確認状況 - ローカルで確認（階層を付け替えたアカウントの親子階層が正しいことを確認） ## 補足 - 相談、参考資料などがあれば --- .../server/src/features/transfer/transfer.service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data_migration_tools/server/src/features/transfer/transfer.service.ts b/data_migration_tools/server/src/features/transfer/transfer.service.ts index cf8dd53..34102cb 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.service.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.service.ts @@ -247,8 +247,7 @@ export class TransferService { //　Countryのレコードは除外する if (account.type !== MIGRATION_TYPE.COUNTRY) { const dealerAccountId = - dealerRecords.get(account.dealerAccountId) ?? - account.dealerAccountId; + dealerRecords.get(account.accountId) ?? account.dealerAccountId; const type = this.getAccountType(account.type); const newAccount: AccountsFile = { accountId: account.accountId, From ce6e09a7d0dc3494bebdd5f360f7b609464cf418 Mon Sep 17 00:00:00 2001 From: "saito.k" Date: Thu, 29 Feb 2024 01:16:15 +0000 Subject: [PATCH 05/12] =?UTF-8?q?Merged=20PR=20791:=20=E3=82=BF=E3=82=B9?= =?UTF-8?q?=E3=82=AF=E4=B8=80=E8=A6=A7=E7=94=BB=E9=9D=A2=E3=81=AE=E5=8F=96?= =?UTF-8?q?=E5=BE=97=E4=BB=B6=E6=95=B0=E3=81=8C10=E4=BB=B6=E3=81=A8?= =?UTF-8?q?=E3=81=AA=E3=81=A3=E3=81=A6=E3=81=84=E3=82=8B=E3=83=90=E3=82=B0?= =?UTF-8?q?=E3=81=AE=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3815: タスク一覧画面の取得件数が10件となっているバグの対応](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3815) - OptionItemのソート順を変換処理の中で行うように修正 - OrderByでソートするのはfindメソッドの作り的に無理そうなので - 調査にも時間がかかるため ## レビューポイント - タスクの中にこのバグが発生した原因を記載し、なぜこの修正にしたのか記述したのでそちらを確認していただいて変なところがあれば指摘していただきたいです。 - 書いている内容がよくわからない場合は、ハドルでの説明をさせてください。 ## UIの変更 - Before/Afterのスクショなど - スクショ置き場 ## 動作確認状況 - ローカルで確認 ## 補足 - 相談、参考資料などがあれば --- .../src/features/tasks/types/convert.ts | 18 ++++++++++++------ .../tasks/tasks.repository.service.ts | 13 ------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/dictation_server/src/features/tasks/types/convert.ts b/dictation_server/src/features/tasks/types/convert.ts index 5353542..2ce6d99 100644 --- a/dictation_server/src/features/tasks/types/convert.ts +++ b/dictation_server/src/features/tasks/types/convert.ts @@ -79,12 +79,18 @@ const createTask = ( const createAudioOptionItems = ( optionItems: AudioOptionItemEntity[], ): AudioOptionItem[] => { - return optionItems.map((x) => { - return { - optionItemLabel: x.label, - optionItemValue: x.value, - }; - }); + // バグ 3786: [FB対応]タスク一覧画面のOptionItemがソート条件によって表示順がおかしくなる の対応 + // 並び順をID順に固定する + // 本来はRepository側でソートするべきだが、TYPEORMの仕様でソートすると取得件数が想定通りに取得できないため、ここでソートする + //　詳細は タスク 3815: タスク一覧画面の取得件数が10件となっているバグの対応 + return optionItems + .sort((a: AudioOptionItemEntity, b: AudioOptionItemEntity) => a.id - b.id) + .map((x) => { + return { + optionItemLabel: x.label, + optionItemValue: x.value, + }; + }); }; // Repository側のDTOからAssigneeオブジェクトを構築する diff --git a/dictation_server/src/repositories/tasks/tasks.repository.service.ts b/dictation_server/src/repositories/tasks/tasks.repository.service.ts index e2bbefa..a95a8ab 100644 --- a/dictation_server/src/repositories/tasks/tasks.repository.service.ts +++ b/dictation_server/src/repositories/tasks/tasks.repository.service.ts @@ -1462,91 +1462,78 @@ const makeOrder = ( priority: 'DESC', job_number: direction, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'STATUS': return { priority: 'DESC', status: direction, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'TRANSCRIPTION_FINISHED_DATE': return { priority: 'DESC', finished_at: direction, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'TRANSCRIPTION_STARTED_DATE': return { priority: 'DESC', started_at: direction, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'AUTHOR_ID': return { priority: 'DESC', file: { author_id: direction }, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'ENCRYPTION': return { priority: 'DESC', file: { is_encrypted: direction }, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'FILE_LENGTH': return { priority: 'DESC', file: { duration: direction }, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'FILE_NAME': return { priority: 'DESC', file: { file_name: direction }, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'FILE_SIZE': return { priority: 'DESC', file: { file_size: direction }, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'RECORDING_FINISHED_DATE': return { priority: 'DESC', file: { finished_at: direction }, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'RECORDING_STARTED_DATE': return { priority: 'DESC', file: { started_at: direction }, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'UPLOAD_DATE': return { priority: 'DESC', file: { uploaded_at: direction }, id: 'ASC', - option_items: { id: 'ASC' }, }; case 'WORK_TYPE': return { priority: 'DESC', file: { work_type_id: direction }, id: 'ASC', - option_items: { id: 'ASC' }, }; default: // switchのcase漏れが発生した場合に型エラーになるようにする From 9ca9b7a1445ebb829a850bb7815c1c1003359fc9 Mon Sep 17 00:00:00 2001 From: Kentaro Fukunaga Date: Thu, 29 Feb 2024 06:36:23 +0000 Subject: [PATCH 06/12] =?UTF-8?q?Merged=20PR=20790:=20Author=E3=81=AENotif?= =?UTF-8?q?ication=E3=83=95=E3=83=A9=E3=82=B0=E3=82=92=E8=A6=8B=E3=81=A6?= =?UTF-8?q?=E3=82=BF=E3=82=B9=E3=82=AF=E5=AE=8C=E4=BA=86=E3=83=A1=E3=83=BC?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E9=80=81=E4=BF=A1=E5=85=88=E3=82=92=E5=A4=89?= =?UTF-8?q?=E6=9B=B4=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3818: 対応](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3818) - AuthorのNotificationフラグがOFFのときには、Authorに対してタスク完了通知メールが送信されないよう修正しました。 ## レビューポイント - 動作確認項目に不足はないか？ ## 動作確認状況 - ローカルで確認しました - AuthorのNotificationON時にはメール宛先に入っており、OFF時には宛先から外れること - TypistはNotificationON/OFF関わらずメール宛先に入っていること --- dictation_server/src/features/tasks/tasks.service.ts | 5 +++-- dictation_server/src/gateways/sendgrid/sendgrid.service.ts | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dictation_server/src/features/tasks/tasks.service.ts b/dictation_server/src/features/tasks/tasks.service.ts index 7b7c127..898a8f9 100644 --- a/dictation_server/src/features/tasks/tasks.service.ts +++ b/dictation_server/src/features/tasks/tasks.service.ts @@ -435,7 +435,7 @@ export class TasksService { `author_id not found. audioFileId: ${audioFileId}. account_id: ${user.account_id}`, ); } - const { external_id: authorExternalId } = + const { external_id: authorExternalId, notification: authorNotification } = await this.usersRepository.findUserByAuthorId( context, task.file.author_id, @@ -454,6 +454,7 @@ export class TasksService { ]); // メール送信に必要な情報を取得 + // Author通知ON/OFF関わらずAuthor名は必要なため、情報の取得は行う const author = usersInfo.find((x) => x.id === authorExternalId); if (!author) { throw new Error(`author not found. id=${authorExternalId}`); @@ -488,7 +489,7 @@ export class TasksService { // メール送信 this.sendgridService.sendMailWithU117( context, - authorEmail, + authorNotification ? authorEmail : null, typistEmail, authorName, task.file.file_name.replace('.zip', ''), diff --git a/dictation_server/src/gateways/sendgrid/sendgrid.service.ts b/dictation_server/src/gateways/sendgrid/sendgrid.service.ts index 17fa39b..ac9a5df 100644 --- a/dictation_server/src/gateways/sendgrid/sendgrid.service.ts +++ b/dictation_server/src/gateways/sendgrid/sendgrid.service.ts @@ -875,7 +875,7 @@ export class SendGridService { */ async sendMailWithU117( context: Context, - authorEmail: string, + authorEmail: string | null, typistEmail: string, authorName: string, fileName: string, @@ -903,7 +903,7 @@ export class SendGridService { // メールを送信する await this.sendMail( context, - [authorEmail, typistEmail], + [authorEmail, typistEmail].filter((x): x is string => x !== null), // authorEmailがnullの場合は除外する [], this.mailFrom, subject, From 0ebd2ab17eadc50d53a51539b9557871dd802a09 Mon Sep 17 00:00:00 2001 From: "maruyama.t" Date: Thu, 29 Feb 2024 06:55:34 +0000 Subject: [PATCH 07/12] =?UTF-8?q?Merged=20PR=20793:=20account=E3=81=AB?= =?UTF-8?q?=E5=90=8D=E5=89=8D=E3=81=8C=E3=81=AA=E3=81=84=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E3=81=8C=E5=AD=98=E5=9C=A8=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3831: accountに名前がないデータが存在する](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3831) 移行元データのアカウントユーザーにfirtst_nameおよびlast_nameが存在しない行が存在しており、 変換ツール側で対応していなかったためAdminNameが空のアカウントユーザーを作成しようとして登録ツール側でエラーになってしまっていた。 →バックログに起票しOMDSさんに確認中 ■暫定対応 first_name\last_nameが存在しない場合はユーザーと同様にメールアドレスをAdminNameとするように修正。 ## レビューポイント - とくになし ## 動作確認状況 - ローカルで確認 ## 補足 - 相談、参考資料などがあれば --- .../server/src/features/transfer/transfer.service.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/data_migration_tools/server/src/features/transfer/transfer.service.ts b/data_migration_tools/server/src/features/transfer/transfer.service.ts index 34102cb..269f609 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.service.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.service.ts @@ -63,8 +63,13 @@ export class TransferService { (country) => country.label === line.country )?.value; // adminNameの変換(last_name + " "+ first_name) - const adminName = `${line.last_name} ${line.first_name}`; - + // もしline.last_nameとline.first_nameが存在しない場合、line.admin_mailをnameにする + let adminName = line.email; + if (line.last_name && line.first_name) { + adminName = `${line.last_name} ${line.first_name}`; + // スペースが前後に入っている場合があるのでTrimする + adminName = adminName.trim(); + } // ランダムパスワードの生成（データ登録ツール側で行うのでやらない） // common/password/password.tsのmakePasswordを使用 // const autoGeneratedPassword = makePassword(); From cad3a99f705ec51c07526386eb959a55f8836806 Mon Sep 17 00:00:00 2001 From: "maruyama.t" Date: Thu, 29 Feb 2024 12:50:17 +0000 Subject: [PATCH 08/12] =?UTF-8?q?Merged=20PR=20794:=20=E7=99=BB=E9=8C=B2?= =?UTF-8?q?=E3=83=84=E3=83=BC=E3=83=AB=E3=81=AB=E3=83=AD=E3=82=B0=E3=82=92?= =?UTF-8?q?=E4=BB=95=E8=BE=BC=E3=82=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3839: 登録ツールにログを仕込む](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3839) 登録ツールが途中で動かなくなってしまう原因調査のために各関数にログを仕込みました。 ## レビューポイント - 特になし ## 動作確認状況 - ローカルで確認 ## 補足 - 相談、参考資料などがあれば --- .../server/src/common/password/password.ts | 8 +++++++- .../server/src/features/accounts/accounts.service.ts | 3 +++ .../server/src/features/register/register.controller.ts | 3 +++ .../server/src/features/users/users.service.ts | 6 +++++- .../server/src/gateways/adb2c/adb2c.service.ts | 3 +++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/data_migration_tools/server/src/common/password/password.ts b/data_migration_tools/server/src/common/password/password.ts index 6fbe071..15d52ce 100644 --- a/data_migration_tools/server/src/common/password/password.ts +++ b/data_migration_tools/server/src/common/password/password.ts @@ -18,7 +18,8 @@ export const makePassword = (): string => { let autoGeneratedPassword: string = ""; while (!valid) { - // パスワードをランダムに決定 + autoGeneratedPassword = ""; + // パスワードをランダムに決定+ while (autoGeneratedPassword.length < passLength) { // 上で決定したcharsの中からランダムに1文字ずつ追加 const index = Math.floor(Math.random() * chars.length); @@ -30,6 +31,11 @@ export const makePassword = (): string => { valid = autoGeneratedPassword.length == passLength && charaTypePattern.test(autoGeneratedPassword); + if (!valid) { + // autoGeneratedPasswordをログに出す + console.log("Password is not valid"); + console.log(autoGeneratedPassword); + } } return autoGeneratedPassword; }; diff --git a/data_migration_tools/server/src/features/accounts/accounts.service.ts b/data_migration_tools/server/src/features/accounts/accounts.service.ts index 2954cbd..fdfb458 100644 --- a/data_migration_tools/server/src/features/accounts/accounts.service.ts +++ b/data_migration_tools/server/src/features/accounts/accounts.service.ts @@ -79,6 +79,7 @@ export class AccountsService { HttpStatus.INTERNAL_SERVER_ERROR ); } + this.logger.log("idpにユーザーを作成成功"); // メールアドレス重複エラー if (isConflictError(externalUser)) { @@ -90,6 +91,7 @@ export class AccountsService { HttpStatus.BAD_REQUEST ); } + this.logger.log("メールアドレスは重複していません"); let account: Account; let user: User; @@ -138,6 +140,7 @@ export class AccountsService { account.id, country ); + this.logger.log("コンテナー作成成功"); } catch (e) { this.logger.error(`[${context.getTrackingId()}] error=${e}`); this.logger.error( diff --git a/data_migration_tools/server/src/features/register/register.controller.ts b/data_migration_tools/server/src/features/register/register.controller.ts index cb36d2a..d9b968f 100644 --- a/data_migration_tools/server/src/features/register/register.controller.ts +++ b/data_migration_tools/server/src/features/register/register.controller.ts @@ -106,8 +106,10 @@ export class RegisterController { } for (const AccountsFile of accountsObject) { + this.logger.log("ランダムパスワード生成開始"); // ランダムなパスワードを生成する const ramdomPassword = makePassword(); + this.logger.log("ランダムパスワード生成完了"); // roleの設定 // roleの値がnullなら"none"、null以外ならroleの値、 // また、roleの値が"author"なら"author"を設定 @@ -123,6 +125,7 @@ export class RegisterController { // ありえないが、roleの値が"none"または"author"の文字列以外の場合はエラーを返す throw new Error("Invalid role value"); } + this.logger.log("account生成開始"); await this.accountsService.createAccount( context, AccountsFile.companyName, diff --git a/data_migration_tools/server/src/features/users/users.service.ts b/data_migration_tools/server/src/features/users/users.service.ts index cb639cd..8134462 100644 --- a/data_migration_tools/server/src/features/users/users.service.ts +++ b/data_migration_tools/server/src/features/users/users.service.ts @@ -74,6 +74,9 @@ export class UsersService { accountId, authorId ); + this.logger.log( + `[${context.getTrackingId()}] isAuthorIdDuplicated=${isAuthorIdDuplicated}` + ); } catch (e) { this.logger.error(`[${context.getTrackingId()}] error=${e}`); throw new HttpException( @@ -88,9 +91,10 @@ export class UsersService { ); } } - + this.logger.log("ランダムパスワード生成開始"); // ランダムなパスワードを生成する const ramdomPassword = makePassword(); + this.logger.log("ランダムパスワード生成完了"); //Azure AD B2Cにユーザーを新規登録する let externalUser: { sub: string } | ConflictError; diff --git a/data_migration_tools/server/src/gateways/adb2c/adb2c.service.ts b/data_migration_tools/server/src/gateways/adb2c/adb2c.service.ts index 5ba0f0e..5dbe646 100644 --- a/data_migration_tools/server/src/gateways/adb2c/adb2c.service.ts +++ b/data_migration_tools/server/src/gateways/adb2c/adb2c.service.ts @@ -82,6 +82,9 @@ export class AdB2cService { }, ], }); + this.logger.log( + `[${context.getTrackingId()}] [ADB2C CREATE] newUser: ${newUser}` + ); return { sub: newUser.id }; } catch (e) { this.logger.error(`[${context.getTrackingId()}] error=${e}`); From 88ce6a2c9e399e92c622ecefea1cf45cecddeab2 Mon Sep 17 00:00:00 2001 From: masaaki Date: Fri, 1 Mar 2024 12:00:42 +0000 Subject: [PATCH 09/12] =?UTF-8?q?Merged=20PR=20796:=20[3=E5=9B=9E=E7=9B=AE?= =?UTF-8?q?=E5=AE=9F=E8=A1=8C][=E3=83=95=E3=83=AB=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF]develop=E7=92=B0=E5=A2=83=E3=81=A7=E3=81=AE=E7=A7=BB?= =?UTF-8?q?=E8=A1=8C=E5=AE=9F=E6=96=BD=E5=BE=8C=E3=81=AE=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E4=BD=9C=E6=A5=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3802: [3回目実行][フルデータ]develop環境での移行実施後の修正作業](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3802) - 元PBI or タスクへのリンク（内容・目的などはそちらにあるはず） - 何をどう変更したか、追加したライブラリなど - このPull Requestでの対象/対象外 - 影響範囲（他の機能にも影響があるか） ## レビューポイント - 特にレビューしてほしい箇所 - 軽微なものや自明なものは記載不要 - 修正範囲が大きい場合などに記載 - 全体的にや仕様を満たしているか等は本当に必要な時のみ記載 ## UIの変更 - Before/Afterのスクショなど - スクショ置き場 ## 動作確認状況 - ローカルで確認、develop環境で確認など ## 補足 - 相談、参考資料などがあれば --- .../features/transfer/transfer.controller.ts | 105 ++++++------ .../src/features/transfer/transfer.service.ts | 160 +++++++++++++----- .../verification/verification.service.ts | 53 ++++-- 3 files changed, 221 insertions(+), 97 deletions(-) diff --git a/data_migration_tools/server/src/features/transfer/transfer.controller.ts b/data_migration_tools/server/src/features/transfer/transfer.controller.ts index ee38969..507ed3f 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.controller.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.controller.ts @@ -73,7 +73,8 @@ export class TransferController { const matchList = line.match(regExp); if (matchList) { matchList.forEach((match) => { - const replaced = match.replace(/,/g, " "); + // カンマを\に変換 + const replaced = match.replace(/,/g, "\\"); line = line.replace(match, replaced); }); } @@ -95,49 +96,50 @@ export class TransferController { HttpStatus.BAD_REQUEST ); } - - csvInputFile.push({ - type: data[0], - account_id: data[1], - parent_id: data[2], - email: data[3], - company_name: data[4], - first_name: data[5], - last_name: data[6], - country: data[7], - state: data[8], - start_date: data[9], - expired_date: data[10], - user_email: data[11], - author_id: data[12], - recording_mode: data[13], - wt1: data[14], - wt2: data[15], - wt3: data[16], - wt4: data[17], - wt5: data[18], - wt6: data[19], - wt7: data[20], - wt8: data[21], - wt9: data[22], - wt10: data[23], - wt11: data[24], - wt12: data[25], - wt13: data[26], - wt14: data[27], - wt15: data[28], - wt16: data[29], - wt17: data[30], - wt18: data[31], - wt19: data[32], - wt20: data[33], - }); + //　data[1]がundefinedの場合、配列には格納しない + if (data[1] !== undefined) { + // バックスラッシュをカンマに戻す + data.forEach((value, index) => { + data[index] = value.replace(/\\/g, ","); + }); + csvInputFile.push({ + type: data[0], + account_id: data[1], + parent_id: data[2], + email: data[3], + company_name: data[4], + first_name: data[5], + last_name: data[6], + country: data[7], + state: data[8], + start_date: data[9], + expired_date: data[10], + user_email: data[11], + author_id: data[12], + recording_mode: data[13], + wt1: data[14], + wt2: data[15], + wt3: data[16], + wt4: data[17], + wt5: data[18], + wt6: data[19], + wt7: data[20], + wt8: data[21], + wt9: data[22], + wt10: data[23], + wt11: data[24], + wt12: data[25], + wt13: data[26], + wt14: data[27], + wt15: data[28], + wt16: data[29], + wt17: data[30], + wt18: data[31], + wt19: data[32], + wt20: data[33], + }); + } }); - // 最後の行がundefinedの場合はその行を削除 - if (csvInputFile[csvInputFile.length - 1].account_id === undefined) { - csvInputFile.pop(); - } - // 各データのバリデーションチェック await this.transferService.validateInputData(context, csvInputFile); @@ -153,12 +155,12 @@ export class TransferController { // アカウントID numberとstring対応表の出力 const accountsMappingFiles: AccountsMappingFile[] = []; accountIdMap.forEach((value, key) => { - const accountsMappingFile = new AccountsMappingFile(); + const accountsMappingFile = new AccountsMappingFile(); accountsMappingFile.accountIdNumber = value; - accountsMappingFile.accountIdText = key - accountsMappingFiles.push(accountsMappingFile) + accountsMappingFile.accountIdText = key; + accountsMappingFiles.push(accountsMappingFile); }); - + fs.writeFileSync( `${inputFilePath}account_map.json`, JSON.stringify(accountsMappingFiles) @@ -188,6 +190,13 @@ export class TransferController { LicensesFile ); + // AuthorIDが重複している場合通番を付与する + const transferDuplicateAuthorResultUsers = + await this.transferService.transferDuplicateAuthor( + context, + resultDuplicateEmail.usersFileLines + ); + // transferResponseCsvを４つのJSONファイルの出力する(出力先はinputと同じにする) const outputFilePath = body.inputFilePath; const WorktypesFile = transferResponseCsv.worktypesFileLines; @@ -195,7 +204,7 @@ export class TransferController { context, outputFilePath, resultDuplicateEmail.accountsFileLines, - resultDuplicateEmail.usersFileLines, + transferDuplicateAuthorResultUsers, resultDuplicateEmail.licensesFileLines, WorktypesFile ); diff --git a/data_migration_tools/server/src/features/transfer/transfer.service.ts b/data_migration_tools/server/src/features/transfer/transfer.service.ts index 269f609..60b8d5c 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.service.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.service.ts @@ -54,6 +54,12 @@ export class TransferService { let userIdIndex = 0; // authorIdとuserIdの対応関係を保持するMapを定義 const authorIdToUserIdMap: Map = new Map(); + + // countryのリストを生成 + const countryAccounts = csvInputFile.filter( + (item) => item.type === "Country" + ); + // csvInputFileを一行読み込みする csvInputFile.forEach((line) => { // typeが"USER"以外の場合、アカウントデータの作成を行う @@ -105,8 +111,13 @@ export class TransferService { authorId: null, }); } else { - // typeが"USER"の場合 - if (line.type == MIGRATION_TYPE.USER) { + // typeが"USER"の場合、かつcountryのアカウントIDに所属していない場合 + if ( + line.type == MIGRATION_TYPE.USER && + !countryAccounts.some( + (countryAccount) => countryAccount.account_id === line.account_id + ) + ) { // line.author_idが存在する場合のみユーザーデータを作成する if (line.author_id) { // userIdIndexをインクリメントする @@ -229,46 +240,38 @@ export class TransferService { const relocatedAccounts: AccountsFile[] = []; const dealerRecords: Map = new Map(); - // accountsFileTypeをループ - accountsFileType.forEach((account) => { - // Distributorの場合はdealerを検索し、COUNTRYかチェックする - if (account.type === MIGRATION_TYPE.DISTRIBUTOR) { - const distributorParent = accountsFileType.find( - (a) => a.accountId === account.dealerAccountId - ); - if (distributorParent.type === MIGRATION_TYPE.COUNTRY) { - dealerRecords.set( - account.accountId, - distributorParent.dealerAccountId // Countryの親、BCのIDを設定 - ); + const countryAccounts = accountsFileType.filter( + (item) => item.type === MIGRATION_TYPE.COUNTRY + ); + + const notCountryAccounts = accountsFileType.filter( + (item) => item.type !== MIGRATION_TYPE.COUNTRY + ); + + notCountryAccounts.forEach((notCountryAccount) => { + let assignDealerAccountId = notCountryAccount.dealerAccountId; + // 親アカウントIDがcountryの場合、countryの親アカウントIDを設定する + for (const countryAccount of countryAccounts) { + if (countryAccount.accountId === notCountryAccount.dealerAccountId) { + assignDealerAccountId = countryAccount.dealerAccountId; } - } else { - dealerRecords.set(account.accountId, account.dealerAccountId); } - }); - // AccountsFileTypeのループを行い、階層情報の置換と新たな配列へのpushを行う - accountsFileType.forEach((account) => { - //　Countryのレコードは除外する - if (account.type !== MIGRATION_TYPE.COUNTRY) { - const dealerAccountId = - dealerRecords.get(account.accountId) ?? account.dealerAccountId; - const type = this.getAccountType(account.type); - const newAccount: AccountsFile = { - accountId: account.accountId, - type: type, - companyName: account.companyName, - country: account.country, - dealerAccountId: dealerAccountId, - adminName: account.adminName, - adminMail: account.adminMail, - userId: account.userId, - role: account.role, - authorId: account.authorId, - }; + const assignType = this.getAccountType(notCountryAccount.type); - relocatedAccounts.push(newAccount); - } + const newAccount: AccountsFile = { + accountId: notCountryAccount.accountId, + type: assignType, + companyName: notCountryAccount.companyName, + country: notCountryAccount.country, + dealerAccountId: assignDealerAccountId, + adminName: notCountryAccount.adminName, + adminMail: notCountryAccount.adminMail, + userId: notCountryAccount.userId, + role: notCountryAccount.role, + authorId: notCountryAccount.authorId, + }; + relocatedAccounts.push(newAccount); }); return relocatedAccounts; @@ -364,6 +367,8 @@ export class TransferService { ); try { + // エラー配列を定義 + let errorArray: string[] = []; // アカウントに対するworktypeのMap配列を作成する const accountWorktypeMap = new Map(); // csvInputFileのバリデーションチェックを行う @@ -383,6 +388,13 @@ export class TransferService { HttpStatus.BAD_REQUEST ); } + // typeがUSER以外の場合で、countryがnullの場合エラー配列に格納する + if (line.type !== MIGRATION_TYPE.USER) { + if (!line.country) { + // countryがnullの場合エラー配列に格納する + errorArray.push(`country is null. index=${index}`); + } + } // countryのバリデーションチェック if (line.country) { if (!COUNTRY_LIST.find((country) => country.label === line.country)) { @@ -451,6 +463,15 @@ export class TransferService { } } }); + // エラー配列に値が存在する場合はエラーファイルを出力する + if (errorArray.length > 0) { + const errorFileJson = JSON.stringify(errorArray); + fs.writeFileSync(`error.json`, errorFileJson); + throw new HttpException( + `errorArray is invalid. errorArray=${errorArray}`, + HttpStatus.BAD_REQUEST + ); + } } catch (e) { this.logger.error(`[${context.getTrackingId()}] error=${e}`); throw new HttpException( @@ -603,4 +624,67 @@ export class TransferService { ); } } + + /** + * transferDuplicateAuthor + * @param usersFileLines: UsersFile[] + * @returns UsersFile[] + */ + async transferDuplicateAuthor( + context: Context, + usersFileLines: UsersFile[] + ): Promise { + // パラメータ内容が長大なのでログには出さない + this.logger.log( + `[IN] [${context.getTrackingId()}] ${this.transferDuplicateAuthor.name}` + ); + + try { + const newUsersFileLines: UsersFile[] = []; + + let processingAccountId: number = 0; //処理中のアカウントID + let duplicateSequence: number = 2; + let authorIdList: String[] = []; + for (const user of usersFileLines) { + if (user.accountId !== processingAccountId) { + //アカウントIDが別になった場合、通番を初期化する + duplicateSequence = 2; + processingAccountId = user.accountId; + authorIdList = []; + } + let assignAuthorId = user.authorId; + if (authorIdList.includes(user.authorId)) { + // 同じauthorIdがいる場合、自分のauthorIdに連番を付与する + assignAuthorId = assignAuthorId + duplicateSequence; + duplicateSequence = duplicateSequence + 1; + } + authorIdList.push(user.authorId); + + // 新しいAuthorIdのユーザに詰め替え + const newUser: UsersFile = { + accountId: user.accountId, + userId: user.userId, + name: user.name, + role: user.role, + authorId: assignAuthorId, + email: user.email, + }; + newUsersFileLines.push(newUser); + } + + return newUsersFileLines; + } catch (e) { + this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); + } finally { + this.logger.log( + `[OUT] [${context.getTrackingId()}] ${ + this.transferDuplicateAuthor.name + }` + ); + } + } } diff --git a/data_migration_tools/server/src/features/verification/verification.service.ts b/data_migration_tools/server/src/features/verification/verification.service.ts index 1de9596..4fda7d7 100644 --- a/data_migration_tools/server/src/features/verification/verification.service.ts +++ b/data_migration_tools/server/src/features/verification/verification.service.ts @@ -53,9 +53,11 @@ export class VerificationService { // 件数情報の取得 this.logger.log(`入力ファイルから件数情報を取得する`); - const accountCountFromFile = csvInputFiles.filter( + const accountFromFile = csvInputFiles.filter( (item) => item.type !== "USER" && item.type !== "Country" - ).length; + ); + const accountCountFromFile = accountFromFile.length; + const cardLicensesCountFromFile = cardlicensesInputFiles.length; const licensesCountFromFile = @@ -66,18 +68,35 @@ export class VerificationService { // 管理ユーザ数のカウント const administratorCountFromFile = accountCountFromFile; + // 一般ユーザ数のカウント - const normaluserCountFromFile = csvInputFiles.filter( - (item) => item.type === "USER" && item.user_email.length !== 0 - ).length; + // countryのアカウントに所属するユーザをカウント対象外とする + const countryAccountFromFile = csvInputFiles.filter( + (item) => item.type === "Country" + ); + + // USER、かつuser_emailが設定なし、かつcountryのアカウントID以外をユーザとする + const normaluserFromFile = csvInputFiles.filter( + (item) => + item.type === "USER" && + item.user_email.length !== 0 && + !countryAccountFromFile.some( + (countryItem) => countryItem.account_id === item.account_id + ) + ); + + const normaluserCountFromFile = normaluserFromFile.length; // ユーザ重複数のカウント let mailAdresses: string[] = []; - csvInputFiles.forEach((item) => { - // メールアドレスの要素を配列に追加（入力データとして管理者とユーザの両方に入ることはない） + accountFromFile.forEach((item) => { + // メールアドレスの要素を配列に追加 if (item.email.length !== 0) { mailAdresses.push(item.email); } + }); + normaluserFromFile.forEach((item) => { + // メールアドレスの要素を配列に追加 if (item.user_email.length !== 0) { mailAdresses.push(item.user_email); } @@ -232,7 +251,11 @@ export class VerificationService { } // dateを任意のフォーマットに変換する -const getFormattedDate = (date: Date | null, format: string) => { +const getFormattedDate = ( + date: Date | null, + format: string, + padHours: boolean = false // trueの場合、hhについてゼロパディングする（00→0、01→1、23→23） +) => { if (!date) { return null; } @@ -244,9 +267,13 @@ const getFormattedDate = (date: Date | null, format: string) => { s: date.getSeconds(), }; + // hhの値をゼロパディングするかどうかのフラグを確認 + const hourSymbol = padHours ? "hh" : "h"; + const formatted = format.replace(/(M+|d+|h+|m+|s+)/g, (v) => ( - (v.length > 1 ? "0" : "") + symbol[v.slice(-1) as keyof typeof symbol] + (v.length > 1 && v !== hourSymbol ? "0" : "") + + symbol[v.slice(-1) as keyof typeof symbol] ).slice(-2) ); @@ -542,12 +569,15 @@ function compareLicenses( VerificationResultDetails.push(VerificationResultDetailsOne); isNoError = false; } + + // expiry_dateについて、時はゼロパディングした値で比較する（×01～09 ○1～9） if ( !licensesFromDatabase[i] || licensesFromFile[i].expired_date !== getFormattedDate( licensesFromDatabase[i].expiry_date, - `yyyy/MM/dd hh:mm:ss` + `yyyy/MM/dd hh:mm:ss`, + true ) ) { const VerificationResultDetailsOne: VerificationResultDetails = { @@ -559,7 +589,8 @@ function compareLicenses( databaseData: licensesFromDatabase[i] ? getFormattedDate( licensesFromDatabase[i].expiry_date, - `yyyy/MM/dd hh:mm:ss` + `yyyy/MM/dd hh:mm:ss`, + true ) : "undifined", reason: "内容不一致", From a47ebaa9dfb668bf44c1f5308eb1d37f503976ba Mon Sep 17 00:00:00 2001 From: masaaki Date: Sat, 2 Mar 2024 02:21:19 +0000 Subject: [PATCH 10/12] =?UTF-8?q?Merged=20PR=20798:=20[4=E5=9B=9E=E7=9B=AE?= =?UTF-8?q?=E5=AE=9F=E8=A1=8C][=E3=83=95=E3=83=AB=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF]develop=E7=92=B0=E5=A2=83=E3=81=A7=E3=81=AE=E7=A7=BB?= =?UTF-8?q?=E8=A1=8C=E5=AE=9F=E6=96=BD=E5=BE=8C=E3=81=AE=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E4=BD=9C=E6=A5=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3821: [4回目実行][フルデータ]develop環境での移行実施後の修正作業](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3821) - 元PBI or タスクへのリンク（内容・目的などはそちらにあるはず） - 移行ツールに対して以下の修正を実施しました。 - アカウントとユーザ間でAuthorIDが重複する際、通番を付与して重複を避けるようにしました - AADB2Cのエラー発生時、リトライ処理を行うように対応しました ## レビューポイント - 特にありません ## UIの変更 - 無し ## 動作確認状況 - ローカルで確認 ## 補足 - 相談、参考資料などがあれば --- .../features/transfer/transfer.controller.ts | 1 + .../src/features/transfer/transfer.service.ts | 61 +++++++------- .../src/gateways/adb2c/adb2c.service.ts | 81 +++++++++++-------- 3 files changed, 82 insertions(+), 61 deletions(-) diff --git a/data_migration_tools/server/src/features/transfer/transfer.controller.ts b/data_migration_tools/server/src/features/transfer/transfer.controller.ts index 507ed3f..c3c2dce 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.controller.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.controller.ts @@ -194,6 +194,7 @@ export class TransferController { const transferDuplicateAuthorResultUsers = await this.transferService.transferDuplicateAuthor( context, + resultDuplicateEmail.accountsFileLines, resultDuplicateEmail.usersFileLines ); diff --git a/data_migration_tools/server/src/features/transfer/transfer.service.ts b/data_migration_tools/server/src/features/transfer/transfer.service.ts index 60b8d5c..8168c04 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.service.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.service.ts @@ -627,11 +627,13 @@ export class TransferService { /** * transferDuplicateAuthor + * @param accountsFileLines: AccountsFile[] * @param usersFileLines: UsersFile[] * @returns UsersFile[] */ async transferDuplicateAuthor( context: Context, + accountsFileLines: AccountsFile[], usersFileLines: UsersFile[] ): Promise { // パラメータ内容が長大なのでログには出さない @@ -642,34 +644,39 @@ export class TransferService { try { const newUsersFileLines: UsersFile[] = []; - let processingAccountId: number = 0; //処理中のアカウントID - let duplicateSequence: number = 2; - let authorIdList: String[] = []; - for (const user of usersFileLines) { - if (user.accountId !== processingAccountId) { - //アカウントIDが別になった場合、通番を初期化する - duplicateSequence = 2; - processingAccountId = user.accountId; - authorIdList = []; - } - let assignAuthorId = user.authorId; - if (authorIdList.includes(user.authorId)) { - // 同じauthorIdがいる場合、自分のauthorIdに連番を付与する - assignAuthorId = assignAuthorId + duplicateSequence; - duplicateSequence = duplicateSequence + 1; - } - authorIdList.push(user.authorId); + for (const accountsFileLine of accountsFileLines) { + let duplicateSequence: number = 2; + let authorIdList: String[] = []; - // 新しいAuthorIdのユーザに詰め替え - const newUser: UsersFile = { - accountId: user.accountId, - userId: user.userId, - name: user.name, - role: user.role, - authorId: assignAuthorId, - email: user.email, - }; - newUsersFileLines.push(newUser); + // メールアドレス重複時はアカウントにもAuthorIdが設定されるので重複チェック用のリストに追加しておく + if (accountsFileLine.authorId) { + authorIdList.push(accountsFileLine.authorId); + } + + const targetaccountUsers = usersFileLines.filter( + (item) => item.accountId === accountsFileLine.accountId + ); + + for (const targetaccountUser of targetaccountUsers) { + let assignAuthorId = targetaccountUser.authorId; + if (authorIdList.includes(targetaccountUser.authorId)) { + // 同じauthorIdがいる場合、自分のauthorIdに連番を付与する + assignAuthorId = assignAuthorId + duplicateSequence; + duplicateSequence = duplicateSequence + 1; + } + authorIdList.push(targetaccountUser.authorId); + + // 新しいAuthorIdのユーザに詰め替え + const newUser: UsersFile = { + accountId: targetaccountUser.accountId, + userId: targetaccountUser.userId, + name: targetaccountUser.name, + role: targetaccountUser.role, + authorId: assignAuthorId, + email: targetaccountUser.email, + }; + newUsersFileLines.push(newUser); + } } return newUsersFileLines; diff --git a/data_migration_tools/server/src/gateways/adb2c/adb2c.service.ts b/data_migration_tools/server/src/gateways/adb2c/adb2c.service.ts index 5dbe646..afb2ab2 100644 --- a/data_migration_tools/server/src/gateways/adb2c/adb2c.service.ts +++ b/data_migration_tools/server/src/gateways/adb2c/adb2c.service.ts @@ -64,44 +64,57 @@ export class AdB2cService { this.logger.log( `[IN] [${context.getTrackingId()}] ${this.createUser.name}` ); - try { - // ユーザをADB2Cに登録 - const newUser = await this.graphClient.api("users/").post({ - accountEnabled: true, - displayName: username, - passwordPolicies: "DisableStrongPassword", - passwordProfile: { - forceChangePasswordNextSignIn: false, - password: password, - }, - identities: [ - { - signinType: ADB2C_SIGN_IN_TYPE.EMAILADDRESS, - issuer: `${this.tenantName}.onmicrosoft.com`, - issuerAssignedId: email, + + const retryCount: number = 3; + let retry = 0; + + while (retry < retryCount) { + try { + // ユーザをADB2Cに登録 + const newUser = await this.graphClient.api("users/").post({ + accountEnabled: true, + displayName: username, + passwordPolicies: "DisableStrongPassword", + passwordProfile: { + forceChangePasswordNextSignIn: false, + password: password, }, - ], - }); - this.logger.log( - `[${context.getTrackingId()}] [ADB2C CREATE] newUser: ${newUser}` - ); - return { sub: newUser.id }; - } catch (e) { - this.logger.error(`[${context.getTrackingId()}] error=${e}`); - if (e?.statusCode === 400 && e?.body) { - const error = JSON.parse(e.body); + identities: [ + { + signinType: ADB2C_SIGN_IN_TYPE.EMAILADDRESS, + issuer: `${this.tenantName}.onmicrosoft.com`, + issuerAssignedId: email, + }, + ], + }); + this.logger.log( + `[${context.getTrackingId()}] [ADB2C CREATE] newUser: ${newUser}` + ); + return { sub: newUser.id }; + } catch (e) { + this.logger.error(`[${context.getTrackingId()}] error=${e}`); + + if (e?.statusCode === 400 && e?.body) { + const error = JSON.parse(e.body); - // エラーが競合エラーである場合は、メールアドレス重複としてエラーを返す - if (error?.details?.find((x) => x.code === "ObjectConflict")) { - return { reason: "email", message: "ObjectConflict" }; + // エラーが競合エラーである場合は、メールアドレス重複としてエラーを返す + if (error?.details?.find((x) => x.code === "ObjectConflict")) { + return { reason: "email", message: "ObjectConflict" }; + } } - } - throw e; - } finally { - this.logger.log( - `[OUT] [${context.getTrackingId()}] ${this.createUser.name}` - ); + if (++retry < retryCount) { + this.logger.log(`ADB2Cエラー発生。5秒sleepしてリトライします (${retry}/${retryCount})...`); + await new Promise(resolve => setTimeout(resolve, 5000)); + } else { + this.logger.log(`リトライ数が上限に達したのでエラーを返却します`); + throw e; + } + } finally { + this.logger.log( + `[OUT] [${context.getTrackingId()}] ${this.createUser.name}` + ); + } } } From 2220e2560fa02d96e18426bd745c5003dfdbed8c Mon Sep 17 00:00:00 2001 From: masaaki Date: Wed, 6 Mar 2024 01:19:23 +0000 Subject: [PATCH 11/12] =?UTF-8?q?Merged=20PR=20799:=20makepassword?= =?UTF-8?q?=E3=81=A7=E6=9D=A1=E4=BB=B6=E3=81=AB=E5=90=88=E8=87=B4=E3=81=97?= =?UTF-8?q?=E3=81=AA=E3=81=84=E3=83=91=E3=82=B9=E3=83=AF=E3=83=BC=E3=83=89?= =?UTF-8?q?=E3=82=92=E7=94=9F=E6=88=90=E3=81=97=E3=81=9F=E9=9A=9B=E7=84=A1?= =?UTF-8?q?=E9=99=90=E3=83=AB=E3=83=BC=E3=83=97=E3=81=AB=E3=81=AA=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3840: makepasswordで条件に合致しないパスワードを生成した際無限ループになる](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3840) - makepasswordで条件に合致しないパスワードを生成した場合、再度生成するループ処理としているが、初期化が行われていないため常に同じパスワードで条件合致のチェックが行われていました。結果、一度条件に合致しないパスワードを生成した場合無限ループとなっていました。 - ループ内で変数を初期化するよう対応。 ## レビューポイント - 特にありません ## UIの変更 - 無し ## 動作確認状況 - ユニットテストが通ることを確認 - ローカルでユーザー作成を実施し、これまで同様作成できることを確認 ## 補足 - 相談、参考資料などがあれば --- dictation_server/src/common/password/password.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dictation_server/src/common/password/password.ts b/dictation_server/src/common/password/password.ts index f68bb3c..72950b8 100644 --- a/dictation_server/src/common/password/password.ts +++ b/dictation_server/src/common/password/password.ts @@ -15,9 +15,12 @@ export const makePassword = (): string => { // autoGeneratedPasswordが以上の条件を満たせばvalidがtrueになる let valid = false; - let autoGeneratedPassword: string = ''; + let autoGeneratedPassword = ''; while (!valid) { + // 再生成用に変数を初期化する + autoGeneratedPassword = ''; + // パスワードをランダムに決定 while (autoGeneratedPassword.length < passLength) { // 上で決定したcharsの中からランダムに1文字ずつ追加 From 7160e0ee2e9cf02d6555d9d211b19690ca8ba96e Mon Sep 17 00:00:00 2001 From: "saito.k" Date: Wed, 6 Mar 2024 01:31:10 +0000 Subject: [PATCH 12/12] =?UTF-8?q?Merged=20PR=20804:=20=E3=83=87=E3=82=B0?= =?UTF-8?q?=E3=83=AC=E5=86=8D=E7=99=BA=E9=98=B2=E6=AD=A2=E3=81=AE=E3=81=9F?= =?UTF-8?q?=E3=82=81=E3=80=81=E4=BF=AE=E6=AD=A3=E3=82=92=E3=83=81=E3=82=A7?= =?UTF-8?q?=E3=83=83=E3=82=AF=E3=81=99=E3=82=8B=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=92=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 概要 [Task3830: デグレ再発防止のため、修正をチェックするテストを作成](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3830) - タスクを100件取得できることを確認するテストを追加 ## レビューポイント - テストでかくにんする項目は足りているか ## UIの変更 - 特になし ## クエリの変更 - 特になし ## 動作確認状況 - ローカルでテストが通ることを確認 ## 補足 - 相談、参考資料などがあれば --- .../src/features/tasks/tasks.service.spec.ts | 66 +++++++++++++++++++ .../src/features/tasks/tasks.service.ts | 14 ++-- .../src/gateways/sendgrid/sendgrid.service.ts | 2 +- 3 files changed, 75 insertions(+), 7 deletions(-) diff --git a/dictation_server/src/features/tasks/tasks.service.spec.ts b/dictation_server/src/features/tasks/tasks.service.spec.ts index 1531077..97f4ee2 100644 --- a/dictation_server/src/features/tasks/tasks.service.spec.ts +++ b/dictation_server/src/features/tasks/tasks.service.spec.ts @@ -946,6 +946,72 @@ describe('TasksService', () => { expect(task.jobNumber).toEqual('00000001'); } }); + it('[Admin] Taskが100件であっても取得できる', async () => { + const notificationhubServiceMockValue = + makeDefaultNotificationhubServiceMockValue(); + if (!source) fail(); + const module = await makeTaskTestingModuleWithNotificaiton( + source, + notificationhubServiceMockValue, + ); + if (!module) fail(); + const { id: accountId } = await makeTestSimpleAccount(source); + const { external_id } = await makeTestUser(source, { + account_id: accountId, + external_id: 'userId', + role: 'none', + }); + const { id: authorUserId, author_id } = await makeTestUser(source, { + account_id: accountId, + external_id: 'userId', + author_id: 'MY_AUTHOR_ID', + role: 'author', + }); + + const service = module.get(TasksService); + for (let i = 0; i < 100; i++) { + await createTask( + source, + accountId, + authorUserId, + author_id ?? '', + `WORKTYPE${i + 1}`, + '01', + // 00000001 ~ 00000100 + `000000${String(i + 1).padStart(2, '0')}`, + 'Uploaded', + ); + } + const offset = 0; + const limit = 100; + const status = ['Uploaded', 'Backup']; + const paramName = 'WORK_TYPE'; + const direction = 'DESC'; + + const { tasks, total } = await service.getTasks( + makeContext('trackingId', 'requestId'), + external_id, + [ADMIN_ROLES.ADMIN, USER_ROLES.NONE], + offset, + limit, + status, + paramName, + direction, + ); + expect(tasks.length).toEqual(100); + expect(total).toEqual(100); + // ソート条件がWORK_TYPEのため、WORK_TYPEが降順になっていることを確認 + expect(tasks[0].workType).toEqual('WORKTYPE99'); + expect(tasks[99].workType).toEqual('WORKTYPE1'); + expect(tasks[0].optionItemList).toEqual( + Array.from({ length: 10 }).map((_, i) => { + return { + optionItemLabel: `label${i}:audio_file_id${tasks[0].audioFileId}`, + optionItemValue: `value${i}:audio_file_id${tasks[0].audioFileId}`, + }; + }), + ); + }); }); }); diff --git a/dictation_server/src/features/tasks/tasks.service.ts b/dictation_server/src/features/tasks/tasks.service.ts index 898a8f9..8ddf007 100644 --- a/dictation_server/src/features/tasks/tasks.service.ts +++ b/dictation_server/src/features/tasks/tasks.service.ts @@ -435,12 +435,14 @@ export class TasksService { `author_id not found. audioFileId: ${audioFileId}. account_id: ${user.account_id}`, ); } - const { external_id: authorExternalId, notification: authorNotification } = - await this.usersRepository.findUserByAuthorId( - context, - task.file.author_id, - user.account_id, - ); + const { + external_id: authorExternalId, + notification: authorNotification, + } = await this.usersRepository.findUserByAuthorId( + context, + task.file.author_id, + user.account_id, + ); // プライマリ管理者を取得 const { external_id: primaryAdminExternalId } = diff --git a/dictation_server/src/gateways/sendgrid/sendgrid.service.ts b/dictation_server/src/gateways/sendgrid/sendgrid.service.ts index ac9a5df..cd3edc4 100644 --- a/dictation_server/src/gateways/sendgrid/sendgrid.service.ts +++ b/dictation_server/src/gateways/sendgrid/sendgrid.service.ts @@ -903,7 +903,7 @@ export class SendGridService { // メールを送信する await this.sendMail( context, - [authorEmail, typistEmail].filter((x): x is string => x !== null), // authorEmailがnullの場合は除外する + [authorEmail, typistEmail].filter((x): x is string => x !== null), // authorEmailがnullの場合は除外する [], this.mailFrom, subject,