Merged PR 826: Azure Functions実装（音声ファイル削除）

## 概要
[Task3880: Azure Functions実装（音声ファイル削除）](https://paruru.nds-tyo.co.jp:8443/tfs/ReciproCollection/fa4924a4-d079-4fab-9fb5-a9a11eb205f0/_workitems/edit/3880)

- 自動音声ファイル削除を実装
- 上記のテストを実装
- テストにMySQLを使用する仕組みを導入

## レビューポイント
- テストケースは十分か
- テスト内容は妥当か
- developにデプロイする前の動作確認・ユニットテストとして十分か

## クエリの変更
- 新規処理のため、既存からの変更はなし

## 動作確認状況
- DBが空の状態でローカル環境で実行し、0件削除のログが出ることを確認
  - 削除対象が正しいか等はdevelopでチェック予定
- 行った修正がデグレを発生させていないことを確認できるか
  - 既存処理の変更はなし

azure-pipelines-staging.yml

@@ -196,22 +196,12 @@ jobs:
     displayName: Bash Script (Test)
     inputs:
       targetType: inline
+      workingDirectory: dictation_function/.devcontainer
       script: |
-        cd dictation_function
+        docker-compose -f pipeline-docker-compose.yml build
-        npm run test
+        docker-compose -f pipeline-docker-compose.yml up -d
-    env:
+        docker-compose exec -T dictation_function sudo npm ci
-      TENANT_NAME: xxxxxxxxxxxx
+        docker-compose exec -T dictation_function sudo npm run test
-      SIGNIN_FLOW_NAME: xxxxxxxxxxxx
-      ADB2C_TENANT_ID: $(adb2c-tenant-id)
-      ADB2C_CLIENT_ID: $(adb2c-client-id)
-      ADB2C_CLIENT_SECRET: $(adb2c-client-secret)
-      ADB2C_ORIGIN: xxxxxx
-      SENDGRID_API_KEY: $(sendgrid-api-key)
-      MAIL_FROM: xxxxxx
-      APP_DOMAIN: xxxxxxxxx
-      REDIS_HOST: xxxxxxxxxxxx
-      REDIS_PORT: 0
-      REDIS_PASSWORD: xxxxxxxxxxxx
   - task: Docker@0
     displayName: build
     inputs:

dictation_function/.devcontainer/Dockerfile

@@ -24,6 +24,21 @@ RUN bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "$
     && apt-get install default-jre -y \
     && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts
 
+# COPY --from=golang:1.18-buster /usr/local/go/ /usr/local/go/
+ENV GO111MODULE=auto
+COPY library-scripts/go-debian.sh /tmp/library-scripts/
+RUN bash /tmp/library-scripts/go-debian.sh "1.18" "/usr/local/go" "${GOPATH}" "${USERNAME}" "false" \
+    && apt-get clean -y && rm -rf /tmp/library-scripts
+ENV PATH="/usr/local/go/bin:${PATH}"
+RUN mkdir -p /tmp/gotools \
+    && cd /tmp/gotools \
+    && export GOPATH=/tmp/gotools \
+    && export GOCACHE=/tmp/gotools/cache \
+    # sql-migrate
+    && go install github.com/rubenv/sql-migrate/sql-migrate@v1.1.2 \
+    && mv /tmp/gotools/bin/* ${TARGET_GOPATH}/bin/ \
+    && rm -rf /tmp/gotools 
+
 # Update NPM
 RUN npm install -g npm
 

dictation_function/.devcontainer/docker-compose.yml

@@ -16,6 +16,15 @@ services:
       - CHOKIDAR_USEPOLLING=true
     networks:
       - external
+  test_mysql_db:
+    image: mysql:8.0-bullseye
+    environment:
+      MYSQL_ROOT_PASSWORD: root_password
+      MYSQL_DATABASE: odms
+      MYSQL_USER: user
+      MYSQL_PASSWORD: password
+    networks:
+      - external
 networks:
   external:
     name: omds_network

dictation_function/.devcontainer/pipeline-docker-compose.yml

@@ -0,0 +1,31 @@
+version: "3"
+
+services:
+  dictation_function:
+    build: .
+    working_dir: /app/dictation_function
+    volumes:
+      - ../../:/app
+      - node_modules:/app/dictation_function/node_modules
+    environment:
+      - CHOKIDAR_USEPOLLING=true
+    depends_on:
+      - test_mysql_db
+    networks:
+      - dictation_function_network
+  test_mysql_db:
+    image: mysql:8.0-bullseye
+    environment:
+      MYSQL_ROOT_PASSWORD: root_password
+      MYSQL_DATABASE: odms
+      MYSQL_USER: user
+      MYSQL_PASSWORD: password
+    networks:
+      - dictation_function_network
+networks:
+  dictation_function_network:
+    name: test_dictation_function_network
+
+# Data Volume として永続化する
+volumes:
+  node_modules:

dictation_function/.env.test

@@ -0,0 +1,35 @@
+STAGE=local
+TENANT_NAME=tenantoname
+SIGNIN_FLOW_NAME=b2c_1_signin_dev
+ADB2C_TENANT_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ADB2C_CLIENT_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ADB2C_CLIENT_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ADB2C_CLIENT_SECRET=xxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+ADB2C_ORIGIN=https://xxxxxxx.b2clogin.com/xxxxxxxx.onmicrosoft.com/b2c_1_signin_dev/
+KEY_VAULT_NAME=xxxxxxxxxxxxxxx
+SENDGRID_API_KEY=SG.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+MAIL_FROM=noreply@se0223.com
+APP_DOMAIN=http://localhost:8081/
+REDIS_HOST=redis-cache
+REDIS_PORT=6379
+REDIS_PASSWORD=omdsredispass
+ADB2C_CACHE_TTL=86400
+STORAGE_ACCOUNT_NAME_US=saxxxxxxxxx
+STORAGE_ACCOUNT_NAME_AU=saxxxxxxxxx
+STORAGE_ACCOUNT_NAME_EU=saxxxxxxxxx
+STORAGE_ACCOUNT_NAME_IMPORT=saxxxxxxxxx
+STORAGE_ACCOUNT_NAME_ANALYSIS=saxxxxxxxxx
+STORAGE_ACCOUNT_KEY_US=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==
+STORAGE_ACCOUNT_KEY_AU=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==
+STORAGE_ACCOUNT_KEY_EU=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==
+STORAGE_ACCOUNT_KEY_IMPORT=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==
+STORAGE_ACCOUNT_KEY_ANALYSIS=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==
+STORAGE_ACCOUNT_ENDPOINT_US=https://saxxxxxxxxx.blob.core.windows.net/
+STORAGE_ACCOUNT_ENDPOINT_AU=https://saxxxxxxxxx.blob.core.windows.net/
+STORAGE_ACCOUNT_ENDPOINT_EU=https://saxxxxxxxxx.blob.core.windows.net/
+STORAGE_ACCOUNT_ENDPOINT_IMPORT=https://saxxxxxxxxx.blob.core.windows.net/
+STORAGE_ACCOUNT_ENDPOINT_ANALYSIS=https://saxxxxxxxxx.blob.core.windows.net/
+BASE_PATH=http://localhost:8081
+ACCESS_TOKEN_LIFETIME_WEB=7200000
+JWT_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA5IZZNgDew9eGmuFTezwdHYLSaJvUPPIKYoiOeVLD1paWNI51\n7Vkaoh0ngprcKOdv6T1N07V4igK7mOim2zY3yCTR6wcWR3PfFJrl9vh5SOo79koZ\noJb27YiM4jtxfx2dezzp0T2GoNR5rRolPUbWFJXnDe0DVXYXpJLb4LAlF2XAyYX0\nSYKUVUsJnzm5k4xbXtnwPwVbpm0EdswBE6qSfiL9zWk9dvHoKzSnfSDzDFoFcEoV\nchawzYXf/MM1YR4wo5XyzECc6Q5Ah4z522//mBNNaDHv83Yuw3mGShT73iJ0JQdk\nTturshv2Ecma38r6ftrIwNYXw4VVatJM8+GOOQIDAQABAoIBADrwp7u097+dK/tw\nWD61n3DIGAqg/lmFt8X4IH8MKLSE/FKr16CS1bqwOEuIM3ZdUtDeXd9Xs7IsyEPE\n5ZwuXK7DSF0M4+Mj8Ip49Q0Aww9aUoLQU9HGfgN/r4599GTrt31clZXA/6Mlighq\ncOZgCcEfdItz8OMu5SQuOIW4CKkCuaWnPOP26UqZocaXNZfpZH0iFLATMMH/TT8x\nay9ToHTQYE17ijdQ/EOLSwoeDV1CU1CIE3P4YfLJjvpKptly5dTevriHEzBi70Jx\n/KEPUn9Jj2gZafrUxRVhmMbm1zkeYxL3gsqRuTzRjEeeILuZhSJyCkQZyUNARxsg\nQY4DZfECgYEA+YLKUtmYTx60FS6DJ4s31TAsXY8kwhq/lB9E3GBZKDd0DPayXEeK\n4UWRQDTT6MI6fedW69FOZJ5sFLp8HQpcssb4Weq9PCpDhNTx8MCbdH3Um5QR3vfW\naKq/1XM8MDUnx5XcNYd87Aw3azvJAvOPr69as8IPnj6sKaRR9uQjbYUCgYEA6nfV\n5j0qmn0EJXZJblk4mvvjLLoWSs17j9YlrZJlJxXMDFRYtgnelv73xMxOMvcGoxn5\nifs7dpaM2x5EmA6jVU5sYaB/beZGEPWqPYGyjIwXPvUGAAv8Gbnvpp+xlSco/Dum\nIq0w+43ry5/xWh6CjfrvKV0J2bDOiJwPEdu/8iUCgYEAnBBSvL+dpN9vhFAzeOh7\nY71eAqcmNsLEUcG9MJqTKbSFwhYMOewF0iHRWHeylEPokhfBJn8kqYrtz4lVWFTC\n5o/Nh3BsLNXCpbMMIapXkeWiti1HgE9ErPMgSkJpwz18RDpYIqM8X+jEQS6D7HSr\nyxfDg+w+GJza0rEVE3hfMIECgYBw+KZ2VfhmEWBjEHhXE+QjQMR3s320MwebCUqE\nNCpKx8TWF/naVC0MwfLtvqbbBY0MHyLN6d//xpA9r3rLbRojqzKrY2KiuDYAS+3n\nzssRzxoQOozWju+8EYu30/ADdqfXyIHG6X3VZs87AGiQzGyJLmP3oR1y5y7MQa09\nJI16hQKBgHK5uwJhGa281Oo5/FwQ3uYLymbNwSGrsOJXiEu2XwJEXwVi2ELOKh4/\n03pBk3Kva3fIwEK+vCzDNnxShIQqBE76/2I1K1whOfoUehhYvKHGaXl2j70Zz9Ks\nrkGW1cx7p+yDqATDrwHBHTHFh5bUTTn8dN40n0e0W/llurpbBkJM\n-----END RSA PRIVATE KEY-----\n"
+JWT_PUBLIC_KEY="-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5IZZNgDew9eGmuFTezwd\nHYLSaJvUPPIKYoiOeVLD1paWNI517Vkaoh0ngprcKOdv6T1N07V4igK7mOim2zY3\nyCTR6wcWR3PfFJrl9vh5SOo79koZoJb27YiM4jtxfx2dezzp0T2GoNR5rRolPUbW\nFJXnDe0DVXYXpJLb4LAlF2XAyYX0SYKUVUsJnzm5k4xbXtnwPwVbpm0EdswBE6qS\nfiL9zWk9dvHoKzSnfSDzDFoFcEoVchawzYXf/MM1YR4wo5XyzECc6Q5Ah4z522//\nmBNNaDHv83Yuw3mGShT73iJ0JQdkTturshv2Ecma38r6ftrIwNYXw4VVatJM8+GO\nOQIDAQAB\n-----END PUBLIC KEY-----\n"

dictation_function/.gitignore

@@ -11,6 +11,8 @@ Publish
 *.Cache
 project.lock.json
 
+.test/
+
 /packages
 /TestResults
 

dictation_function/package.json

@@ -9,7 +9,7 @@
     "clean": "rimraf dist",
     "prestart": "npm run clean && npm run build",
     "start": "func start",
-    "test": "jest",
+    "test": "sql-migrate up -config=/app/dictation_server/db/dbconfig.yml -env=test && jest -w 1",
     "codegen": "sh codegen.sh"
   },
   "dependencies": {

dictation_function/src/blobstorage/audioBlobStorage.service.ts

@@ -0,0 +1,146 @@
+import {
+  BlobServiceClient,
+  ContainerClient,
+  StorageSharedKeyCredential,
+} from "@azure/storage-blob";
+import {
+  BLOB_STORAGE_REGION_AU,
+  BLOB_STORAGE_REGION_EU,
+  BLOB_STORAGE_REGION_US,
+} from "../constants";
+import { InvocationContext } from "@azure/functions";
+
+export class AudioBlobStorageService {
+  private readonly sharedKeyCredentialUS: StorageSharedKeyCredential;
+  private readonly sharedKeyCredentialEU: StorageSharedKeyCredential;
+  private readonly sharedKeyCredentialAU: StorageSharedKeyCredential;
+
+  private readonly blobServiceClientUS: BlobServiceClient;
+  private readonly blobServiceClientEU: BlobServiceClient;
+  private readonly blobServiceClientAU: BlobServiceClient;
+
+  constructor() {
+    if (
+      !process.env.STORAGE_ACCOUNT_ENDPOINT_US ||
+      !process.env.STORAGE_ACCOUNT_ENDPOINT_AU ||
+      !process.env.STORAGE_ACCOUNT_ENDPOINT_EU ||
+      !process.env.STORAGE_ACCOUNT_NAME_US ||
+      !process.env.STORAGE_ACCOUNT_KEY_US ||
+      !process.env.STORAGE_ACCOUNT_NAME_AU ||
+      !process.env.STORAGE_ACCOUNT_KEY_AU ||
+      !process.env.STORAGE_ACCOUNT_NAME_EU ||
+      !process.env.STORAGE_ACCOUNT_KEY_EU
+    ) {
+      throw new Error("Storage account information is missing");
+    }
+
+    // リージョンごとのSharedKeyCredentialを生成
+    this.sharedKeyCredentialUS = new StorageSharedKeyCredential(
+      process.env.STORAGE_ACCOUNT_NAME_US,
+      process.env.STORAGE_ACCOUNT_KEY_US
+    );
+    this.sharedKeyCredentialAU = new StorageSharedKeyCredential(
+      process.env.STORAGE_ACCOUNT_NAME_AU,
+      process.env.STORAGE_ACCOUNT_KEY_AU
+    );
+    this.sharedKeyCredentialEU = new StorageSharedKeyCredential(
+      process.env.STORAGE_ACCOUNT_NAME_EU,
+      process.env.STORAGE_ACCOUNT_KEY_EU
+    );
+
+    // Audioファイルの保存先のリージョンごとにBlobServiceClientを生成
+    this.blobServiceClientUS = new BlobServiceClient(
+      process.env.STORAGE_ACCOUNT_ENDPOINT_US,
+      this.sharedKeyCredentialUS
+    );
+    this.blobServiceClientAU = new BlobServiceClient(
+      process.env.STORAGE_ACCOUNT_ENDPOINT_AU,
+      this.sharedKeyCredentialAU
+    );
+    this.blobServiceClientEU = new BlobServiceClient(
+      process.env.STORAGE_ACCOUNT_ENDPOINT_EU,
+      this.sharedKeyCredentialEU
+    );
+  }
+
+  /**
+   * 指定されたファイルを削除します。
+   * @param context
+   * @param accountId
+   * @param country
+   * @param fileName
+   * @returns file
+   */
+  async deleteFile(
+    context: InvocationContext,
+    accountId: number,
+    country: string,
+    fileName: string
+  ): Promise<void> {
+    context.log(
+      `[IN] ${this.deleteFile.name} | params: { ` +
+        `accountId: ${accountId} ` +
+        `country: ${country} ` +
+        `fileName: ${fileName} };`
+    );
+
+    try {
+      // 国に応じたリージョンでコンテナ名を指定してClientを取得
+      const containerClient = this.getContainerClient(
+        context,
+        accountId,
+        country
+      );
+      // コンテナ内のBlobパス名を指定してClientを取得
+      const blobClient = containerClient.getBlobClient(fileName);
+
+      const { succeeded, errorCode, date } = await blobClient.deleteIfExists();
+      context.log(
+        `succeeded: ${succeeded}, errorCode: ${errorCode}, date: ${date}`
+      );
+
+      // 失敗時、Blobが存在しない場合以外はエラーとして例外をスローする
+      // Blob不在の場合のエラーコードは「BlobNotFound」以下を参照
+      // https://learn.microsoft.com/ja-jp/rest/api/storageservices/blob-service-error-codes
+      if (!succeeded && errorCode !== "BlobNotFound") {
+        throw new Error(
+          `delete blob failed. succeeded: ${succeeded}, errorCode: ${errorCode}, date: ${date}`
+        );
+      }
+    } catch (e) {
+      context.error(`error=${e}`);
+      throw e;
+    } finally {
+      context.log(`[OUT] ${this.deleteFile.name}`);
+    }
+  }
+
+  /**
+   * 指定してアカウントIDと国に応じたリージョンのコンテナクライアントを取得します。
+   * @param context
+   * @param accountId
+   * @param country
+   * @returns
+   */
+  private getContainerClient(
+    context: InvocationContext,
+    accountId: number,
+    country: string
+  ): ContainerClient {
+    context.log(
+      `[IN] ${this.getContainerClient.name} | params: { ` +
+        `accountId: ${accountId}; country: ${country} };`
+    );
+
+    const containerName = `account-${accountId}`;
+    if (BLOB_STORAGE_REGION_US.includes(country)) {
+      return this.blobServiceClientUS.getContainerClient(containerName);
+    } else if (BLOB_STORAGE_REGION_AU.includes(country)) {
+      return this.blobServiceClientAU.getContainerClient(containerName);
+    } else if (BLOB_STORAGE_REGION_EU.includes(country)) {
+      return this.blobServiceClientEU.getContainerClient(containerName);
+    } else {
+      throw new Error("invalid country");
+    }
+  }
+}

dictation_function/src/constants/index.ts

@@ -338,6 +338,12 @@ export const SYSTEM_IMPORT_USERS = "import-users";
 
 export const ROW_START_INDEX = 2;
 
+/**
+ * ファイル保持日数の初期値
+ * @const {number}
+ */
+export const FILE_RETENTION_DAYS_DEFAULT = 30;
+
 /**
  * ライセンス数推移出力機能のCSVヘッダ
  * @const {string[]}

dictation_function/src/database/initializeDataSource.ts

@@ -6,8 +6,11 @@ import {
   LicenseAllocationHistoryArchive,
   LicenseArchive,
 } from "../entity/license.entity";
-import {  InvocationContext, } from "@azure/functions";
+import { InvocationContext } from "@azure/functions";
 import { DataSource } from "typeorm";
+import { Task } from "../entity/task.entity";
+import { AudioFile } from "../entity/audio_file.entity";
+import { AudioOptionItem } from "../entity/audio_option_item.entity";
 
 export async function initializeDataSource(
   context: InvocationContext
@@ -25,6 +28,9 @@ export async function initializeDataSource(
         UserArchive,
         Account,
         AccountArchive,
+        Task,
+        AudioFile,
+        AudioOptionItem,
         License,
         LicenseArchive,
         LicenseAllocationHistory,

dictation_function/src/entity/account.entity.ts

@@ -11,6 +11,7 @@ import {
   JoinColumn,
   OneToMany,
 } from "typeorm";
+import { FILE_RETENTION_DAYS_DEFAULT } from "../constants";
 
 @Entity({ name: "accounts" })
 export class Account {
@@ -47,6 +48,12 @@ export class Account {
   @Column({ nullable: true, type: "bigint", transformer: bigintTransformer })
   active_worktype_id: number | null;
 
+  @Column({ default: false })
+  auto_file_delete: boolean;
+
+  @Column({ default: FILE_RETENTION_DAYS_DEFAULT })
+  file_retention_days: number;
+
   @Column({ nullable: true, type: "datetime" })
   deleted_at: Date | null;
 

dictation_function/src/entity/audio_file.entity.ts

@@ -0,0 +1,40 @@
+import { Entity, Column, PrimaryGeneratedColumn, OneToOne } from "typeorm";
+
+@Entity({ name: "audio_files" })
+export class AudioFile {
+  @PrimaryGeneratedColumn()
+  id: number;
+
+  @Column()
+  account_id: number;
+  @Column()
+  owner_user_id: number;
+  @Column()
+  url: string;
+  @Column()
+  file_name: string;
+  @Column()
+  author_id: string;
+  @Column()
+  work_type_id: string;
+  @Column()
+  started_at: Date;
+  @Column({ type: "time" })
+  duration: string;
+  @Column()
+  finished_at: Date;
+  @Column()
+  uploaded_at: Date;
+  @Column()
+  file_size: number;
+  @Column()
+  priority: string;
+  @Column()
+  audio_format: string;
+  @Column({ nullable: true, type: "varchar" })
+  comment: string | null;
+  @Column({ nullable: true, type: "datetime" })
+  deleted_at: Date | null;
+  @Column()
+  is_encrypted: boolean;
+}

dictation_function/src/entity/audio_option_item.entity.ts

@@ -0,0 +1,13 @@
+import { Entity, Column, PrimaryGeneratedColumn } from "typeorm";
+
+@Entity({ name: "audio_option_items" })
+export class AudioOptionItem {
+  @PrimaryGeneratedColumn()
+  id: number;
+  @Column()
+  audio_file_id: number;
+  @Column()
+  label: string;
+  @Column()
+  value: string;
+}

dictation_function/src/entity/task.entity.ts

@@ -0,0 +1,59 @@
+import { AudioOptionItem } from "./audio_option_item.entity";
+import { AudioFile } from "./audio_file.entity";
+import {
+  Entity,
+  Column,
+  PrimaryGeneratedColumn,
+  OneToOne,
+  JoinColumn,
+  OneToMany,
+  ManyToOne,
+  CreateDateColumn,
+  UpdateDateColumn,
+} from "typeorm";
+import { bigintTransformer } from "../common/entity";
+import { Account } from "./account.entity";
+
+@Entity({ name: "tasks" })
+export class Task {
+  @PrimaryGeneratedColumn()
+  id: number;
+  @Column()
+  job_number: string;
+  @Column()
+  account_id: number;
+  @Column({ nullable: true, type: "boolean" })
+  is_job_number_enabled: boolean | null;
+  @Column()
+  audio_file_id: number;
+  @Column()
+  status: string;
+  @Column({ nullable: true, type: "bigint", transformer: bigintTransformer })
+  typist_user_id: number | null;
+  @Column()
+  priority: string;
+  @Column({ nullable: true, type: "bigint", transformer: bigintTransformer })
+  template_file_id: number | null;
+  @Column({ nullable: true, type: "datetime" })
+  started_at: Date | null;
+  @Column({ nullable: true, type: "datetime" })
+  finished_at: Date | null;
+
+  @Column({ nullable: true, type: "datetime" })
+  created_by: string | null;
+
+  @CreateDateColumn({
+    default: () => "datetime('now', 'localtime')",
+    type: "datetime",
+  }) // defaultはSQLite用設定値.本番用は別途migrationで設定
+  created_at: Date;
+
+  @Column({ nullable: true, type: "datetime" })
+  updated_by: string | null;
+
+  @UpdateDateColumn({
+    default: () => "datetime('now', 'localtime')",
+    type: "datetime",
+  }) // defaultはSQLite用設定値.本番用は別途migrationで設定
+  updated_at: Date;
+}

dictation_function/src/functions/deleteAudioFiles.ts

@@ -0,0 +1,187 @@
+import { app, InvocationContext, Timer } from "@azure/functions";
+import { DataSource, In } from "typeorm";
+import { Task } from "../entity/task.entity";
+import { AudioFile } from "../entity/audio_file.entity";
+import { AudioOptionItem } from "../entity/audio_option_item.entity";
+import { MANUAL_RECOVERY_REQUIRED } from "../constants";
+import * as dotenv from "dotenv";
+import { initializeDataSource } from "../database/initializeDataSource";
+import { AudioBlobStorageService } from "../blobstorage/audioBlobStorage.service";
+
+app.timer("deleteAudioFiles", {
+  schedule: "0 3 * * *", // 毎日UTC 3:00に実行
+  handler: deleteAudioFiles,
+});
+
+// 削除対象となる音声ファイルとタスクを特定する為の情報
+type TargetTaskInfo = {
+  /**
+   * タスクID
+   * column: tasks.id
+   */
+  id: number;
+
+  /**
+   * ファイルID
+   * column: audio_files.id
+   */
+  audio_file_id: number;
+
+  /**
+   * ファイル名
+   * column: audio_files.file_name
+   */
+  file_name: string;
+
+  /**
+   * アカウントID
+   * column: accounts.id
+   */
+  account_id: number;
+
+  /**
+   * アカウントの所属する地域
+   * column: accounts.country
+   */
+  country: string;
+};
+
+export async function deleteAudioFilesProcessing(
+  context: InvocationContext,
+  dataSource: DataSource,
+  blobStorageService: AudioBlobStorageService,
+  now: Date
+): Promise<void> {
+  context.log(`[IN] deleteAudioFilesProcessing. now=${now}`);
+  try {
+    // 削除対象のタスクとファイルを取得
+    const targets = await getProcessTargets(dataSource, now);
+    context.log(`delete targets: ${JSON.stringify(targets)}`);
+
+    // DBからレコードを削除
+    await deleteRecords(dataSource, targets);
+
+    // タスクに紐づくファイルを削除
+    for (const target of targets) {
+      try {
+        const { account_id, country, file_name } = target;
+        await blobStorageService.deleteFile(
+          context,
+          account_id,
+          country,
+          file_name
+        );
+        context.log(`file delete success. target=${JSON.stringify(target)}`);
+      } catch (e) {
+        context.log(
+          `${MANUAL_RECOVERY_REQUIRED} file delete failed. target=${JSON.stringify(
+            target
+          )}`
+        );
+      }
+    }
+  } catch (error) {
+    // DB関連で例外が発生した場合、アラートを出す為のログを出力する
+    context.error(
+      `${MANUAL_RECOVERY_REQUIRED} Failed to execute auto file deletion function. error=${error}`
+    );
+    throw error;
+  } finally {
+    context.log(`[OUT] deleteAudioFilesProcessing`);
+  }
+}
+
+async function deleteAudioFiles(
+  myTimer: Timer,
+  context: InvocationContext
+): Promise<void> {
+  context.log("[IN]deleteAudioFiles");
+
+  dotenv.config({ path: ".env" });
+  dotenv.config({ path: ".env.local", override: true });
+  const dataSource = await initializeDataSource(context);
+  try {
+    await deleteAudioFilesProcessing(
+      context,
+      dataSource,
+      new AudioBlobStorageService(),
+      new Date()
+    );
+  } catch (e) {
+    context.log("deleteAudioFilesProcessing failed.");
+    context.error(e);
+    throw e;
+  } finally {
+    await dataSource.destroy();
+    context.log("[OUT]deleteAudioFiles");
+  }
+}
+
+/**
+ * タスクに紐づくファイルを削除する
+ * @param dataSource
+ * @param targets
+ * @returns Promise<void>
+ */
+// テスト容易性を高めて開発効率を上げるため、本来はexport不要だがexportを行う 2024/03/13
+export async function deleteRecords(
+  dataSource: DataSource,
+  targets: TargetTaskInfo[]
+): Promise<void> {
+  const taskIds = targets.map((target) => target.id);
+  const audioFileIds = targets.map((target) => target.audio_file_id);
+
+  // taskIdsに紐づくタスクを削除
+  await dataSource.transaction(async (manager) => {
+    const taskRepository = manager.getRepository(Task);
+    const audioFileRepository = manager.getRepository(AudioFile);
+    const audioOptionItem = manager.getRepository(AudioOptionItem);
+
+    // tasksテーブルから削除
+    await taskRepository.delete({
+      id: In(taskIds),
+    });
+    // audio_filesテーブルから削除
+    await audioFileRepository.delete({
+      id: In(audioFileIds),
+    });
+    // audio_option_itemsテーブルから削除
+    await audioOptionItem.delete({
+      audio_file_id: In(audioFileIds),
+    });
+  });
+}
+
+/*
+ * ファイル削除対象のタスクを取得する
+ * @param dataSource
+ * @param now
+ * @returns Promise<{ account_id: number; audio_file_id: number; file_name: string; }[]>
+ */
+// テスト容易性を高めて開発効率を上げるため、本来はexport不要だがexportを行う 2024/03/13
+export async function getProcessTargets(
+  dataSource: DataSource,
+  now: Date
+): Promise<TargetTaskInfo[]> {
+  return await dataSource
+    .createQueryBuilder(Task, "tasks")
+    // SELECTでエイリアスを指定して、結果オブジェクトのプロパティとTargetTaskInfoのプロパティを一致させる
+    .select("tasks.id", "id")
+    .addSelect("tasks.audio_file_id", "audio_file_id")
+    .addSelect("audio_files.file_name", "file_name")
+    .addSelect("accounts.id", "account_id")
+    .addSelect("accounts.country", "country")
+    .where("tasks.finished_at IS NOT NULL")
+    .innerJoin("accounts", "accounts", "accounts.id = tasks.account_id")
+    .innerJoin(
+      "audio_files",
+      "audio_files",
+      "audio_files.id = tasks.audio_file_id"
+    )
+    .andWhere("accounts.auto_file_delete = true")
+    .andWhere(
+      "DATE_ADD(tasks.finished_at, INTERVAL accounts.file_retention_days DAY) < :now",
+      { now }
+    )
+    .getRawMany<TargetTaskInfo>();
+}

dictation_function/src/test/analysisLicenses.spec.ts

@@ -33,7 +33,7 @@ import { BlobstorageService } from "../blobstorage/blobstorage.service";
 import { User, UserArchive } from "../entity/user.entity";
 describe("analysisLicenses", () => {
   dotenv.config({ path: ".env" });
-  dotenv.config({ path: ".env.local", override: true });
+  dotenv.config({ path: ".env.test", override: true });
   let source: DataSource | null = null;
 
   beforeEach(async () => {

dictation_function/src/test/common/init.ts

@@ -0,0 +1,21 @@
+import { DataSource } from 'typeorm';
+
+export const truncateAllTable = async (source: DataSource) => {
+  const entities = source.entityMetadatas;
+  const queryRunner = source.createQueryRunner();
+
+  try {
+    await queryRunner.startTransaction();
+    await queryRunner.query('SET FOREIGN_KEY_CHECKS=0');
+    for (const entity of entities) {
+      await queryRunner.query(`TRUNCATE TABLE \`${entity.tableName}\``);
+    }
+    await queryRunner.query('SET FOREIGN_KEY_CHECKS=1');
+    await queryRunner.commitTransaction();
+  } catch (err) {
+    await queryRunner.rollbackTransaction();
+    throw err;
+  } finally {
+    await queryRunner.release();
+  }
+};

dictation_function/src/test/common/logger.ts

@@ -0,0 +1,129 @@
+import { Logger, QueryRunner } from "typeorm";
+import * as fs from "fs";
+import * as path from "path";
+
+interface IOutput {
+  initialize(): void;
+  write(message: string): void;
+}
+
+class ConsoleOutput implements IOutput {
+  initialize(): void {
+    // do nothing
+  }
+
+  write(message: string): void {
+    console.log(message);
+  }
+}
+
+class FileOutput implements IOutput {
+  private logPath = path.join("/app/dictation_function/.test", "logs");
+  private fileName = new Date().getTime();
+
+  initialize(): void {
+    if (!fs.existsSync(this.logPath)) {
+      fs.mkdirSync(this.logPath, { recursive: true });
+    }
+  }
+
+  write(message: string): void {
+    const logFile = path.join(this.logPath, `${this.fileName}.log`);
+    fs.appendFileSync(logFile, `${message}\n`);
+  }
+}
+
+class NoneOutput implements IOutput {
+  initialize(): void {
+    // do nothing
+  }
+
+  write(message: string): void {
+    // do nothing
+  }
+}
+
+export class TestLogger implements Logger {
+  out: IOutput;
+
+  constructor(output: "none" | "file" | "console") {
+    switch (output) {
+      case "none":
+        this.out = new NoneOutput();
+        break;
+      case "file":
+        this.out = new FileOutput();
+        break;
+      case "console":
+        this.out = new ConsoleOutput();
+        break;
+      default:
+        this.out = new NoneOutput();
+        break;
+    }
+    this.out.initialize();
+  }
+
+  private write(message: string): void {
+    this.out.write(message);
+  }
+
+  logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) {
+    const raw = `Query: ${query} -- Parameters: ${JSON.stringify(parameters)}`;
+    // ex: 2024-03-08T06:38:43.125Z を TIME という文字列に置換
+    const dateRemoved = raw.replace(
+      /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g,
+      "TIME"
+    );
+    // ex: /* コメント内容 */ を /* コメント */ という文字列に置換
+    const commentRemoved = dateRemoved.replace(
+      /\/\*.*\*\//g,
+      "/* RequestID */"
+    );
+
+    // UUIDを固定文字列に置換する ex: 88a9c78e-115a-439c-9e23-731d649f0c27 を XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX という文字列に置換
+    const uuidRemoved = commentRemoved.replace(
+      /[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/g,
+      "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"
+    );
+    this.write(uuidRemoved);
+  }
+
+  logQueryError(
+    error: string,
+    query: string,
+    parameters?: any[],
+    queryRunner?: QueryRunner
+  ) {
+    this.write(
+      `ERROR: ${error} -- Query: ${query} -- Parameters: ${JSON.stringify(
+        parameters
+      )}`
+    );
+  }
+
+  logQuerySlow(
+    time: number,
+    query: string,
+    parameters?: any[],
+    queryRunner?: QueryRunner
+  ) {
+    this.write(
+      `SLOW QUERY: ${time}ms -- Query: ${query} -- Parameters: ${JSON.stringify(
+        parameters
+      )}`
+    );
+  }
+
+  logSchemaBuild(message: string, queryRunner?: QueryRunner) {
+    this.write(`Schema Build: ${message}`);
+  }
+
+  logMigration(message: string, queryRunner?: QueryRunner) {
+    this.write(`Migration: ${message}`);
+  }
+
+  log(level: "log" | "info" | "warn", message: any, queryRunner?: QueryRunner) {
+    this.write(`${level.toUpperCase()}: ${message}`);
+  }
+}

dictation_function/src/test/common/utility.ts

@@ -1,14 +1,19 @@
 import { v4 as uuidv4 } from "uuid";
-import { DataSource } from "typeorm";
+import { DataSource, In } from "typeorm";
 import { User, UserArchive } from "../../entity/user.entity";
 import { Account, AccountArchive } from "../../entity/account.entity";
-import { ADMIN_ROLES, USER_ROLES } from "../../constants";
+import { ADMIN_ROLES, TASK_STATUS, USER_ROLES } from "../../constants";
 import {
   License,
   LicenseAllocationHistory,
   LicenseArchive,
   LicenseAllocationHistoryArchive,
 } from "../../entity/license.entity";
+import { bigintTransformer } from "../../common/entity";
+import { Task } from "../../entity/task.entity";
+import { AudioFile } from "../../entity/audio_file.entity";
+import { AudioOptionItem } from "../../entity/audio_option_item.entity";
+import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
 
 type InitialTestDBState = {
   tier1Accounts: { account: Account; users: User[] }[];
@@ -41,6 +46,8 @@ type OverrideUserArchive = Omit<
   "id" | "account" | "license" | "userGroupMembers"
 >;
 
+type OverrideTask = Omit<Task, "id">;
+
 type AccountDefault = { [K in keyof OverrideAccount]?: OverrideAccount[K] };
 type UserDefault = { [K in keyof OverrideUser]?: OverrideUser[K] };
 type AccountArchiveDefault = {
@@ -49,6 +56,7 @@ type AccountArchiveDefault = {
 type UserArchiveDefault = {
   [K in keyof OverrideUserArchive]?: OverrideUserArchive[K];
 };
+type TaskDefault = { [K in keyof OverrideTask]?: OverrideTask[K] };
 
 /**
  * テスト ユーティリティ: 指定したプロパティを上書きしたユーザーを作成する
@@ -117,6 +125,8 @@ export const makeTestAccount = async (
       locked: d?.locked ?? false,
       company_name: d?.company_name ?? "test inc.",
       verified: d?.verified ?? true,
+      auto_file_delete: d?.auto_file_delete ?? false,
+      file_retention_days: d?.file_retention_days ?? 30,
       deleted_at: d?.deleted_at ?? "",
       created_by: d?.created_by ?? "test_runner",
       created_at: d?.created_at ?? new Date(),
@@ -124,7 +134,7 @@ export const makeTestAccount = async (
       updated_at: d?.updated_at ?? new Date(),
     });
     const result = identifiers.pop() as Account;
-    accountId = result.id;
+    accountId = bigintTransformer.from(result.id);
   }
   {
     const d = defaultAdminUserValue;
@@ -148,7 +158,7 @@ export const makeTestAccount = async (
     });
 
     const result = identifiers.pop() as User;
-    userId = result.id;
+    userId = bigintTransformer.from(result.id);
   }
 
   // Accountの管理者を設定する
@@ -471,3 +481,176 @@ export const createLicenseAllocationHistoryArchive = async (
     });
   identifiers.pop() as LicenseAllocationHistoryArchive;
 };
+
+export const makeTestTask = async (
+  datasource: DataSource,
+  accountId: number,
+  ownerUserId: number,
+  identifierName: string,
+  defaultTaskValue?: TaskDefault
+): Promise<{ task: Task; file: AudioFile; options: AudioOptionItem[] }> => {
+  const d = defaultTaskValue;
+
+  // AudioFileを作成
+  const { identifiers: identifiers1 } = await datasource
+    .getRepository(AudioFile)
+    .insert({
+      account_id: accountId,
+      owner_user_id: ownerUserId,
+      url: `https://example.com/${identifierName}`,
+      file_name: `test${identifierName}.wav`,
+      author_id: "test_author",
+      work_type_id: "test_work_type",
+      started_at: new Date(),
+      duration: "00:00:00",
+      finished_at: new Date(),
+      uploaded_at: new Date(),
+      file_size: 1024,
+      priority: "01",
+      audio_format: "wav",
+      comment: `test_comment_${identifierName}`,
+      deleted_at: new Date(),
+      is_encrypted: false,
+    });
+
+  const result = identifiers1.pop() as AudioFile;
+  const audioFileId = bigintTransformer.from(result.id);
+
+  // AudioFileを取得
+  const file = await datasource.getRepository(AudioFile).findOne({
+    where: {
+      id: audioFileId,
+    },
+  });
+  if (!file) {
+    throw new Error("Unexpected null");
+  }
+
+  // Taskを作成
+  const { identifiers: identifiers2 } = await datasource
+    .getRepository(Task)
+    .insert({
+      job_number: d?.job_number ?? "0001",
+      account_id: accountId,
+      is_job_number_enabled: d?.is_job_number_enabled ?? true,
+      audio_file_id: file.id,
+      status: d?.status ?? "Uploaded",
+      typist_user_id: d?.typist_user_id,
+      priority: d?.priority ?? "01",
+      template_file_id: d?.template_file_id,
+      started_at: d?.started_at ?? new Date(),
+      finished_at: d?.finished_at ?? new Date(),
+      created_by: d?.created_by ?? "test_runner",
+      created_at: d?.created_at ?? new Date(),
+      updated_by: d?.updated_by ?? "updater",
+      updated_at: d?.updated_at ?? new Date(),
+    });
+  const result2 = identifiers2.pop() as Task;
+  const taskId = bigintTransformer.from(result2.id);
+
+  const task = await datasource.getRepository(Task).findOne({
+    where: {
+      id: taskId,
+    },
+  });
+  if (!task) {
+    throw new Error("Unexpected null");
+  }
+
+  // AudioOptionItemを作成
+  const item01 = await datasource.getRepository(AudioOptionItem).insert({
+    audio_file_id: audioFileId,
+    label: `test_option_label_${identifierName}_01`,
+    value: `test_option_value_${identifierName}_01`,
+  });
+  const item02 = await datasource.getRepository(AudioOptionItem).insert({
+    audio_file_id: audioFileId,
+    label: `test_option_label_${identifierName}_02`,
+    value: `test_option_value_${identifierName}_02`,
+  });
+  const optionItemResult01 = item01.identifiers.pop() as AudioOptionItem;
+  const optionItemResult02 = item02.identifiers.pop() as AudioOptionItem;
+  const optionItemID01 = bigintTransformer.from(optionItemResult01.id);
+  const optionItemID02 = bigintTransformer.from(optionItemResult02.id);
+
+  const optionItems = await datasource.getRepository(AudioOptionItem).find({
+    where: {
+      id: In([optionItemID01, optionItemID02]),
+    },
+  });
+
+  return { task, file, options: optionItems };
+};
+
+export const makeManyTestTasks = async (
+  datasource: DataSource,
+  inputFiles: QueryDeepPartialEntity<AudioFile>[],
+  task_finished_at: Date
+): Promise<void> => {
+  const fileRepository = datasource.getRepository(AudioFile);
+  const result = await fileRepository.insert(inputFiles);
+  const audioFileIds = result.identifiers.map((id) => id.id);
+  const files = await fileRepository.find({
+    where: {
+      id: In(audioFileIds),
+    },
+  });
+
+  const tasks = files.map((file, index): QueryDeepPartialEntity<Task> => {
+    return {
+      job_number: `0001_${index}`,
+      account_id: file.account_id,
+      is_job_number_enabled: true,
+      audio_file_id: file.id,
+      status: TASK_STATUS.FINISHED,
+      typist_user_id: null,
+      priority: "01",
+      template_file_id: null,
+      started_at: new Date(),
+      finished_at: task_finished_at,
+      created_by: "test_runner",
+      created_at: new Date(),
+      updated_by: "updater",
+      updated_at: new Date(),
+    };
+  });
+
+  const taskRepository = datasource.getRepository(Task);
+  const x = await taskRepository.insert(tasks);
+
+  const partialOptions = files.flatMap(
+    (file, index): QueryDeepPartialEntity<AudioOptionItem>[] => {
+      return [
+        {
+          audio_file_id: file.id,
+          label: `test_option_label_${index}_01`,
+          value: `test_option_value_${index}_01`,
+        },
+        {
+          audio_file_id: file.id,
+          label: `test_option_label_${index}_02`,
+          value: `test_option_value_${index}_02`,
+        },
+      ];
+    }
+  );
+
+  const optionRepository = datasource.getRepository(AudioOptionItem);
+  await optionRepository.insert(partialOptions);
+};
+
+export const getTasks = async (datasource: DataSource): Promise<Task[]> => {
+  return await datasource.getRepository(Task).find();
+};
+
+export const getAudioFiles = async (
+  datasource: DataSource
+): Promise<AudioFile[]> => {
+  return await datasource.getRepository(AudioFile).find();
+};
+
+export const getAudioOptionItems = async (
+  datasource: DataSource
+): Promise<AudioOptionItem[]> => {
+  return await datasource.getRepository(AudioOptionItem).find();
+};

dictation_function/src/test/importUsers.spec.ts

@@ -11,7 +11,7 @@ import { AxiosRequestConfig, AxiosResponse } from "axios";
 
 describe("importUsersProcessing", () => {
   dotenv.config({ path: ".env" });
-  dotenv.config({ path: ".env.local", override: true });
+  dotenv.config({ path: ".env.test", override: true });
 
   it("stage.jsonがない状態でユーザー追加できること", async () => {
     const context = new InvocationContext();

dictation_function/src/test/licenseAlert.spec.ts

@@ -18,7 +18,7 @@ import { promisify } from "util";
 
 describe("licenseAlert", () => {
   dotenv.config({ path: ".env" });
-  dotenv.config({ path: ".env.local", override: true });
+  dotenv.config({ path: ".env.test", override: true });
   let source: DataSource | null = null;
   const redisClient = createClient();
 

dictation_function/src/test/licenseAutoAllocation.spec.ts

@@ -18,7 +18,7 @@ import { InvocationContext } from "@azure/functions";
 
 describe("licenseAutoAllocation", () => {
   dotenv.config({ path: ".env" });
-  dotenv.config({ path: ".env.local", override: true });
+  dotenv.config({ path: ".env.test", override: true });
   let source: DataSource | null = null;
   beforeEach(async () => {
     source = new DataSource({