Add: spam filter test runner to PostController

Author: winverse

packages/velog-cron/.gitignore

@@ -1,7 +1,9 @@
 env/*
 !env/.env.example
 
-
 prisma
 
-dist
+dist
+
+# spam test data
+spam_post.json

packages/velog-cron/src/common/plugins/globals/errorHandlerPlugin.ts

@@ -4,8 +4,9 @@ import { FastifyPluginCallback } from 'fastify'
 import fp from 'fastify-plugin'
 
 const pluginFn: FastifyPluginCallback = (fastify, _, done) => {
-  fastify.addHook('onError', (request, reply, error) => {
-    console.log('fastify hook error:', error)
+  fastify.addHook('onError', (request, reply, error, done) => {
+    request.log.error(error, 'fastify onError')
+    done()
   })
   fastify.setErrorHandler((error, _, reply) => {
     if (isHttpError(error)) {

packages/velog-cron/src/routes/posts/v1/PostController.ts

@@ -2,10 +2,13 @@ import { BadRequestError } from '@errors/BadRequestErrors.js'
 import { NotFoundError } from '@errors/NotfoundError.js'
 import { DbService } from '@lib/db/DbService.js'
 import { PostService } from '@services/PostService/index.js'
-import { injectable, singleton } from 'tsyringe'
+import { container, injectable, singleton } from 'tsyringe'
 import { utcToZonedTime } from 'date-fns-tz'
 import { startOfDay, subMonths } from 'date-fns'
 import { ENV } from '@env'
+import fs from 'fs'
+import path from 'path'
+import { UtilsService } from '@lib/utils/UtilsService.js'
 
 interface Controller {
   updatePostScore(postId: string): Promise<void>
@@ -62,4 +65,68 @@ export class PostController implements Controller {
 
     return posts.length
   }
+  async spamFilterTestRunner() {
+    const utils = container.resolve(UtilsService)
+    const postService = container.resolve(PostService)
+    try {
+      if (ENV.appEnv !== 'development') return
+
+      const filePath = path.resolve(utils.resolveDir('./src/routes/posts/v1/spam_post.json'))
+
+      const fileExits = fs.existsSync(filePath)
+      if (!fileExits) return
+
+      const readFileResult = fs.readFileSync(filePath, { encoding: 'utf-8' })
+      const data = JSON.parse(readFileResult)
+      const key = Object.keys(data)[0]
+      const posts: PostData[] = data[key]
+        .filter((v: any) => !!v.title)
+        .map((v: any, index: number) => ({ id: index, ...v }))
+
+      const postLength = 5000
+      const set = new Set()
+
+      const bannedUesrnames: string[] = []
+
+      for (const post of posts.slice(0, postLength)) {
+        const { id, title, body, username } = post
+        if (bannedUesrnames.includes(username)) {
+          set.add(id)
+        }
+
+        const isSpam = await postService.checkIsSpam(title, body, username, '', 'US')
+        if (isSpam) {
+          set.add(id)
+          continue
+        }
+
+        const isSpam2 = await postService.checkIsSpam(title, body, username, '', 'KR')
+        if (isSpam2) {
+          set.add(id)
+        }
+      }
+
+      const isSpamCount = set.size
+      console.log('isSpamCount: ', isSpamCount)
+      console.log('ratio: ', isSpamCount / postLength)
+
+      const allowIds: number[] = []
+      for (const id of allowIds) {
+        set.add(id)
+      }
+
+      const notFilteredPosts = posts.filter((post) => !set.has(post.id))
+      console.log('notFilteredPosts', notFilteredPosts[0])
+    } catch (error) {
+      throw error
+    }
+  }
+}
+
+type PostData = {
+  id: number
+  title: string
+  body: string
+  tags: string
+  username: string
 }

packages/velog-cron/src/routes/posts/v1/index.ts

@@ -24,6 +24,11 @@ const v1: FastifyPluginCallback = (fastify, opts, done) => {
     reply.status(HttpStatus.OK).send({ processedPostsCount })
   })
 
+  fastify.post('/test/spam-filter', async (_, reply) => {
+    await postController.spamFilterTestRunner()
+    reply.status(HttpStatus.OK).send(HttpStatusMessage.Ok)
+  })
+
   done()
 }
 

packages/velog-cron/src/services/PostService/index.ts

@@ -111,11 +111,17 @@ export class PostService implements Service {
       .join('')
       .concat(user.profile?.short_bio ?? '', user.profile?.display_name ?? '')
 
-    const isSpam = await this.checkIsSpam(post.title ?? '', post.body ?? '', extraText, country)
+    const isSpam = await this.checkIsSpam(
+      post.title ?? '',
+      post.body ?? '',
+      user.username,
+      extraText,
+      country,
+    )
 
     if (!isSpam) return
 
-    this.db.post.update({
+    await this.db.post.update({
       where: {
         id: post.id,
       },
@@ -130,9 +136,10 @@ export class PostService implements Service {
 
     this.discord.sendMessage('spam', JSON.stringify(message))
   }
-  private async checkIsSpam(
+  public async checkIsSpam(
     title: string,
     body: string,
+    username: string,
     extraText: string,
     country: string,
   ): Promise<boolean> {
@@ -144,22 +151,25 @@ export class PostService implements Service {
       return true
     }
 
-    const checkTitle = await this.spamFilter(title!, isForeign, true)
-
+    const checkTitle = await this.spamFilter(title!, username, isForeign, true)
     if (checkTitle) {
       return true
     }
 
-    const checkBody = await this.spamFilter(body!.concat(extraText), isForeign)
-
+    const checkBody = await this.spamFilter(body!.concat(extraText), username, isForeign)
     if (checkBody) {
       return true
     }
 
     return false
   }
 
-  private async spamFilter(text: string, isForeign: boolean, isTitle = false): Promise<boolean> {
+  private async spamFilter(
+    text: string,
+    username: string,
+    isForeign: boolean,
+    isTitle = false,
+  ): Promise<boolean> {
     const includesCN = /[\u4e00-\u9fa5]/.test(text)
     const includesKR = /[ㄱ-ㅎ|ㅏ-ㅣ|가-힣]/.test(text)
 
@@ -171,20 +181,30 @@ export class PostService implements Service {
     // replace image markdown
     replaced = replaced.replace(/!\[([\s\S]*?)\]\(([\s\S]*?)\)/g, '')
 
-    const alphanumericKorean = replaced
-      .replace(/[^a-zA-Zㄱ-힣0-9 \n]/g, '') // remove non-korean
-      .toLowerCase()
+    if (isTitle) {
+      replaced = replaced.replace(/\s/g, '')
+    }
 
     const hasLink = /http/.test(replaced)
 
+    const phoneRegex = [/\+\d{13}/, /\+\d{11}/]
+
+    const containsPhoneNumber = phoneRegex.some((regex) => regex.test(replaced))
+
+    if (containsPhoneNumber) {
+      return true
+    }
+
     if (!isTitle && isForeign && hasLink) {
       const lines = replaced.split('\n').filter((line) => line.trim().length > 1)
       const koreanLinesCount = lines.filter((line) => this.hasKorean(line)).length
       const confidence = koreanLinesCount / lines.length
       return confidence < 0.3
     }
 
-    const spaceReplaced = alphanumericKorean.replace(/\s/g, '')
+    const removeDuplicatedWords = Array.from(
+      new Set(replaced.toLocaleLowerCase().replace(/\s/g, '').split(/\n| /)),
+    ).join(' ')
 
     const oneMonthAgo = subMonths(new Date(), 1)
     const bannedKeywords = await this.db.dynamicConfigItem.findMany({
@@ -196,38 +216,17 @@ export class PostService implements Service {
       },
     })
 
-    const removeDuplicated = Array.from(
-      new Set([text, alphanumericKorean, spaceReplaced].join(',').split(' ')),
-    )
-    const keywordsToUpdate: string[] = []
-
     const checkKeyword = bannedKeywords
       .map((keyword) => keyword.value)
       .some((keyword) => {
-        if (removeDuplicated.includes(keyword)) {
-          keywordsToUpdate.push(keyword)
+        if (removeDuplicatedWords.includes(keyword)) {
+          this.updateDynmicConfigItem(keyword)
           return true
         } else {
           return false
         }
       })
 
-    if (keywordsToUpdate.length > 0) {
-      this.db.dynamicConfigItem.updateMany({
-        where: {
-          value: {
-            in: keywordsToUpdate,
-          },
-        },
-        data: {
-          last_used_at: new Date(),
-          usage_count: {
-            increment: 1,
-          },
-        },
-      })
-    }
-
     if (checkKeyword) {
       return true
     }
@@ -238,23 +237,55 @@ export class PostService implements Service {
       },
     })
 
-    const score = bannedAltKeywords
-      .map(({ value }) => value)
-      .reduce((acc, current) => {
-        if (alphanumericKorean.includes(current)) {
-          return acc + 1
-        }
-        return acc
-      }, 0)
+    let score = 0
 
-    if (score >= 2 && isForeign) {
-      return true
+    if (hasLink) {
+      score++
+    }
+
+    const isOnlyNumbers = /^\d+$/.test(username)
+    if (isOnlyNumbers) {
+      score++
     }
+
+    const notAlphanumbericKorean = replaced.replace(/[a-zA-Zㄱ-힣0-9]/g, '') // remove korean
+    if (notAlphanumbericKorean.length / replaced.length > 0.35) {
+      score++
+    }
+
+    for (const { value: keyword } of bannedAltKeywords) {
+      if (removeDuplicatedWords.includes(keyword)) {
+        this.updateDynmicConfigItem(keyword)
+        score++
+      }
+
+      if (score >= 2 && isForeign) {
+        return true
+      }
+
+      if (score >= 3) {
+        return true
+      }
+    }
+
     return false
   }
   private hasKorean(text: string) {
     return /[ㄱ-힣]/g.test(text)
   }
+  private async updateDynmicConfigItem(value: string) {
+    await this.db.dynamicConfigItem.updateMany({
+      where: {
+        value,
+      },
+      data: {
+        last_used_at: new Date(),
+        usage_count: {
+          increment: 1,
+        },
+      },
+    })
+  }
 }
 
 type FindByUserIdParams = {