Feat: implments spam filter in cron package

Author: winverse

packages/velog-cron/env/.env.example

@@ -3,4 +3,5 @@ DATABASE_URL=
 CRON_API_KEY=
 REDIS_HOST=
 DISCORD_BOT_TOKEN=
-DISCORD_STATS_CHANNEL=
+DISCORD_STATS_CHANNEL=
+DISCORD_SPAM_CHANNEL=

packages/velog-cron/package.json

@@ -39,6 +39,7 @@
     "fastify": "^4.18.0",
     "fastify-cron": "^1.3.1",
     "fastify-plugin": "^4.5.1",
+    "geoip-country": "^4.2.68",
     "inquirer": "^9.2.7",
     "ioredis": "^5.3.2",
     "pino-pretty": "^10.0.0",
@@ -50,6 +51,7 @@
     "zod": "^3.21.4"
   },
   "devDependencies": {
+    "@types/geoip-country": "^4.0.2",
     "@types/inquirer": "^9.0.3",
     "@types/jest": "^29.5.2",
     "@types/node": "^20.5.0",

packages/velog-cron/src/env.ts

@@ -41,6 +41,7 @@ const env = z.object({
   redisHost: z.string(),
   discordBotToken: z.string(),
   discordStatsChannel: z.string(),
+  discordSpamChannel: z.string(),
 })
 
 export const ENV = env.parse({
@@ -52,4 +53,5 @@ export const ENV = env.parse({
   redisHost: process.env.REDIS_HOST,
   discordBotToken: process.env.DISCORD_BOT_TOKEN,
   discordStatsChannel: process.env.DISCORD_STATS_CHANNEL,
+  discordSpamChannel: process.env.DISCORD_SPAM_CHANNEL,
 })

packages/velog-cron/src/lib/discord/DiscordService.ts

@@ -43,6 +43,7 @@ export class DiscordService {
 
       const channelMapper: Record<MessageType, string> = {
         stats: ENV.discordStatsChannel,
+        spam: ENV.discordSpamChannel,
       }
 
       const channelId = channelMapper[type]
@@ -70,4 +71,4 @@ export class DiscordService {
   }
 }
 
-type MessageType = 'stats'
+type MessageType = 'stats' | 'spam'

packages/velog-cron/src/lib/redis/RedisService.ts

@@ -28,9 +28,10 @@ export class RedisService extends Redis implements Service {
     }
   }
 
-  get queueName() {
+  get queueName(): Record<QueueName, string> {
     return {
       createFeed: 'queue:feed',
+      checkPostSpam: 'queue:checkPostSpam',
     }
   }
 }
@@ -39,4 +40,10 @@ type GenerateRedisKey = {
   trendingWriters: () => string
 }
 
-type QueueName = 'createFeed'
+type QueueName = 'createFeed' | 'checkPostSpam'
+
+export type CheckPostSpamArgs = {
+  post_id: string
+  user_id: string
+  ip: string
+}

packages/velog-cron/src/services/PostService/index.ts

@@ -1,16 +1,25 @@
 import { DbService } from '@lib/db/DbService.js'
+import { CheckPostSpamArgs } from '@lib/redis/RedisService'
 import { Post, Prisma } from '@prisma/client'
 import { injectable, singleton } from 'tsyringe'
+import geoip from 'geoip-country'
+import { subMonths } from 'date-fns'
+import { DiscordService } from '@lib/discord/DiscordService'
 
 interface Service {
   findById(postId: string): Promise<Post | null>
   scoreCalculator(postId: string): Promise<void>
+  checkSpam(args: CheckPostSpamArgs): Promise<void>
 }
 
 @singleton()
 @injectable()
 export class PostService implements Service {
-  constructor(private readonly db: DbService) {}
+  constructor(
+    private readonly db: DbService,
+    private readonly discord: DiscordService,
+  ) {}
+
   public async findById(postId: string): Promise<Post | null> {
     const post = await this.db.post.findUnique({
       where: {
@@ -19,6 +28,7 @@ export class PostService implements Service {
     })
     return post
   }
+
   public async findByUserId({ userId, ...queries }: FindByUserIdParams): Promise<Post[]> {
     const { where, ...query } = queries
     const posts = await this.db.post.findMany({
@@ -30,6 +40,7 @@ export class PostService implements Service {
     })
     return posts
   }
+
   public async scoreCalculator(postId: string): Promise<void> {
     const post = await this.findById(postId)
 
@@ -59,6 +70,190 @@ export class PostService implements Service {
       },
     })
   }
+
+  public async checkSpam({ post_id, user_id, ip }: CheckPostSpamArgs): Promise<void> {
+    const post = await this.db.post.findUnique({
+      where: {
+        id: post_id,
+      },
+      include: {
+        postTags: {
+          include: {
+            tag: true,
+          },
+        },
+      },
+    })
+
+    if (!post) {
+      throw new Error('Not found Post')
+    }
+
+    const user = await this.db.user.findUnique({
+      where: {
+        id: user_id,
+      },
+      include: {
+        profile: true,
+      },
+    })
+
+    if (!user) {
+      throw new Error('Not found User')
+    }
+
+    const country = geoip.lookup(ip)?.country ?? ''
+
+    const extraText = post.postTags
+      .flatMap((postTag) => postTag.tag)
+      .map((tag) => tag?.name ?? '')
+      .join('')
+      .concat(user.profile?.short_bio ?? '', user.profile?.display_name ?? '')
+
+    const isSpam = await this.checkIsSpam(post.title ?? '', post.body ?? '', extraText, country)
+
+    if (!isSpam) return
+
+    this.db.post.update({
+      where: {
+        id: post.id,
+      },
+      data: {
+        is_private: true,
+      },
+    })
+
+    const message = {
+      text: `*userId*: ${user_id}\ntitle: ${post.title}, ip: ${ip}, country: ${country} type: isSpam`,
+    }
+
+    this.discord.sendMessage('spam', JSON.stringify(message))
+  }
+  private async checkIsSpam(
+    title: string,
+    body: string,
+    extraText: string,
+    country: string,
+  ): Promise<boolean> {
+    const allowList = ['KR', 'GB', '']
+    const blockList = ['IN', 'PK', 'CN', 'VN', 'TH', 'PH']
+    const isForeign = !allowList.includes(country)
+
+    if (blockList.includes(country)) {
+      return true
+    }
+
+    const checkTitle = await this.spamFilter(title!, isForeign, true)
+
+    if (checkTitle) {
+      return true
+    }
+
+    const checkBody = await this.spamFilter(body!.concat(extraText), isForeign)
+
+    if (checkBody) {
+      return true
+    }
+
+    return false
+  }
+
+  private async spamFilter(text: string, isForeign: boolean, isTitle = false): Promise<boolean> {
+    const includesCN = /[\u4e00-\u9fa5]/.test(text)
+    const includesKR = /[ㄱ-ㅎ|ㅏ-ㅣ|가-힣]/.test(text)
+
+    if (includesCN && !includesKR) {
+      return true
+    }
+
+    let replaced = text.replace(/```([\s\S]*?)```/g, '') // remove code blocks
+    // replace image markdown
+    replaced = replaced.replace(/!\[([\s\S]*?)\]\(([\s\S]*?)\)/g, '')
+
+    const alphanumericKorean = replaced
+      .replace(/[^a-zA-Zㄱ-힣0-9 \n]/g, '') // remove non-korean
+      .toLowerCase()
+
+    const hasLink = /http/.test(replaced)
+
+    if (!isTitle && isForeign && hasLink) {
+      const lines = replaced.split('\n').filter((line) => line.trim().length > 1)
+      const koreanLinesCount = lines.filter((line) => this.hasKorean(line)).length
+      const confidence = koreanLinesCount / lines.length
+      return confidence < 0.3
+    }
+
+    const spaceReplaced = alphanumericKorean.replace(/\s/g, '')
+
+    const oneMonthAgo = subMonths(new Date(), 1)
+    const bannedKeywords = await this.db.dynamicConfigItem.findMany({
+      where: {
+        type: 'bannedKeyword',
+        last_used_at: {
+          gte: oneMonthAgo,
+        },
+      },
+    })
+
+    const removeDuplicated = Array.from(
+      new Set([text, alphanumericKorean, spaceReplaced].join(',').split(' ')),
+    )
+    const keywordsToUpdate: string[] = []
+
+    const checkKeyword = bannedKeywords
+      .map((keyword) => keyword.value)
+      .some((keyword) => {
+        if (removeDuplicated.includes(keyword)) {
+          keywordsToUpdate.push(keyword)
+          return true
+        } else {
+          return false
+        }
+      })
+
+    if (keywordsToUpdate.length > 0) {
+      this.db.dynamicConfigItem.updateMany({
+        where: {
+          value: {
+            in: keywordsToUpdate,
+          },
+        },
+        data: {
+          last_used_at: new Date(),
+          usage_count: {
+            increment: 1,
+          },
+        },
+      })
+    }
+
+    if (checkKeyword) {
+      return true
+    }
+
+    const bannedAltKeywords = await this.db.dynamicConfigItem.findMany({
+      where: {
+        type: 'bannedAltKeyword',
+      },
+    })
+
+    const score = bannedAltKeywords
+      .map(({ value }) => value)
+      .reduce((acc, current) => {
+        if (alphanumericKorean.includes(current)) {
+          return acc + 1
+        }
+        return acc
+      }, 0)
+
+    if (score >= 2 && isForeign) {
+      return true
+    }
+    return false
+  }
+  private hasKorean(text: string) {
+    return /[ㄱ-힣]/g.test(text)
+  }
 }
 
 type FindByUserIdParams = {

packages/velog-scripts/lib/blockList/BlockListService.mts

@@ -14,14 +14,14 @@ export class BlockListService implements Service {
     await this.db.dynamicConfigItem.create({
       data: {
         value: username,
-        type: 'username',
+        type: 'blockUsername',
       },
     })
   }
   public async readBlockList() {
     const blockList = await this.db.dynamicConfigItem.findMany({
       where: {
-        type: 'username',
+        type: 'blockUsername',
       },
     })
     return blockList.map((item) => item.value)

packages/velog-scripts/scripts/addBannedKeywords.mts

@@ -10,7 +10,7 @@ class Runner {
   public async run(bannedKeywords: string[]) {
     const data = bannedKeywords.map((value) => ({
       value,
-      type: 'banned',
+      type: 'bannedKeyword',
     }))
 
     try {

packages/velog-server/src/env.ts

@@ -3,8 +3,6 @@ import { existsSync } from 'fs'
 import { z } from 'zod'
 import { fileURLToPath } from 'url'
 import { dirname, join } from 'path'
-import { container } from 'tsyringe'
-import { DbService } from '@lib/db/DbService.js'
 
 type DockerEnv = 'development' | 'stage' | 'production'
 type AppEnvironment = 'development' | 'production'
@@ -85,13 +83,9 @@ const env = z.object({
   discordErrorChannel: z.string(),
   discordSpamChannel: z.string(),
   turnstileSecretKey: z.string(),
-  bannedKeywords: z.array(z.string()),
-  bannedAltKeywords: z.array(z.string()),
   graphcdnToken: z.string(),
 })
 
-const { bannedKeywords, bannedAltKeywords } = await readEnvFromDatabase()
-
 export const ENV = env.parse({
   dockerEnv,
   appEnv,
@@ -131,17 +125,5 @@ export const ENV = env.parse({
   discordErrorChannel: process.env.DISCORD_ERROR_CHANNEL,
   discordSpamChannel: process.env.DISCORD_SPAM_CHANNEL,
   turnstileSecretKey: process.env.TURNSTILE_SECRET_KEY,
-  bannedKeywords: bannedKeywords,
-  bannedAltKeywords: bannedAltKeywords,
   graphcdnToken: process.env.GRAPHCDN_TOKEN,
 })
-
-async function readEnvFromDatabase() {
-  const db = container.resolve(DbService)
-  const items = await db.dynamicConfigItem.findMany()
-
-  return {
-    bannedKeywords: items.filter((item) => item.type === 'banned').map((item) => item.value),
-    bannedAltKeywords: items.filter((item) => item.type === 'bannedAlt').map((item) => item.value),
-  }
-}