KnowledgeCrawlerOfSpring4All

Author: jack

build.gradle

@@ -59,6 +59,8 @@ dependencies {
 
 	compile group: 'org.jetbrains.kotlinx', name: 'kotlinx-coroutines-core', version: '0.19.2'
 
+    // https://mvnrepository.com/artifact/com.alibaba/fastjson
+    compile group: 'com.alibaba', name: 'fastjson', version: '1.2.47'
 
 
 }

src/main/kotlin/com/light/saber/controller/KnowledgeCrawController.kt

@@ -1,5 +1,6 @@
 package com.light.saber.controller
 
+import com.light.saber.crawler.KnowledgeCrawlerOfSpring4All
 import com.light.saber.service.CrawKnowledgeService
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.web.bind.annotation.GetMapping
@@ -12,6 +13,10 @@ class KnowledgeCrawController {
     lateinit var CrawKnowledgeService: CrawKnowledgeService
 
 
+    @Autowired
+    lateinit var KnowledgeCrawlerOfSpring4All: KnowledgeCrawlerOfSpring4All
+
+
     @GetMapping("/knowledge/doCrawJianShu")
     fun doCrawJianShu(): String {
         Thread {
@@ -74,5 +79,25 @@ class KnowledgeCrawController {
 
         return "DONE"
     }
+    
+
+    @GetMapping("/knowledge/doCrawBlockChainKnowledge")
+    fun doCrawBlockChainKnowledge(): String {
+        Thread {
+            CrawKnowledgeService.doCrawBlockChainKnowledge()
+        }.start()
+
+        return "DONE"
+    }
+
+
+    @GetMapping("/knowledge/KnowledgeCrawlerOfSpring4All")
+    fun KnowledgeCrawlerOfSpring4All(): String {
+        Thread {
+            KnowledgeCrawlerOfSpring4All.doCraw()
+        }.start()
+
+        return "DONE"
+    }
 
 }

src/main/kotlin/com/light/saber/crawler/KnowledgeCrawlerOfSpring4All.kt

@@ -0,0 +1,49 @@
+package com.light.saber.crawler
+
+import com.alibaba.fastjson.JSON
+import com.alibaba.fastjson.JSONArray
+import com.light.saber.service.KnowledgeService
+import com.light.saber.webclient.CrawlerWebClient
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.beans.factory.annotation.Qualifier
+import org.springframework.scheduling.annotation.Scheduled
+import org.springframework.stereotype.Service
+import java.net.URL
+
+@Service
+class KnowledgeCrawlerOfSpring4All : KnowledgeJsonCrawler {
+    @Autowired
+    @Qualifier("knowledgeService")
+    lateinit var KnowledgeService: KnowledgeService
+    @Autowired
+    lateinit var CrawlerWebClient: CrawlerWebClient
+
+    override fun getArticleBody(e: Element) = e.getElementsByClass("fmt").html()
+
+    override fun pageUrls(page: Int): String = "http://www.spring4all.com/common/articles/${page}"
+
+    override fun getPageJson(url: String) = URL(url).readText(Charsets.UTF_8)
+
+    @Scheduled(cron = "0 30 0 1/1 * ?")
+    override fun doCraw() {
+        for (p in 1..62) {
+            val pageUrl = pageUrls(p)
+            val pageJson = getPageJson(pageUrl)
+            val map = JSON.parse(pageJson) as Map<*, *>
+            val data = map["data"] as Map<*, *>
+            val list = data["list"] as JSONArray
+            list.forEach {
+                val articleUrl = "http://www.spring4all.com/article/${(it as Map<*, *>)["id"]}"
+                val articleTitle = it["title"] as String
+                val articleHTML = CrawlerWebClient.getPageHtmlText(articleUrl)
+                val articleDocument = Jsoup.parse(articleHTML)
+                val articleBody = getArticleBody(articleDocument)
+                println(articleTitle)
+                println(articleUrl)
+                KnowledgeService.doSaveKnowledge(articleUrl, articleTitle, articleBody)
+            }
+        }
+    }
+}

src/main/kotlin/com/light/saber/crawler/KnowledgeJsonCrawler.kt

@@ -0,0 +1,10 @@
+package com.light.saber.crawler
+
+import org.jsoup.nodes.Element
+
+interface KnowledgeJsonCrawler {
+    fun pageUrls(page: Int): String
+    fun getPageJson(url: String): String
+    fun getArticleBody(e: Element): String
+    fun doCraw()
+}

src/main/kotlin/com/light/saber/crawler/KnowledgeWebCrawler.kt

@@ -0,0 +1,17 @@
+package com.light.saber.crawler
+
+import org.jsoup.nodes.Element
+import org.jsoup.select.Elements
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.util.StringUtils
+import java.util.*
+
+interface KnowledgeWebCrawler {
+    fun pageUrls(page: Int): String
+    fun getArticleListDocument(url: String, className: String): Elements
+    fun getArticleUrl(e: Element, className: String): String
+    fun getArticleTitle(e: Element, className: String): String
+    fun getArticleBody(e: Element): String
+
+    fun doCraw()
+}

src/main/kotlin/com/light/saber/service/CrawKnowledgeService.kt

@@ -17,7 +17,8 @@ class CrawKnowledgeService {
     lateinit var KnowledgeDao: KnowledgeDao
     @Autowired
     lateinit var CrawSourceDao: CrawSourceDao
-
+    @Autowired
+    lateinit var CrawlerWebClient: CrawlerWebClient
 
     fun doCrawBlockChainKnowledge() {
         for (page in 0..40) {

src/main/kotlin/com/light/saber/service/KnowledgeService.kt

@@ -6,4 +6,6 @@ import org.springframework.data.domain.Pageable
 
 interface KnowledgeService {
     abstract fun page(title: String?, page: Pageable): Page<Knowledge>
+    fun doSaveKnowledge(articleUrl: String, articleTitle: String, articleBody: String?)
+
 }

src/main/kotlin/com/light/saber/service/KnowledgeServiceImpl.kt

@@ -7,9 +7,27 @@ import org.springframework.data.domain.Page
 import org.springframework.data.domain.Pageable
 import org.springframework.stereotype.Service
 import org.springframework.util.StringUtils
+import java.util.*
 
-@Service
+@Service("knowledgeService")
 class KnowledgeServiceImpl : KnowledgeService {
+    override fun doSaveKnowledge(url: String, title: String, content: String?) {
+        if (StringUtils.isEmpty(url) || StringUtils.isEmpty(title) || StringUtils.isEmpty(content)) {
+            return
+        }
+
+        val Knowledge = Knowledge()
+        Knowledge.title = title ?: ""
+        Knowledge.content = content ?: ""
+        Knowledge.gmtCreate = Date()
+        Knowledge.gmtModified = Date()
+        try {
+            KnowledgeDao.save(Knowledge)
+        } catch (e: Exception) {
+            e.printStackTrace()
+        }
+    }
+
     @Autowired lateinit var KnowledgeDao: KnowledgeDao
     override fun page(title: String?, page: Pageable): Page<Knowledge> {
         if (StringUtils.isEmpty(title)) {

src/main/kotlin/com/light/saber/webclient/CrawlerWebClient.kt

@@ -3,30 +3,27 @@ package com.light.saber.webclient
 
 import com.gargoylesoftware.htmlunit.WebClient
 import com.gargoylesoftware.htmlunit.html.HtmlPage
+import org.springframework.stereotype.Service
 
-
-object CrawlerWebClient {
-    private var webClient: WebClient? = null
+@Service
+class CrawlerWebClient {
 
     private fun instanceWebClient(javaScriptTimeout: Long): WebClient {
-        if (webClient == null) {
-            webClient = WebClient()
-        }
+        val webClient = WebClient()
         if (javaScriptTimeout > 0) {
-            webClient?.javaScriptTimeout = javaScriptTimeout
+            webClient.javaScriptTimeout = javaScriptTimeout
         }
-        webClient?.options?.isJavaScriptEnabled = true //启用JS解释器，默认为true
-        webClient?.options?.isCssEnabled = false
-        webClient?.options?.isThrowExceptionOnScriptError = false //js运行错误时，是否抛出异常
-        webClient?.options?.isUseInsecureSSL = true
+        webClient.options.isJavaScriptEnabled = true //启用JS解释器，默认为true
+        webClient.options.isCssEnabled = false
+        webClient.options.isThrowExceptionOnScriptError = false //js运行错误时，是否抛出异常
+        webClient.options.isUseInsecureSSL = true
         return webClient as WebClient
     }
 
-    @Synchronized
     fun getPageHtmlText(url: String): String? {
-        webClient = instanceWebClient(3000)
+        val webClient = instanceWebClient(3000)
         return try {
-            webClient?.getPage<HtmlPage>(url)?.asXml()
+            webClient.getPage<HtmlPage>(url).asXml()
         } catch (e: Exception) {
             null
         }

src/main/resources/templates/common/head.ftl

@@ -58,6 +58,8 @@
                         <dd><a href="/knowledge/doCrawImportNewKnowledge" target="_blank">抓取ImportNew</a></dd>
                         <dd><a href="/knowledge/doCrawCNBlogKnowledge" target="_blank">抓取CNBlog</a></dd>
                         <dd><a href="/knowledge/doCrawInfoQKnowledge" target="_blank">抓取InfoQ</a></dd>
+                        <dd><a href="/knowledge/doCrawBlockChainKnowledge" target="_blank">BlockChain</a></dd>
+                        <dd><a href="/knowledge/KnowledgeCrawlerOfSpring4All" target="_blank">Spring4All</a></dd>
                         <dd><a href="">超链接</a></dd>
                     </dl>
                 </li>