OSChina文章

Author: universsky

app.sql

@@ -13,6 +13,8 @@ INSERT INTO `saber`.`craw_source` (`type`, `url`) VALUES ('JIAN_SHU', 'http://ww
 INSERT INTO `saber`.`craw_source` (`type`, `url`) VALUES ('JIAN_SHU', 'http://www.jianshu.com/c/04cb7410c597');
 
 
+delete from knowledge where content is null or content ='' or title is null or title='';
+
 	id	type	url
 	2	JIAN_SHU	http://www.jianshu.com/c/498ebcfd27ad
 	3	JIAN_SHU	http://www.jianshu.com/c/c3fe8e7aeb09

src/main/kotlin/com/light/saber/controller/KnowledgeController.kt

@@ -53,10 +53,14 @@ class KnowledgeController {
         } else if (isTitleExist(title)) {
             return Result(title, "问题已经存在，请换一个问题", false)
         } else {
-            knowledge.gmtCreate = Date()
-            knowledge.gmtModified = Date()
-            KnowledgeDao.save(knowledge)
-            return Result(title, "保存成功", true)
+            try {
+                knowledge.gmtCreate = Date()
+                knowledge.gmtModified = Date()
+                KnowledgeDao.save(knowledge)
+                return Result(title, "保存成功", true)
+            } catch (e: Exception) {
+                return Result(title, "系统出错啦:${e.message}", false)
+            }
         }
 
     }

src/main/kotlin/com/light/saber/controller/KnowledgeCrawController.kt

@@ -14,13 +14,29 @@ class KnowledgeCrawController {
 
     @GetMapping("/knowledge/doCrawJianShu")
     fun doCrawJianShu(): String {
-        CrawKnowledgeService.doCrawJianShuKnowledge()
+        Thread {
+            CrawKnowledgeService.doCrawJianShuKnowledge()
+        }.start()
+
+
         return "DONE"
     }
 
     @GetMapping("/knowledge/doCrawSegmentFaultKnowledge")
     fun doCrawSegmentFaultKnowledge(): String {
-        CrawKnowledgeService.doCrawSegmentFaultKnowledge()
+        Thread {
+            CrawKnowledgeService.doCrawSegmentFaultKnowledge()
+        }.start()
+
+        return "DONE"
+    }
+
+    @GetMapping("/knowledge/doCrawOSChinaKnowledge")
+    fun doCrawOSChinaKnowledge(): String {
+        Thread {
+            CrawKnowledgeService.doCrawOSChinaKnowledge()
+        }.start()
+
         return "DONE"
     }
 

src/main/kotlin/com/light/saber/model/Knowledge.kt

@@ -13,7 +13,7 @@ class Knowledge {
     @Column(length = 200, unique = true, nullable = false)
     var title = ""
 
-    @Column(length = 200, unique = true, nullable = false)
+    @Column(length = 200)
     var url = ""
 
     @Lob

src/main/kotlin/com/light/saber/service/CrawKnowledgeService.kt

@@ -4,13 +4,12 @@ import com.light.saber.dao.CrawSourceDao
 import com.light.saber.dao.KnowledgeDao
 import com.light.saber.model.Knowledge
 import com.light.saber.webclient.CrawlerWebClient
-import kotlinx.coroutines.experimental.CommonPool
-import kotlinx.coroutines.experimental.launch
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Service
+import org.springframework.util.StringUtils
 
 @Service
 class CrawKnowledgeService {
@@ -20,27 +19,74 @@ class CrawKnowledgeService {
     lateinit var CrawSourceDao: CrawSourceDao
 
     fun doCrawJianShuKnowledge() {
-
         val 简书专题URLs = CrawSourceDao.findJianShu()
-
         简书专题URLs.forEach {
-            launch(CommonPool) {
-                for (page in 1..100) {
-                    crawJianShuArticles(page, it.url)
-                }
+            for (page in 1..100) {
+                crawJianShuArticles(page, it.url)
             }
         }
     }
 
 
     fun doCrawSegmentFaultKnowledge() {
         for (page in 1..803) {
-            launch(CommonPool) {
-                crawSegmentFault(page)
+            crawSegmentFault(page)
+        }
+    }
+
+    fun doCrawOSChinaKnowledge() {
+        for (page in 1..560) {
+            crawOSChina(page)
+        }
+    }
+
+    private fun crawOSChina(page: Int) {
+        val pageUrl = "https://www.oschina.net/action/ajax/get_more_recommend_blog?classification=0&p=$page"
+        val 文章列表HTML = CrawlerWebClient.getPageHtmlText(pageUrl)
+        val document = Jsoup.parse(文章列表HTML)
+
+//        document.getElementsByClassName("blog-name")[0]
+
+        val titles = arrayListOf<String>()
+
+        document.getElementsByClass("blog-name").forEach {
+            titles.add(it.html())
+        }
+
+//        document.getElementsByClassName("blog-title-link")[0]
+//<a href=​"https:​/​/​my.oschina.net/​u/​3115385/​blog/​1819321" class=​"sc overh blog-title-link" target=​"_blank" title=​"JVM调优-堆大小设置、回收器选择">​…​</a>​
+
+        val links = document.getElementsByClass("blog-title-link")
+
+        if (titles.size != links.size) {
+            return
+        }
+
+        links.forEachIndexed { index, it ->
+            val url = it.attr("href")
+            if (KnowledgeDao.countByUrl(url) == 0) {
+                val OSChina文章HTML = CrawlerWebClient.getPageHtmlText(url)
+                val OSChina文章Document = Jsoup.parse(OSChina文章HTML)
+                val content = 获取OSChina文章内容(OSChina文章Document)
+                println(url)
+                println(content)
+
+                doSaveKnowledge(
+                        url = url,
+                        title = titles[index],
+                        content = content
+                )
+
             }
         }
     }
 
+    private fun 获取OSChina文章内容(osChina文章Document: Document?): String? {
+//        document.getElementById("blogBody")
+        return osChina文章Document?.getElementById("blogBody")?.html()
+    }
+
+
     private fun crawSegmentFault(page: Int) {
         val SegmentFault文章列表的HTML = CrawlerWebClient.getPageHtmlText("https://segmentfault.com/blogs?page=$page")
         val document = Jsoup.parse(SegmentFault文章列表的HTML)
@@ -125,10 +171,15 @@ class CrawKnowledgeService {
     }
 
     private fun doSaveKnowledge(url: String, title: String?, content: String?) {
+        if (StringUtils.isEmpty(url) || StringUtils.isEmpty(title) || StringUtils.isEmpty(content)) {
+            return
+        }
+
         val Knowledge = Knowledge()
         Knowledge.url = url
         Knowledge.title = title ?: ""
         Knowledge.content = content ?: ""
+
         try {
             KnowledgeDao.save(Knowledge)
         } catch (e: Exception) {

src/main/resources/templates/add.ftl

@@ -77,16 +77,22 @@
 
             var title = $('#title').val();
             var content = layedit.getContent(layeditIndex);
+            var data = {title: title, content: content};
+            console.log(data)
 
             $.ajax({
                 url: '/addKnowledge',
-                data: {title: title, content: content},
+                data: data,
                 method: 'POST',
                 async: false,
                 success: (result) => {
-                    layer.msg(result.msg)
-                    //alert(JSON.stringify(result.msg))
-                    location.reload()
+                    if (result.success === true) {
+                        layer.msg(result.msg)
+                        location.reload()
+                    } else {
+                        layer.msg(result.msg)
+                    }
+
                 },
                 error: (err) => {
                     layer.msg(err)

src/main/resources/templates/common/head.ftl

@@ -48,7 +48,7 @@
                     <dl class="layui-nav-child">
                         <dd><a href="/knowledge/doCrawJianShu" target="_blank">抓取简书</a></dd>
                         <dd><a href="/knowledge/doCrawSegmentFaultKnowledge" target="_blank">抓取SegmentFault</a></dd>
-                        <dd><a href="javascript:;">列表三</a></dd>
+                        <dd><a href="/knowledge/doCrawOSChinaKnowledge" target="_blank">抓取OSChina</a></dd>
                         <dd><a href="">超链接</a></dd>
                     </dl>
                 </li>

src/main/resources/templates/detail.ftl

@@ -16,7 +16,7 @@
                                 <h5 class="layui-timeline-title">
                                 ${Knowledge.gmtCreate?string("yyyy-MM-dd HH:mm:ss")}
                                 </h5>
-                                <textarea class="content" id="content-${Knowledge.id}">
+                                <textarea class="content" id="content-${Knowledge.id?string("#")}">
                                 ${Knowledge.content}
                                 </textarea>
                             </div>
@@ -34,7 +34,12 @@
     $(function () {
         layui.use('layedit', function () {
             var layedit = layui.layedit;
-            layedit.build('content-${Knowledge.id}', {height: 1000});
+            layedit.build('content-${Knowledge.id?string("#")}',
+                    {
+                        height: 1000,
+                        tool: []
+                    }
+            );
         });
     })
 </script>

src/main/resources/templates/index.ftl

@@ -29,12 +29,12 @@
                             <i class="layui-icon layui-timeline-axis"></i>
                             <div class="layui-timeline-content layui-text">
                                 <h3 class="layui-timeline-title">
-                                    <a href="/knowledge/${e.id}" target="_blank" style="color: #469689">${e.title}</a>
+                                    <a href="/knowledge/${e.id?string("#")}" target="_blank" style="color: #469689">${e.title}</a>
                                 </h3>
                                 <h5 class="layui-timeline-title">
                                     ${e.gmtCreate?string("yyyy-MM-dd HH:mm:ss")}
                                 </h5>
-                                <textarea class="content" id="content-${e.id}">
+                                <textarea class="content" id="content-${e.id?string("#")}">
                                     ${e.content}
                                 </textarea>
                             </div>
@@ -65,9 +65,9 @@
             var layedit = layui.layedit;
             <#if feeds?exists>
                 <#list feeds as e>
-                layedit.build('content-${e.id}',
+                layedit.build('content-${e.id?string("#")}',
                         {
-                            height: 300,
+                            height: 520,
                             tool: []
                         }
                 );
@@ -79,9 +79,9 @@
         //完整功能
         laypage.render({
             elem: 'pagination'
-            , count: <#if total?exists>${total}<#else>0</#if> //数据总数，从服务端得到
-            , limit: ${pageSize!10}
-            , curr: ${pageNum!1}
+            , count: <#if total?exists>${total?string("#")}<#else>0</#if> //数据总数，从服务端得到
+            , limit: ${pageSize!10?string("#")}
+            , curr: ${pageNum!1?string("#")}
             , layout: ['count', 'prev', 'page', 'next', 'limit', 'refresh', 'skip']
             , jump: function (obj, first) {
                 //obj包含了当前分页的所有参数，比如：