抽取出了公共类，成为单独的工程，以后任何地方都可用

yijiangwen · Jun 7, 2019 · dac64ac · dac64ac
1 parent b4d4375
commit dac64ac
Show file tree

Hide file tree

Showing 36 changed files with 54 additions and 840 deletions.
diff --git a/GolangSpider/common/HttpRequest.go b/GolangSpider/common/HttpRequest.go
diff --git a/GolangSpider/example/cbooo/cbooo.go b/GolangSpider/example/cbooo/cbooo.go
@@ -1,10 +1,9 @@
 package cbooo
 
 import (
-	"GolangSpider/GolangSpider/common"
-	"GolangSpider/GolangSpider/util"
 	"encoding/csv"
 	"github.com/astaxie/beego/logs"
+	"github.com/jaydenwen123/go-util"
 	"os"
 	"regexp"
 	"strconv"
@@ -23,7 +22,7 @@ func Main() {
 func SpiderMovieInfo(movieId string,downloadPoster bool) {
 	//1.发送请求
 	detailUrl:=getDetailUrl(movieId)
-	_, content := common.Request(detailUrl)
+	_, content := util.Request(detailUrl)
 	//fmt.Println(content)
 	//content=TrimSpace(content)
 	//fmt.Println(content)
@@ -44,7 +43,7 @@ func SpiderMovieInfo(movieId string,downloadPoster bool) {
 
 func SpiderCboooHome() {
 	//1.发送http请求
-	_, content := common.Request(cboooUrl)
+	_, content := util.Request(cboooUrl)
 	//fmt.Println(content)
 	//解析CBO实时票房榜数据
 	SpiderBoxOffice(content)

diff --git a/GolangSpider/example/cbooo/movie.go b/GolangSpider/example/cbooo/movie.go
@@ -1,7 +1,7 @@
 package cbooo
 
 import (
-	"GolangSpider/GolangSpider/util"
+	"github.com/jaydenwen123/go-util"
 	"strings"
 )
 

diff --git a/GolangSpider/example/gifshow/hot.go b/GolangSpider/example/gifshow/hot.go
@@ -1,9 +1,8 @@
 package gifshow
 
 import (
-	"GolangSpider/GolangSpider/common"
-	"GolangSpider/GolangSpider/example/kugou"
-	"GolangSpider/GolangSpider/util"
+	"github.com/jaydenwen123/GolangSpider/GolangSpider/example/kugou"
+	"github.com/jaydenwen123/go-util"
 	"fmt"
 	"github.com/astaxie/beego/logs"
 	"github.com/tidwall/gjson"
@@ -36,7 +35,7 @@ func DownloadWithBatch(epoch int)  {
 
 func DownloadHotVideo(id int,batch chan int)  {
 	cost:=util.NewCost(time.Now())
-	jsonStr := common.RequestJsonWithPost(gifUrl, headers, bodyData)
+	jsonStr := util.RequestJsonWithPost(gifUrl, headers, bodyData)
 	//jsonStr:=SendPostRequest(url,headers,bodyData)
 	if !gjson.Valid(jsonStr){
 		logs.Error("获取json数据失败")

diff --git a/GolangSpider/example/gushiwen/db/db.go b/GolangSpider/example/gushiwen/db/db.go
@@ -1,7 +1,7 @@
 package db
 
 import (
-	"GolangSpider/GolangSpider/util"
+	"github.com/jaydenwen123/go-util"
 	"github.com/jinzhu/gorm"
 )
 

diff --git a/GolangSpider/example/gushiwen/poem.go b/GolangSpider/example/gushiwen/poem.go
@@ -2,7 +2,7 @@ package gushiwen
 
 import (
 	"GolangSpider/GolangSpider/example/gushiwen/db"
-	"GolangSpider/GolangSpider/util"
+	"github.com/jaydenwen123/go-util"
 	"fmt"
 	"github.com/astaxie/beego/logs"
 	"io/ioutil"

diff --git a/GolangSpider/example/gushiwen/shiwen.go b/GolangSpider/example/gushiwen/shiwen.go
@@ -1,9 +1,8 @@
 package gushiwen
 
 import (
-	"GolangSpider/GolangSpider/common"
+	"github.com/jaydenwen123/go-util"
 	"GolangSpider/GolangSpider/example/gushiwen/db"
-	"GolangSpider/GolangSpider/util"
 	"github.com/PuerkitoBio/goquery"
 	"github.com/astaxie/beego/logs"
 	_ "github.com/jinzhu/gorm/dialects/postgres"
@@ -13,7 +12,7 @@ import (
 //1.爬取古诗文网，诗文栏目的所有诗文类型，作者，朝代，形式（诗、词、曲、文言文）
 //https://www.gushiwen.org/shiwen/
 func SpiderShiwenKindUrl() ( map[string]string, map[string]string, map[string]string, map[string]string ) {
-	_, resp := common.Request(SHIWEN_URL)
+	_, resp := util.Request(SHIWEN_URL)
 	//fmt.Println(resp)
 	//util.TrimSpace(resp)
 	//利用正则表达式解析出所有的分类（类型、作者、朝代、形式）
@@ -60,7 +59,7 @@ func SpiderShiwenKindUrl() ( map[string]string, map[string]string, map[string]st
 //2.1爬取不同类型诗文下的诗文链接
 //<a href="/gushi/guiyuan.aspx">闺怨</a>
 func SpiderShiwenByType(url string) []*db.Poem {
-	_, resp := common.Request(url)
+	_, resp := util.Request(url)
 	targets := util.MatchTarget(typePoemRe, resp)
 	curUrl:=""
 	poems:=make([]*db.Poem ,0)
@@ -76,7 +75,7 @@ func SpiderShiwenByType(url string) []*db.Poem {
 //2.2爬取不同类型诗文下的诗文链接
 //<a href="/gushi/guiyuan.aspx">闺怨</a>
 func SpiderShiwenUrlByType(url string) []string {
-	_, resp := common.Request(url)
+	_, resp := util.Request(url)
 	targets := util.MatchTarget(typePoemRe, resp)
 	curUrl:=""
 	urls:=make([]string,0)
@@ -93,7 +92,7 @@ func SpiderShiwenUrlByType(url string) []string {
 //原文、译文、注释、背景、赏析、作者介绍
 func SpiderShiwenContent(shiwenUrl string) *db.Poem {
 	poem:=&db.Poem{}
-	reader := common.ResponseWithReader(shiwenUrl)
+	reader := util.ResponseWithReader(shiwenUrl)
 	//采用新的方案来解决
 	//https://github.com/PuerkitoBio/goquery
 	doc, err := goquery.NewDocumentFromReader(reader)

diff --git a/GolangSpider/example/juejin/config_test.go b/GolangSpider/example/juejin/config_test.go
@@ -1,7 +1,7 @@
 package juejin
 
 import (
-	"GolangSpider/GolangSpider/util"
+	"github.com/jaydenwen123/go-util"
 	"fmt"
 	"testing"
 )