From 72ccf8a395203671f8a1e80d04c7418544f315f9 Mon Sep 17 00:00:00 2001
From: sunty <1172534965@qq.com>
Date: 星期四, 03 六月 2021 11:47:30 +0800
Subject: [PATCH] reptile first commit
---
code/code.go | 17 +++
controllers/reptileControllers.go | 108 +++++++++++++++++++++
go.mod | 16 +++
main.go | 60 ++++++++++++
util/util.go | 21 ++++
router/router.go | 20 ++++
6 files changed, 242 insertions(+), 0 deletions(-)
diff --git a/code/code.go b/code/code.go
new file mode 100644
index 0000000..f663f88
--- /dev/null
+++ b/code/code.go
@@ -0,0 +1,17 @@
+package code
+
+import "net/http"
+
+// Code 閿欒杈撳嚭鏁版嵁缁撴瀯
+type Code struct {
+ Status int `json:"status"` // HTTP 鐘舵��
+ Success bool `json:"success"` // 鎴愬姛鎴栬�呭け璐�
+ Message string `json:"msg"` // 鎻忚堪淇℃伅
+}
+
+var (
+ // Success 璇锋眰澶勭悊鎴愬姛
+ Success = &Code{http.StatusOK, true, "璇锋眰澶勭悊鎴愬姛"}
+ // RequestParamError 璇锋眰鍙傛暟閿欒
+ RequestParamError = &Code{http.StatusBadRequest, false, "璇锋眰鍙傛暟鏈夎"}
+)
diff --git a/controllers/reptileControllers.go b/controllers/reptileControllers.go
new file mode 100644
index 0000000..1477728
--- /dev/null
+++ b/controllers/reptileControllers.go
@@ -0,0 +1,108 @@
+package controllers
+
+import (
+ "fmt"
+ "github.com/gin-gonic/gin"
+ "github.com/gocolly/colly"
+ "log"
+ "reptile/code"
+ "reptile/util"
+)
+
+type ReptileControllers struct{}
+
+type FieldSelectorInfo struct {
+ Name string `json:"name"`
+ Selector string `json:"selector"`
+}
+
+type FieldResult struct {
+ Name string
+ Value string
+}
+
+type ReptileInfo struct {
+ Url string `json:"url"`
+ UserName string `json:"userName"`
+ UserPassWD string `json:"userPassWD"`
+ TitleSelectorStr string `json:"titleSelectorStr"`
+ ListSelectorStr string `json:"listSelectorStr"`
+ ListRequest bool `json:"listRequest"`
+ FspArray []FieldSelectorInfo `json:"fspArray"`
+}
+
+func (rc *ReptileControllers) Reptile(c *gin.Context) {
+ var rpif ReptileInfo
+ c.BindJSON(&rpif)
+ url := rpif.Url
+ titleSelectorStr := rpif.TitleSelectorStr
+ listSelectorStr := rpif.ListSelectorStr
+ fspArray := rpif.FspArray
+ listRequest := rpif.ListRequest
+ if listRequest == true {
+ if url != "" && titleSelectorStr != "" && listSelectorStr != "" && fspArray != nil && len(fspArray) > 0 {
+ res := ReptileList(url, titleSelectorStr, listSelectorStr, fspArray)
+ util.ResponseFormat(c, code.Success, res)
+ return
+ } else {
+ util.ResponseFormat(c, code.RequestParamError, "璇锋眰鍙傛暟鏈夎")
+ return
+ }
+ } else {
+ if url != "" && titleSelectorStr != "" && fspArray != nil && len(fspArray) > 0 {
+ res := ReptileNoList(url, titleSelectorStr, fspArray)
+ util.ResponseFormat(c, code.Success, res)
+ return
+ } else {
+ util.ResponseFormat(c, code.RequestParamError, "璇锋眰鍙傛暟鏈夎")
+ return
+ }
+ }
+ return
+}
+
+func ReptileList(url string, titleSelectorStr string, listSelectorStr string, fspArray []FieldSelectorInfo) interface{} {
+ cy := colly.NewCollector()
+ rfr := make([]interface{}, 0)
+ cy.OnHTML(titleSelectorStr, func(e *colly.HTMLElement) {
+ fmt.Println("+++++++++++++++++++++++++++")
+ e.ForEach(listSelectorStr, func(i int, ele *colly.HTMLElement) {
+ fr := []FieldResult{}
+ for _, v := range fspArray {
+ fr = append(fr, FieldResult{
+ Name: v.Name,
+ Value: ele.ChildText(v.Selector),
+ })
+ }
+ rfr = append(rfr, fr)
+ })
+ })
+ err := cy.Request("GET", url, nil, nil, nil)
+ if err != nil {
+ log.Fatal("connect error: ", err)
+ }
+ return rfr
+}
+
+func ReptileNoList(url string, titleSelectorStr string, fspArray []FieldSelectorInfo) []interface{} {
+ c := colly.NewCollector()
+ rfr := make([]interface{}, 0)
+ titleSelectorStr = "body"
+ c.OnHTML(titleSelectorStr, func(e *colly.HTMLElement) {
+ fmt.Println("+++++++++++++++++++++++++++")
+ fr := []FieldResult{}
+ for _, v := range fspArray {
+ fmt.Println("v.Selector: ", v.Selector)
+ fr = append(fr, FieldResult{
+ Name: v.Name,
+ Value: e.DOM.Find(v.Selector).Eq(0).Text(),
+ })
+ }
+ rfr = append(rfr, fr)
+ })
+ err := c.Request("GET", url, nil, nil, nil)
+ if err != nil {
+ log.Fatal("connect error: ", err)
+ }
+ return rfr
+}
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..5aa79ea
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,16 @@
+module reptile
+
+go 1.12
+
+require (
+ github.com/PuerkitoBio/goquery v1.6.0 // indirect
+ github.com/antchfx/htmlquery v1.2.3 // indirect
+ github.com/antchfx/xmlquery v1.3.3 // indirect
+ github.com/gin-gonic/gin v1.6.3
+ github.com/gobwas/glob v0.2.3 // indirect
+ github.com/gocolly/colly v1.2.0
+ github.com/kennygrant/sanitize v1.2.4 // indirect
+ github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+ github.com/temoto/robotstxt v1.1.1 // indirect
+ google.golang.org/appengine v1.6.7 // indirect
+)
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..77a9cae
--- /dev/null
+++ b/main.go
@@ -0,0 +1,60 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "mime/multipart"
+ "net/http"
+ "reptile/router"
+ "time"
+)
+
+func main() {
+ r := router.NewRouter()
+ r.Run("0.0.0.0:7070")
+}
+
+func GetLoginToken(method string, url string, parama []byte) (buf []byte, err error) {
+ //defer elapsed("page")()
+ timeout := time.Duration(100 * time.Second)
+ client := http.Client{
+ Timeout: timeout,
+ }
+ //loginInfo :=`{"login":"stybd","password":"suntianyu2486"}`
+ p := new(bytes.Buffer)
+ w := multipart.NewWriter(p)
+
+ content_type := w.FormDataContentType()
+ //w.WriteField("username","basic")
+ //w.WriteField("password","basic2019")
+ //w.Close()
+
+ //logif := "login=stybd,password=suntianyu2486"
+
+ request, err := http.NewRequest(method, url, p)
+ request.Header.Set("Content-Type", content_type)
+ if err != nil {
+ fmt.Println("build request fail !")
+ return nil, err
+ }
+
+ resp, err := client.Do(request)
+ if err != nil {
+ fmt.Println("request error: ", err)
+ return nil, err
+ }
+
+ defer resp.Body.Close()
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ fmt.Println(err)
+ return nil, err
+ }
+ var info interface{}
+ json.Unmarshal(body, &info)
+ fmt.Println("access_token: ", info)
+ fmt.Println("body: ", resp.StatusCode)
+ return body, nil
+}
diff --git a/router/router.go b/router/router.go
new file mode 100644
index 0000000..03464e2
--- /dev/null
+++ b/router/router.go
@@ -0,0 +1,20 @@
+package router
+
+import (
+ "github.com/gin-gonic/gin"
+ "reptile/controllers"
+)
+
+func NewRouter() *gin.Engine {
+ r := gin.Default()
+
+ reptile := new(controllers.ReptileControllers)
+
+ urlPrefix := "/basic/api-v"
+
+ reptileApi := r.Group(urlPrefix + "/reptile")
+ {
+ reptileApi.POST("/reptile", reptile.Reptile)
+ }
+ return r
+}
diff --git a/util/util.go b/util/util.go
new file mode 100644
index 0000000..54b58bf
--- /dev/null
+++ b/util/util.go
@@ -0,0 +1,21 @@
+package util
+
+import (
+ "github.com/gin-gonic/gin"
+ "reptile/code"
+)
+
+//
+// ResponseFormat 杩斿洖鏁版嵁鏍煎紡鍖�
+func ResponseFormat(c *gin.Context, respStatus *code.Code, data interface{}) {
+ if respStatus == nil {
+ respStatus = code.RequestParamError
+ }
+ c.JSON(respStatus.Status, gin.H{
+ "code": respStatus.Status,
+ "success": respStatus.Success,
+ "msg": respStatus.Message,
+ "data": data,
+ })
+ return
+}
--
Gitblit v1.8.0