引入bleve全文搜索引擎，创建位置报表索引，支持位置报表全文搜索

zhangqian

2023-11-17 844f645172ebde7e94307004647e4a41f71030be

引入bleve全文搜索引擎，创建位置报表索引，支持位置报表全文搜索

 .gitignore

@@ -28,3 +28,4 @@
logs
wms*
WMS*
*.bleve

 controllers/report_forms_controller.go

@@ -10,6 +10,7 @@
    "wms/models"
    "wms/request"
    "wms/response"
    "wms/service"
)

type ReportFormsController struct {
@@ -245,12 +246,20 @@
            ids = append(ids, location.Id)
        }
    }

    amounts, total, err := models.NewLocationProductAmountSearch().SetPage(params.Page, params.PageSize).SetPreload(true).SetKeyword(params.KeyWord).SetProductId(params.ProductId).SetLocationIds(ids).FindByPage()
    var (
        amounts []*models.LocationProductAmount
        total   int64
        err     error
    )
    if params.KeyWord != "" {
        amounts, total, err = service.SearchLocationReport(params.KeyWord, params.Page, params.PageSize)
    } else {
        amounts, total, err = models.NewLocationProductAmountSearch().SetPage(params.Page, params.PageSize).SetPreload(true).SetKeyword(params.KeyWord).SetProductId(params.ProductId).SetLocationIds(ids).FindByPage()
    if err != nil {
        util.ResponseFormat(c, code.RequestParamError, "查询数量失败")
        return
    }
    }

    var result []response.LocationForms
    for _, amount := range amounts {

 go.mod

@@ -31,8 +31,26 @@
require (
    github.com/KyleBanks/depth v1.2.1 // indirect
    github.com/OneOfOne/xxhash v1.2.8 // indirect
    github.com/RoaringBitmap/roaring v1.2.3 // indirect
    github.com/agnivade/levenshtein v1.1.1 // indirect
    github.com/beorn7/perks v1.0.1 // indirect
    github.com/bits-and-blooms/bitset v1.2.0 // indirect
    github.com/blevesearch/bleve/v2 v2.3.10 // indirect
    github.com/blevesearch/bleve_index_api v1.0.6 // indirect
    github.com/blevesearch/geo v0.1.18 // indirect
    github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
    github.com/blevesearch/gtreap v0.1.1 // indirect
    github.com/blevesearch/mmap-go v1.0.4 // indirect
    github.com/blevesearch/scorch_segment_api/v2 v2.1.6 // indirect
    github.com/blevesearch/segment v0.9.1 // indirect
    github.com/blevesearch/snowballstem v0.9.0 // indirect
    github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
    github.com/blevesearch/vellum v1.0.10 // indirect
    github.com/blevesearch/zapx/v11 v11.3.10 // indirect
    github.com/blevesearch/zapx/v12 v12.3.10 // indirect
    github.com/blevesearch/zapx/v13 v13.3.10 // indirect
    github.com/blevesearch/zapx/v14 v14.3.10 // indirect
    github.com/blevesearch/zapx/v15 v15.3.13 // indirect
    github.com/bytedance/sonic v1.9.2 // indirect
    github.com/cespare/xxhash/v2 v2.2.0 // indirect
    github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
@@ -52,6 +70,7 @@
    github.com/go-sql-driver/mysql v1.7.0 // indirect
    github.com/gobwas/glob v0.2.3 // indirect
    github.com/goccy/go-json v0.10.2 // indirect
    github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
    github.com/golang/protobuf v1.5.3 // indirect
    github.com/golang/snappy v0.0.4 // indirect
    github.com/gorilla/mux v1.8.0 // indirect
@@ -70,6 +89,7 @@
    github.com/mitchellh/mapstructure v1.5.0 // indirect
    github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
    github.com/modern-go/reflect2 v1.0.2 // indirect
    github.com/mschoch/smat v0.2.0 // indirect
    github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
    github.com/pelletier/go-toml/v2 v2.0.9 // indirect
    github.com/prometheus/client_golang v1.16.0 // indirect
@@ -88,7 +108,9 @@
    github.com/ugorji/go/codec v1.2.11 // indirect
    github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
    github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
    github.com/yanyiwu/gojieba v1.3.0 // indirect
    github.com/yashtewari/glob-intersection v0.2.0 // indirect
    go.etcd.io/bbolt v1.3.7 // indirect
    go.opentelemetry.io/otel v1.19.0 // indirect
    go.opentelemetry.io/otel/metric v1.19.0 // indirect
    go.opentelemetry.io/otel/sdk v1.19.0 // indirect

 go.sum

@@ -48,6 +48,10 @@
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY=
github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
@@ -56,6 +60,40 @@
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg=
github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA=
github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8=
github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw=
github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA=
github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc=
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A=
github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 h1:3uZCA/BLTIu+DqCfguByNMJa2HVHpXvjfy0Dy7g6fuA=
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
github.com/bytedance/sonic v1.9.2 h1:GDaNjuWSGu09guE9Oql0MSTNhNCLlWwO8y/xM5BzcbM=
@@ -110,6 +148,7 @@
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -146,6 +185,8 @@
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -230,10 +271,12 @@
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kirinlabs/HttpRequest v1.1.1 h1:eBbFzpRd/Y7vQhRY30frHK3yAJiT1wDlB31Ryzyklc0=
github.com/kirinlabs/HttpRequest v1.1.1/go.mod h1:XV38fA4rXZox83tlEV9KIQ7Cdsut319x6NGzVLuRlB8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -270,6 +313,9 @@
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -347,6 +393,8 @@
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
github.com/yanyiwu/gojieba v1.3.0 h1:6VeaPOR+MawnImdeSvWNr7rP4tvUfnGlEKaoBnR33Ds=
github.com/yanyiwu/gojieba v1.3.0/go.mod h1:54wkP7sMJ6bklf7yPl6F+JG71dzVUU1WigZbR47nGdY=
github.com/yashtewari/glob-intersection v0.2.0 h1:8iuHdN88yYuCzCdjt0gDe+6bAhUwBeEWqThExu54RFg=
github.com/yashtewari/glob-intersection v0.2.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -355,6 +403,8 @@
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

 main.go

@@ -17,6 +17,7 @@
    "wms/proto/product_inventory"
    "wms/proto/purchase_wms"
    "wms/router"
    "wms/service"
)

func main() {
@@ -66,6 +67,9 @@
            panic(fmt.Sprintf("grpc server init error: %v", err.Error()))
        }
    }()

    go service.InitLocationReportData()

    logx.Error(server.ListenAndServe().Error())
}


 pkg/blevex/analyzer.go

New file
@@ -0,0 +1,62 @@
package blevex

import (
    "errors"

    "github.com/blevesearch/bleve/v2/analysis"
    "github.com/blevesearch/bleve/v2/registry"
)

func analyzerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {
    tokenizerName, ok := config["tokenizer"].(string)
    if !ok {
        return nil, errors.New("must specify tokenizer")
    }
    tokenizer, err := cache.TokenizerNamed(tokenizerName)
    if err != nil {
        return nil, err
    }

    jbtk, ok := tokenizer.(*JiebaTokenizer)
    if !ok {
        return nil, errors.New("tokenizer must be of type jieba")
    }
    alz := &JiebaAnalyzer{
        Tokenizer: jbtk,
    }
    return alz, nil
}

func init() {
    registry.RegisterAnalyzer("gojieba", analyzerConstructor)
}

// JiebaAnalyzer from analysis.DefaultAnalyzer
type JiebaAnalyzer struct {
    CharFilters  []analysis.CharFilter
    Tokenizer    *JiebaTokenizer
    TokenFilters []analysis.TokenFilter
}

func (a *JiebaAnalyzer) Analyze(input []byte) analysis.TokenStream {
    if a.CharFilters != nil {
        for _, cf := range a.CharFilters {
            input = cf.Filter(input)
        }
    }
    tokens := a.Tokenizer.Tokenize(input)
    if a.TokenFilters != nil {
        for _, tf := range a.TokenFilters {
            tokens = tf.Filter(tokens)
        }
    }
    return tokens
}

func (a *JiebaAnalyzer) Free() {
    if a.Tokenizer != nil {
        a.Tokenizer.Free()
    } else {
        panic("JiebaAnalyzer.Tokenizer is nil, this should not happen")
    }
}

 pkg/blevex/bleve.go

New file
@@ -0,0 +1,105 @@
package blevex

import (
    "github.com/blevesearch/bleve/v2"
    "github.com/blevesearch/bleve/v2/mapping"
    "github.com/yanyiwu/gojieba"
    "sync"
)

// InitAnalyzer 加载自定义分词器（结巴分词）

var defaultAnalyzer *mapping.IndexMappingImpl

func InitAnalyzer() {
    indexMapping := bleve.NewIndexMapping()
    //os.RemoveAll(IndexDir)
    //// clean index when example finished
    //defer os.RemoveAll(IndexDir)

    err := indexMapping.AddCustomTokenizer("gojieba",
        map[string]interface{}{
            "dictpath":     gojieba.DICT_PATH,
            "hmmpath":      gojieba.HMM_PATH,
            "userdictpath": gojieba.USER_DICT_PATH,
            "idf":          gojieba.IDF_PATH,
            "stop_words":   gojieba.STOP_WORDS_PATH,
            "type":         "gojieba",
        },
    )
    if err != nil {
        panic(err)
    }
    err = indexMapping.AddCustomAnalyzer("gojieba",
        map[string]interface{}{
            "type":      "gojieba",
            "tokenizer": "gojieba",
        },
    )
    if err != nil {
        panic(err)
    }
    indexMapping.DefaultAnalyzer = "gojieba"
    defaultAnalyzer = indexMapping
}

var indexMap sync.Map

func NewIndex(indexName string) (bleve.Index, error) {
    if defaultAnalyzer == nil {
        InitAnalyzer()
    }
    index, err := bleve.New(indexName, defaultAnalyzer)
    if err == bleve.ErrorIndexPathExists {
        index, err = bleve.Open(indexName)
        if err != nil {
            return nil, err
        }
        return index, nil
    } else if err != nil {
        return nil, err
    }
    return index, nil
}

func LoadIndex(indexName string) (bleve.Index, error) {
    if v, ok := indexMap.Load(indexName); ok {
        return v.(bleve.Index), nil
    }
    index, err := NewIndex(indexName)
    if err == nil {
        indexMap.Store(indexName, index)
    }
    return index, err
}
func Search(indexName string, keyword string) (ids []string, err error) {
    index, err := LoadIndex(indexName)
    if err != nil {
        return nil, err
    }
    req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(keyword))
    res, err := index.Search(req)
    if err != nil {
        panic(err)
    }
    if res == nil || res.Total == 0 {
        return ids, nil
    }
    for _, ret := range dealResult(res) {
        ids = append(ids, ret.Id)
    }
    return ids, nil
}

type Result struct {
    Id    string  `json:"id"`
    Score float64 `json:"score"`
}

func dealResult(res *bleve.SearchResult) []Result {
    var results []Result
    for _, item := range res.Hits {
        results = append(results, Result{item.ID, item.Score})
    }
    return results
}

 pkg/blevex/operate.go

New file
@@ -0,0 +1,37 @@
package blevex

func Add(indexName string, id string, data interface{}) error {
    i, err := LoadIndex(indexName)
    if err != nil {
        return err
    }
    return i.Index(id, data)
}

func Delete(indexName string, id string) error {
    i, err := LoadIndex(indexName)
    if err != nil {
        return err
    }
    return i.Delete(id)
}

func Exists(indexName string, id string) (ok bool, err error) {
    i, err := LoadIndex(indexName)
    if err != nil {
        return false, err
    }
    doc, err := i.Document(id)
    if err != nil {
        return false, err
    }
    return doc.ID() == id, nil
}

func DocCount(indexName string) (uint64, error) {
    i, err := LoadIndex(indexName)
    if err != nil {
        return 0, err
    }
    return i.DocCount()
}

 pkg/blevex/tokenizer.go

New file
@@ -0,0 +1,71 @@
package blevex

import (
    "errors"

    "github.com/blevesearch/bleve/v2/analysis"
    "github.com/blevesearch/bleve/v2/registry"
    "github.com/yanyiwu/gojieba"
)

type JiebaTokenizer struct {
    handle *gojieba.Jieba
}

var _ analysis.Tokenizer = &JiebaTokenizer{}

func NewJiebaTokenizer(dictpath, hmmpath, userdictpath, idf, stop_words string) *JiebaTokenizer {
    x := gojieba.NewJieba(dictpath, hmmpath, userdictpath, idf, stop_words)
    return &JiebaTokenizer{x}
}

func (x *JiebaTokenizer) Free() {
    x.handle.Free()
}

// Analyze([]byte) TokenStream
func (x *JiebaTokenizer) Tokenize(sentence []byte) analysis.TokenStream {
    result := make(analysis.TokenStream, 0)
    pos := 1
    words := x.handle.Tokenize(string(sentence), gojieba.SearchMode, true)
    for _, word := range words {
        token := analysis.Token{
            Term:     []byte(word.Str),
            Start:    word.Start,
            End:      word.End,
            Position: pos,
            Type:     analysis.Ideographic,
        }
        result = append(result, &token)
        pos++
    }
    return result
}

func tokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
    dictpath, ok := config["dictpath"].(string)
    if !ok {
        return nil, errors.New("config dictpath not found")
    }
    hmmpath, ok := config["hmmpath"].(string)
    if !ok {
        return nil, errors.New("config hmmpath not found")
    }
    userdictpath, ok := config["userdictpath"].(string)
    if !ok {
        return nil, errors.New("config userdictpath not found")
    }
    idf, ok := config["idf"].(string)
    if !ok {
        return nil, errors.New("config idf not found")
    }
    stop_words, ok := config["stop_words"].(string)
    if !ok {
        return nil, errors.New("config stop_words not found")
    }
    return NewJiebaTokenizer(dictpath, hmmpath, userdictpath, idf, stop_words), nil
}

func init() {
    registry.RegisterTokenizer("gojieba", tokenizerConstructor)
}

 service/search.go

New file
@@ -0,0 +1,66 @@
package service

import (
    "github.com/spf13/cast"
    "strconv"
    "wms/models"
    "wms/pkg/blevex"
    "wms/pkg/logx"
)

type LocationReport struct {
    LocationProductAmount int
    LocationName          string `json:"name" gorm:"index;type:varchar(255);not null;comment:位置名称"` //位置名称
    LocationJointName     string `json:"jointName" gorm:"type:varchar(255);comment:拼接名称"`           //拼接名称
    MaterialName          string `json:"materialName"`                                              //物料名称
    ProductCategoryName   string `json:"productCategoryName"`                                       //分类名称
}

const (
    LocationReportIndexName = "locationReport.bleve"
)

func InitLocationReportData() {
    docCount, err := blevex.DocCount(LocationReportIndexName)
    if err != nil {
        logx.Errorf("InitLocationReportData get doc count err:%v", err)
        return
    }
    if docCount > 0 {
        return
    }

    reports := make([]*LocationReport, 0)
    search := models.NewLocationProductAmountSearch()
    err = search.Orm.Model(&models.LocationProductAmount{}).
        Raw("select wms_location_product_amount.id as LocationProductAmount, wms_location.name as LocationName, wms_location.joint_name as LocationJointName, material.name as MaterialName, wms_product_category.Name as ProductCategoryName from wms_location_product_amount " +
            "left join wms_location on wms_location.id=wms_location_product_amount.location_id " +
            "left join material on material.id = wms_location_product_amount.product_id " +
            "left join wms_product_category on wms_product_category.id=material.category_id").Scan(&reports).Error
    if err != nil {
        logx.Errorf("InitLocationReportData scan err:%v", err)
    }
    for _, result := range reports {
        err = blevex.Add(LocationReportIndexName, strconv.Itoa(result.LocationProductAmount), result)
        if err != nil {
            logx.Errorf("InitLocationReportData add failed, err:%v, index:%v, data:%v", err, LocationReportIndexName, result)
        }
    }
    return
}

func SearchLocationReport(keyword string, page, pageSize int) (list []*models.LocationProductAmount, total int64, err error) {
    ids, err := blevex.Search(LocationReportIndexName, keyword)
    if err != nil {
        return
    }
    if len(ids) == 0 {
        return
    }
    recordIds := make([]int, 0, len(ids))
    for _, id := range ids {
        recordIds = append(recordIds, cast.ToInt(id))
    }
    list, total, err = models.NewLocationProductAmountSearch().SetPage(page, pageSize).SetPreload(true).SetIds(recordIds).FindByPage()
    return
}