换一个纯go实现的中文分词包

zhangqian

2023-11-22 deb1fb612b471789f5452f9536421ebee0f36155

 pkg/blevex/bleve.go

@@ -4,43 +4,34 @@
   "fmt"
   "github.com/blevesearch/bleve/v2"
   "github.com/blevesearch/bleve/v2/mapping"
   "github.com/yanyiwu/gojieba"
   "sync"
)

// InitAnalyzer 加载自定义分词器（结巴分词）
// InitAnalyzer 加载自定义分词器（sego）

var defaultAnalyzer *mapping.IndexMappingImpl

func InitAnalyzer() {
   indexMapping := bleve.NewIndexMapping()
   //os.RemoveAll(IndexDir)
   //// clean index when example finished
   //defer os.RemoveAll(IndexDir)

   err := indexMapping.AddCustomTokenizer("gojieba",
   err := indexMapping.AddCustomTokenizer("sego",
      map[string]interface{}{
         "dictpath":     gojieba.DICT_PATH,
         "hmmpath":      gojieba.HMM_PATH,
         "userdictpath": gojieba.USER_DICT_PATH,
         "idf":          gojieba.IDF_PATH,
         "stop_words":   gojieba.STOP_WORDS_PATH,
         "type":         "gojieba",
         "dictpath": "conf/dictionary.txt", // 替换为实际的字典路径
         "type":     "sego",
      },
   )
   if err != nil {
      panic(err)
   }
   err = indexMapping.AddCustomAnalyzer("gojieba",
   err = indexMapping.AddCustomAnalyzer("sego",
      map[string]interface{}{
         "type":      "gojieba",
         "tokenizer": "gojieba",
         "type":      "sego",
         "tokenizer": "sego",
      },
   )
   if err != nil {
      panic(err)
   }
   indexMapping.DefaultAnalyzer = "gojieba"
   indexMapping.DefaultAnalyzer = "sego"
   defaultAnalyzer = indexMapping
}