package main import ( "flag" "fmt" "log" "os" "strings" "gopkg.in/guregu/null.v3" "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/mapping" _ "xiaoniaokuaiyan.com/xiaoniao/search/analyzer" "xiaoniaokuaiyan.com/xiaoniao/util" ) type DetectItem struct { Name string `db:"name" json:"name"` //MarketPrice float32 `db:"market_price" json:"marketPrice"` } type ProductDB struct { Id string `db:"id" json:"id"` Name string `db:"name" json:"name"` Price int `db:"price" json:"price"` IsRecommend int `db:"is_recommend" json:"isRecommend"` TKeywords null.String `db:"keywords" json:"tkeywords"` Keywords string `db:"-" json:"keywords"` Picture string `db:"picture" json:"picture"` Items string `db:"-" json:"items"` CityIds []string `db:"-" json:"cityIds"` Pinyin string `db:"-" json:"py"` JP string `db:"-" json:"jp"` Cates []string `db:"-" json:"cates"` SaleNum int `db:"sale_num" json:"sale_num"` SortNo int `db:"sort_no" json:"sort_no"` CreatedAt string `db:"created_at" json:"created_at"` TPutawayTime null.String `db:"putaway_time" json:"-"` PutawayTime string `db:"-" json:"putaway_time"` MarketPrice int `db:"market_price" json:"market_price"` } func buildIndexMapping() (*mapping.IndexMappingImpl, error) { indexMapping := bleve.NewIndexMapping() err := indexMapping.AddCustomTokenizer("gojieba", map[string]interface{}{ "dictpath": "dict/jieba.dict.utf8", //gojieba.DICT_PATH, "hmmpath": "dict/hmm_model.utf8", //gojieba.HMM_PATH, "userdictpath": "dict/user.dict.utf8", //gojieba.USER_DICT_PATH, "idf": "dict/idf.utf8", //gojieba.IDF_PATH, "stop_words": "dict/stop_words.utf8", //gojieba.STOP_WORDS_PATH, "type": "gojieba", }, ) if err != nil { panic(err) } err = indexMapping.AddCustomAnalyzer("gojieba", map[string]interface{}{ "type": "gojieba", "tokenizer": "gojieba", }, ) if err != nil { panic(err) } indexMapping.DefaultAnalyzer = "gojieba" pyFieldMapping := mapping.NewTextFieldMapping() pyFieldMapping.Name = "py" pyFieldMapping.Analyzer = "standard" jpFieldMapping := mapping.NewTextFieldMapping() jpFieldMapping.Name = "jp" jpFieldMapping.Analyzer = "standard" cityFieldMapping := mapping.NewTextFieldMapping() cityFieldMapping.Name = "cityIds" cityFieldMapping.Analyzer = "standard" catesMapping := mapping.NewTextFieldMapping() catesMapping.Name = "cates" catesMapping.Analyzer = "standard" pyMapping := mapping.NewDocumentMapping() pyMapping.AddFieldMappingsAt("py", pyFieldMapping) pyMapping.AddFieldMappingsAt("jp", jpFieldMapping) pyMapping.AddFieldMappingsAt("cityIds", cityFieldMapping) pyMapping.AddFieldMappingsAt("cates", catesMapping) pictureFieldMapping := bleve.NewTextFieldMapping() pictureFieldMapping.Index = false pyMapping.AddFieldMappingsAt("picture", pictureFieldMapping) //pyMapping.AddSubDocumentMapping("picture", bleve.NewDocumentStaticMapping()) //pyMapping.AddSubDocumentMapping("picture", bleve.NewDocumentDisabledMapping()) pyMapping.AddSubDocumentMapping("tkeywords", bleve.NewDocumentDisabledMapping()) //indexMapping.AddDocumentMapping("pinyin", pyMapping) indexMapping.DefaultMapping = pyMapping return indexMapping, nil } func dump(idx bleve.Index) { ma := bleve.NewMatchAllQuery() request := bleve.NewSearchRequest(ma) request.Fields = []string{"id", "price"} request.SortBy([]string{"_id"}) request.Size = 1000 dc, _ := idx.DocCount() fmt.Println(dc) sr, err := idx.Search(request) if err != nil { log.Fatal(err) } //fmt.Println(sr) for _, hit := range sr.Hits { fmt.Printf("%s\t,%v\n", hit.Fields["id"], hit.Fields["price"]) } } func main() { isDump := flag.Bool("dump", false, "--dump") flag.Parse() mapping, err := buildIndexMapping() if err != nil { log.Fatal(err) } /*docMapping := mapping.DefaultMapping docMapping.AddSubDocumentMapping("isRecommend", bleve.NewDocumentStaticMapping()) docMapping.AddSubDocumentMapping("picture", bleve.NewDocumentStaticMapping()) */ indexName := "xn_product.bleve" var productIndex bleve.Index var ierr error if _, err := os.Stat(indexName); os.IsNotExist(err) { productIndex, ierr = bleve.New(indexName, mapping) } else if err == nil { if *isDump { productIndex, ierr = bleve.OpenUsing(indexName, map[string]interface{}{ "read_only": true, }) } else { productIndex, ierr = bleve.Open(indexName) } } if ierr != nil { log.Fatal(ierr) } defer productIndex.Close() if *isDump { dump(productIndex) return } /*p := Product{ Id: "200817010121", No: "sdsdfsjosdifkids", Name: "空腹血糖", Tags: "血糖", Tips: "需要空腹测试", } productIndex.Index(p.Id, p) */ strSql := "select id, name,price, is_recommend, picture, keywords,sale_num,putaway_time,sort_no,created_at,market_price from t_product where is_putaway = 0 and is_delete = 0;" db := util.GetSqlDB() productList := []ProductDB{} err = db.Select(&productList, strSql) if err != nil { log.Fatal(err) } for _, product := range productList { strSql := "select t1.name from t_detect_product t1 left join t_product_detect_product t2 on t1.id = t2.detect_product_id where t2.product_id = " + product.Id itemList := []string{} db.Select(&itemList, strSql) //strSql = "select tag_name from t_product_tag t1 left join t_tag t2 on t1.tag_id = t2.id where t1.product_id = ?" //tagList := []string{} //db.Select(&tagList, strSql, product.Id) strSql = "select city_id from v_product_city where product_id = ?" cityIds := []string{} db.Select(&cityIds, strSql, product.Id) product.Items = strings.Join(itemList, " ") //product.Tags = strings.Join(tagList, " ") product.CityIds = cityIds product.Pinyin, product.JP, _ = util.GetPinyin(product.Name) product.Keywords = product.TKeywords.String product.PutawayTime = product.TPutawayTime.String strSql = "SELECT cat_id from t_product_category_product where product_id = ?" var cateIds = []string{} db.Select(&cateIds, strSql, product.Id) product.Cates = cateIds productIndex.Index(product.Id, product) } //bytes, _ := json.Marshal(&productList) //fmt.Println(string(bytes)) }