gobackend-qingma/cmd/search_index.go

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strings"

	"gopkg.in/guregu/null.v3"

	"github.com/blevesearch/bleve"
	"github.com/blevesearch/bleve/mapping"
	_ "xiaoniaokuaiyan.com/xiaoniao/search/analyzer"
	"xiaoniaokuaiyan.com/xiaoniao/util"
)

type DetectItem struct {
	Name string `db:"name" json:"name"`
	//MarketPrice float32 `db:"market_price" json:"marketPrice"`
}

type ProductDB struct {
	Id           string      `db:"id" json:"id"`
	Name         string      `db:"name" json:"name"`
	Price        int         `db:"price" json:"price"`
	IsRecommend  int         `db:"is_recommend" json:"isRecommend"`
	TKeywords    null.String `db:"keywords" json:"tkeywords"`
	Keywords     string      `db:"-" json:"keywords"`
	Picture      string      `db:"picture" json:"picture"`
	Items        string      `db:"-" json:"items"`
	CityIds      []string    `db:"-" json:"cityIds"`
	Pinyin       string      `db:"-" json:"py"`
	JP           string      `db:"-" json:"jp"`
	Cates        []string    `db:"-" json:"cates"`
	SaleNum      int         `db:"sale_num" json:"sale_num"`
	SortNo       int         `db:"sort_no" json:"sort_no"`
	CreatedAt    string      `db:"created_at" json:"created_at"`
	TPutawayTime null.String `db:"putaway_time" json:"-"`
	PutawayTime  string      `db:"-" json:"putaway_time"`
	MarketPrice  int         `db:"market_price" json:"market_price"`
}

func buildIndexMapping() (*mapping.IndexMappingImpl, error) {
	indexMapping := bleve.NewIndexMapping()
	err := indexMapping.AddCustomTokenizer("gojieba",
		map[string]interface{}{
			"dictpath":     "dict/jieba.dict.utf8", //gojieba.DICT_PATH,
			"hmmpath":      "dict/hmm_model.utf8",  //gojieba.HMM_PATH,
			"userdictpath": "dict/user.dict.utf8",  //gojieba.USER_DICT_PATH,
			"idf":          "dict/idf.utf8",        //gojieba.IDF_PATH,
			"stop_words":   "dict/stop_words.utf8", //gojieba.STOP_WORDS_PATH,
			"type":         "gojieba",
		},
	)
	if err != nil {
		panic(err)
	}
	err = indexMapping.AddCustomAnalyzer("gojieba",
		map[string]interface{}{
			"type":      "gojieba",
			"tokenizer": "gojieba",
		},
	)
	if err != nil {
		panic(err)
	}
	indexMapping.DefaultAnalyzer = "gojieba"
	pyFieldMapping := mapping.NewTextFieldMapping()
	pyFieldMapping.Name = "py"
	pyFieldMapping.Analyzer = "standard"
	jpFieldMapping := mapping.NewTextFieldMapping()
	jpFieldMapping.Name = "jp"
	jpFieldMapping.Analyzer = "standard"
	cityFieldMapping := mapping.NewTextFieldMapping()
	cityFieldMapping.Name = "cityIds"
	cityFieldMapping.Analyzer = "standard"
	catesMapping := mapping.NewTextFieldMapping()
	catesMapping.Name = "cates"
	catesMapping.Analyzer = "standard"
	pyMapping := mapping.NewDocumentMapping()
	pyMapping.AddFieldMappingsAt("py", pyFieldMapping)
	pyMapping.AddFieldMappingsAt("jp", jpFieldMapping)
	pyMapping.AddFieldMappingsAt("cityIds", cityFieldMapping)
	pyMapping.AddFieldMappingsAt("cates", catesMapping)
	pictureFieldMapping := bleve.NewTextFieldMapping()
	pictureFieldMapping.Index = false
	pyMapping.AddFieldMappingsAt("picture", pictureFieldMapping)
	//pyMapping.AddSubDocumentMapping("picture", bleve.NewDocumentStaticMapping())
	//pyMapping.AddSubDocumentMapping("picture", bleve.NewDocumentDisabledMapping())
	pyMapping.AddSubDocumentMapping("tkeywords", bleve.NewDocumentDisabledMapping())
	//indexMapping.AddDocumentMapping("pinyin", pyMapping)
	indexMapping.DefaultMapping = pyMapping
	return indexMapping, nil
}

func dump(idx bleve.Index) {
	ma := bleve.NewMatchAllQuery()
	request := bleve.NewSearchRequest(ma)
	request.Fields = []string{"id", "price"}
	request.SortBy([]string{"_id"})

	request.Size = 1000
	dc, _ := idx.DocCount()
	fmt.Println(dc)

	sr, err := idx.Search(request)
	if err != nil {
		log.Fatal(err)
	}
	//fmt.Println(sr)

	for _, hit := range sr.Hits {
		fmt.Printf("%s\t,%v\n", hit.Fields["id"], hit.Fields["price"])
	}
}

func main() {
	isDump := flag.Bool("dump", false, "--dump")
	flag.Parse()
	mapping, err := buildIndexMapping()
	if err != nil {
		log.Fatal(err)
	}
	/*docMapping := mapping.DefaultMapping
	docMapping.AddSubDocumentMapping("isRecommend", bleve.NewDocumentStaticMapping())
	docMapping.AddSubDocumentMapping("picture", bleve.NewDocumentStaticMapping())
	*/
	indexName := "xn_product.bleve"
	var productIndex bleve.Index

	var ierr error
	if _, err := os.Stat(indexName); os.IsNotExist(err) {
		productIndex, ierr = bleve.New(indexName, mapping)
	} else if err == nil {
		if *isDump {
			productIndex, ierr = bleve.OpenUsing(indexName, map[string]interface{}{
				"read_only": true,
			})
		} else {
			productIndex, ierr = bleve.Open(indexName)
		}
	}
	if ierr != nil {
		log.Fatal(ierr)
	}
	defer productIndex.Close()
	if *isDump {
		dump(productIndex)
		return
	}
	/*p := Product{
		Id:   "200817010121",
		No:   "sdsdfsjosdifkids",
		Name: "空腹血糖",
		Tags: "血糖",
		Tips: "需要空腹测试",
	}
	productIndex.Index(p.Id, p)
	*/
	strSql := "select id, name,price, is_recommend, picture, keywords,sale_num,putaway_time,sort_no,created_at,market_price from t_product  where is_putaway = 0 and is_delete = 0;"
	db := util.GetSqlDB()
	productList := []ProductDB{}
	err = db.Select(&productList, strSql)
	if err != nil {
		log.Fatal(err)
	}
	for _, product := range productList {
		strSql := "select t1.name from t_detect_product t1 left join t_product_detect_product t2 on t1.id = t2.detect_product_id where t2.product_id = " + product.Id
		itemList := []string{}
		db.Select(&itemList, strSql)
		//strSql = "select tag_name from t_product_tag t1 left join t_tag t2 on t1.tag_id = t2.id where t1.product_id = ?"
		//tagList := []string{}
		//db.Select(&tagList, strSql, product.Id)
		strSql = "select city_id from v_product_city where product_id = ?"
		cityIds := []string{}
		db.Select(&cityIds, strSql, product.Id)
		product.Items = strings.Join(itemList, " ")
		//product.Tags = strings.Join(tagList, " ")
		product.CityIds = cityIds
		product.Pinyin, product.JP, _ = util.GetPinyin(product.Name)
		product.Keywords = product.TKeywords.String
		product.PutawayTime = product.TPutawayTime.String
		strSql = "SELECT cat_id from t_product_category_product where product_id = ?"
		var cateIds = []string{}
		db.Select(&cateIds, strSql, product.Id)
		product.Cates = cateIds
		productIndex.Index(product.Id, product)
	}
	//bytes, _ := json.Marshal(&productList)
	//fmt.Println(string(bytes))

}