Elasticsearch 8.4 から利用可能な従来の検索機能と近似近傍探索を組み合わせたハイブリッド検索を試す

1
2
3
4
5

FROM docker.elastic.co/elasticsearch/elasticsearch:8.4.3
# set the specific password for Elasticsearch
RUN echo "elastic" | bin/elasticsearch-keystore add "bootstrap.password" -xf
RUN bin/elasticsearch-plugin install analysis-kuromoji
RUN bin/elasticsearch-plugin install analysis-icu

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

{
  "settings": {
    "analysis": {
      "analyzer": {
        "custome_ja_analyzer": {
          "type": "custom",
          "char_filter": ["icu_normalizer"],
          "tokenizer": "kuromoji_tokenizer",
          "filter": [
            "kuromoji_baseform",
            "kuromoji_part_of_speech",
            "ja_stop",
            "kuromoji_number",
            "kuromoji_stemmer"
          ]
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "vector": {
        "type": "dense_vector",
        "dims": 768,
        "index": true,
        "similarity": "l2_norm"
      },
      "name": {
        "type": "text",
        "analyzer": "custome_ja_analyzer"
      },
      "yomi": {
        "type": "text",
        "analyzer": "custome_ja_analyzer"
      },
      "description": {
        "type": "text",
        "analyzer": "custome_ja_analyzer"
      }
    }
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

curl --cacert http_ca.crt -u elastic:elastic -X POST "https://localhost:9200/himitsu_dogu/_analyze?pretty" -H 'Content-Type: application/json' -d'{"analyzer": "custome_ja_analyzer","text":"ドラえもんのひみつ道具はどこでもドア以外にもたくさん存在する"}'
{
  "tokens" : [
    {
      "token" : "ドラえもん",
      "start_offset" : 0,
      "end_offset" : 5,
      "type" : "word",
      "position" : 0
    },
    {
      "token" : "ひむ",
      "start_offset" : 6,
      "end_offset" : 8,
      "type" : "word",
      "position" : 2
    },
    {
      "token" : "道具",
      "start_offset" : 9,
      "end_offset" : 11,
      "type" : "word",
      "position" : 4
    },
    {
      "token" : "どこ",
      "start_offset" : 12,
      "end_offset" : 14,
      "type" : "word",
      "position" : 6
    },
    {
      "token" : "ドア",
      "start_offset" : 16,
      "end_offset" : 18,
      "type" : "word",
      "position" : 8
    },
    {
      "token" : "以外",
      "start_offset" : 18,
      "end_offset" : 20,
      "type" : "word",
      "position" : 9
    },
    {
      "token" : "たくさん",
      "start_offset" : 22,
      "end_offset" : 26,
      "type" : "word",
      "position" : 12
    },
    {
      "token" : "存在",
      "start_offset" : 26,
      "end_offset" : 28,
      "type" : "word",
      "position" : 13
    }
  ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

$ curl --cacert http_ca.crt -u elastic:elastic -X POST "https://localhost:9200/himitsu_dogu/_search?pretty" -H 'Content-Type: application/json' -d'{"size": 1, "query": {"multi_match" : {"query":"どこでもドア", "fields": [ "name", "description" ] }}}'

{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 44,
      "relation" : "eq"
    },
    "max_score" : 11.774845,
    "hits" : [
      {
        "_index" : "himitsu_dogu",
        "_id" : "816",
        "_score" : 11.774845,
        "_source" : {
          "id" : 816,
          "name" : "どこでもドア",
          "yomi" : "どこでもどあ",
          "description" : "このドアを開けるだけで、行きたいところへ、どこへでも行くことができる。",
          "vector" : [
            -0.23546676337718964,
            ...
            -0.37538981437683105
          ]
        }
      }
    ]
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

multimatch_query: dict = {
		"size": TOP_K,
		"query": {"multi_match": {"query": keyword, "fields": ["name", "description"]}},
}
ann_query: dict = {
		"knn": {
				"field": "vector",
				"query_vector": sentence_embeddings[0],
				"k": TOP_K,
				"num_candidates": 100,
		}
}
# NOTE: score = match score + ANN score
hybrid_query: dict = {
		"size": TOP_K,
		"query": {"multi_match": {"query":keyword, "fields": ["name", "description"]}},
		"knn": {
				"field": "vector",
				"query_vector": sentence_embeddings[0],
				"k": TOP_K,
				"num_candidates": 100,
		},
}