From ad01920ddfc3e84df39a70fc9d2c0708f111b441 Mon Sep 17 00:00:00 2001
From: jaysunxiao <jaysunxiao@gmail.com>
Date: Tue, 6 Jul 2021 11:50:18 +0800
Subject: [PATCH] =?UTF-8?q?perf[zapp]:=20=E6=96=B0=E5=A2=9E=E6=B5=8B?=
 =?UTF-8?q?=E8=AF=95=E7=94=A8=E4=BE=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 doc/elastic/elastic-search-command.md         | 806 ++++++++++++++++++
 doc/elastic/elastic-search-setup.md           |  83 ++
 doc/elastic/elastic-search.md                 |  14 +
 .../zfoo/orm/cache/guava/GuavaCacheTest.java  |  87 ++
 4 files changed, 990 insertions(+)
 create mode 100644 doc/elastic/elastic-search-command.md
 create mode 100644 doc/elastic/elastic-search-setup.md
 create mode 100644 doc/elastic/elastic-search.md
 create mode 100644 orm/src/test/java/com/zfoo/orm/cache/guava/GuavaCacheTest.java

diff --git a/doc/elastic/elastic-search-command.md b/doc/elastic/elastic-search-command.md
new file mode 100644
index 00000000..6189b490
--- /dev/null
+++ b/doc/elastic/elastic-search-command.md
@@ -0,0 +1,806 @@
+# 一、索引管理
+
+- es所有的命令都是通过http完成的
+
+- es的和传统的数据库对于规则如下：
+
+```
+db              es
+数据库           索引
+表              类型type
+行              文档document
+列              字段field
+表结构schema     映射mapping
+索引             全文索引
+sql             http接口
+```
+
+- Text：会分词，然后进行索引，支持模糊、精确查询，不支持聚合
+
+- keyword： 不分词，直接索引，支持模糊、精确查询，支持聚合
+
+## 1.索引的增加
+
+- put blog，索引必须全部小写，es7过后取消type概念，默认为_doc
+
+```json
+{
+  "settings": {
+    "number_of_shards": 1
+  },
+  "mappings": {
+    "properties": {
+      "id": {
+        "type": "long"
+      },
+      "content": {
+        "type": "text"
+      },
+      "postTime": {
+        "type": "date"
+      },
+      "title": {
+        "type": "text"
+      }
+    }
+  }
+}
+```
+
+## 2.删除
+
+- delete blog
+- 全部删除
+
+```
+curl -X DELETE http://localhost:9200/_all
+```
+
+## 3.改
+
+- put blog/_settings
+
+```json
+{
+  "blocks.read": true
+}
+```
+
+```
+blocks.read，禁止对当前索引进行读操作
+blocks.read_only，只允许读，不允许写
+blocks.write，禁止写操作
+```
+
+- put blog/_mappings
+
+```json
+{
+    "properties": {
+        "id": {
+            "type": "text"
+        },
+        "content": {
+            "type": "text"
+        },
+        "postTime": {
+            "type": "date"
+        },
+        "title": {
+            "type": "text"
+        },
+        "aaa": {
+            "type": "text"
+        }
+    }
+}
+```
+
+## 4.查
+
+- 查询全部的索引
+
+```
+curl -X GET http://localhost:9200/_cat/indices
+```
+
+- get blog/_settings
+- get blog/_mappings
+
+## 5.索引打开和关闭
+
+- post blog/_close，已经关闭的索引不能进行任何读写操作
+- post blog/_open
+
+- post _all/_close
+- post _all/_open，全部打开
+
+# 二、文档管理
+
+## 1.增加文档
+
+- put blog/_doc/1，index/type/id
+
+```json
+{
+  "id": 1,
+  "title": "Java讲义",
+  "postTime": "2019-06-09",
+  "content": "全功能的一门编程语言"
+}
+```
+
+- post blog/_doc，这种方式id会随机生成
+
+```json
+{
+  "id": 3,
+  "title": "Java讲义3",
+  "postTime": "2019-06-09",
+  "content": "全功能的一门编程语言3"
+}
+```
+
+## 2.修改文档
+
+- put blog/_doc/1
+
+```json
+{
+  "title": "Java讲义bbb",
+  "postTime": "2019-06-09",
+  "content": "全功能的一门编程语言bbb"
+}
+```
+
+## 3.删除文档
+
+- delete blog/_doc/1
+
+## 4.查询文档
+
+- get blog/_doc/1
+
+## 5.路由机制
+
+- shard = hash(routing) % shard_num
+
+```
+routing默认为文档id，也可以指定routing，这样可以避免查询的时候广播查询，如下
+get blog/article/1?routing=jay，routing值固定的话，可以让查询集中在一个片上
+```
+
+# 三、映射
+
+- get blog/_mapping，查看映射信息，默认是动态映射
+
+- put blog，创建索引时就添加静态映射，添加更精准，更详细的配置信息
+
+```
+boost为权重权重，默认为1
+format匹配的格式
+keyword不会被分词，ignore_above和keyword一起使用，指定分词和索引的最大长度，超过的部分不会被索引
+index是否进行到排序索引，false不进行索引，也就不可搜索，默认开启，动态mapping解析出来为数字类型、布尔类型的字段除外
+doc_values默认开启，会增加一个列存储索引，对于不需要排序和聚合的字段可以关闭
+similarity默认使用BM25评分模型
+{
+    "settings": {
+        "number_of_shards": 1
+    },
+    "mappings": {
+        "properties": {
+            "id": {
+                "type": "long"
+            },
+            "title": {
+                "type": "keyword",
+                "ignore_above": 20
+            },
+            "content": {
+                "type": "text",
+                "boost": 2,
+                "analyzer": "ik_max_word",
+                "search_analyzer": "ik_smart"
+            },
+            "postTime": {
+                "type": "date",
+                "format": "yyyy-MM-dd HH:mm:ss",
+                "doc_values": true,
+                "index": false
+            }
+        }
+    }
+}
+```
+
+- post _bulk，批量插入，一个document必须在一行，而且还要换行，下面的结构不要改
+
+```
+{"index":{ "_index": "blog", "_type": "_doc", "_id": "1" }}
+{"id": 1, "title": "Java讲义", "postTime": "2017-06-09 12:30:00", "content": "全功能的一门编程语言，既可以做前端，也可以做后端"}
+
+{"index":{ "_index": "blog", "_type": "_doc", "_id": "2" }}
+{"id": 2, "title": "精通C#", "postTime": "2018-06-09 12:30:00", "content": "大部分用来在windows上做前端"}
+
+{"index":{ "_index": "blog", "_type": "_doc", "_id": "3" }}
+{"id": 3, "title": "Go实战", "postTime": "2019-06-09 12:30:00", "content": "Google全力主推的一门语言"}
+
+```
+
+# 四、复杂查询
+
+- get/post，blog/_search或者/_search查询全部
+
+```json
+{
+  "query": {
+    "match_all": {}
+  }
+}
+```
+
+- get，/_search，分页查询，_source指定需要返回的字段，version默认不返回，min_score过滤最低的分数，highlight高亮显示，不指定sort默认用评分排序
+
+```json
+{
+  "from": 0,
+  "size": 100,
+  "query": {
+    "term": {
+      "content": "前端"
+    }
+  },
+  "_source": [
+    "title",
+    "content"
+  ],
+  "version": true,
+  "min_score": 0.6,
+  "highlight": {
+    "fields": {
+      "content": {
+        "pre_tags": [
+          "<strong>"
+        ],
+        "post_tags": [
+          "</strong"
+        ]
+      }
+    }
+  },
+  "sort": [
+    {
+      "id": {
+        "order": "desc"
+      }
+    },
+    {
+      "title": {
+        "order": "asc"
+      }
+    }
+  ]
+}
+```
+
+## 1.全文查询
+
+### 1).match
+
+- query会分词，operator分词为或关系
+
+```json
+{
+    "query": {
+        "match": {
+            "content": {
+                "query": "编程语言",
+                "operator": "or"
+            }
+        }
+    }
+}
+```
+
+- multi_match是match的升级，可以指定多个字段
+
+```json
+{
+    "query": {
+        "multi_match": {
+            "query": "语言",
+            "fields": [
+                "title",
+                "content"
+            ]
+        }
+    }
+}
+```
+
+### 2).match_phrase
+
+- 会分词，分词后的所有词项都要出现在该字段中；字段中的词项顺序要一致
+
+```json
+{
+    "query": {
+        "match_phrase": {
+            "content": "编程语言"
+        }
+    }
+}
+```
+
+- match_phrase_prefix与match_phrase类似，只不过支持最后一个词项前缀匹配
+
+```json
+{
+    "query": {
+        "match_phrase_prefix": {
+            "content": "语言 既"
+        }
+    }
+}
+```
+
+### 3).simple_query_string
+
+```json
+{
+    "query": {
+        "simple_query_string": {
+            "query": "编程语言",
+            "analyzer": "ik_max_word",
+            "fields": ["title", "content"],
+            "default_operator": "or"
+        }
+    }
+}
+```
+
+## 2.词项查询
+
+### 1).terms
+
+- term，查询含有一个词项
+
+```json
+{
+    "query": {
+        "term": {
+            "content": "前端"
+        }
+    }
+}
+```
+
+- 查询content中包含语言的多个词项，含有一个词项只能有term
+
+```json
+{
+  "query": {
+    "terms": {
+      "content": [
+        "前端",
+        "后端"
+      ]
+    }
+  }
+}
+```
+
+### 2).range query
+
+- 查询title，因为keyword类型的不分词，如果查“讲义”是查询不到的，term结构化字段查询，匹配一个值，且输入的值不会被分词器分词。
+
+```json
+{
+  "query": {
+    "range": {
+      "id": {
+        "gt": 2,
+        "lte": 3
+      }
+    }
+  }
+}
+```
+
+### 3).exists query
+
+- 查询title，因为keyword类型的不分词，如果查“讲义”是查询不到的，term结构化字段查询，匹配一个值，且输入的值不会被分词器分词。
+
+```json
+{
+    "query": {
+        "exists": {
+            "field": "title"
+        }
+    }
+}
+```
+
+### 4).prefix query
+
+- 查询某些以win开头的词项
+
+```json
+{
+    "query": {
+        "prefix": {
+            "content": "win"
+        }
+    }
+}
+```
+
+### 5).wildcard query
+
+- 通配符查询，?匹配一个字符，*匹配任意多个字符
+
+```json
+{
+    "query": {
+        "wildcard": {
+            "content": "win*"
+        }
+    }
+}
+```
+
+### 6).regexp query
+
+- 正则表达式查询，和java的正则表达式类似
+
+```json
+{
+  "query": {
+    "regexp": {
+      "content": "windows"
+    }
+  }
+}
+```
+
+### 7).fuzzy query
+
+- 模糊查询效率不高，如下把windows写错了，仍然可以查到
+
+```json
+{
+    "query": {
+        "fuzzy": {
+            "content": "windosw"
+        }
+    }
+}
+```
+
+### 8).type query
+
+- es7中默认的为_doc
+
+```json
+{
+    "query": {
+        "type": {
+            "value": "_doc"
+        }
+    }
+}
+```
+
+### 8).ids query
+
+- 查询含有某些_id的文档
+
+```json
+{
+    "query": {
+        "ids": {
+            "type": "_doc",
+            "values": [1, 2]
+        }
+    }
+}
+```
+
+## 3.复合查询
+
+### 1).bool query
+
+- 逻辑查询，must=and，should=or
+
+```json
+{
+  "query": {
+    "bool": {
+      "minimum_should_match": 1,
+      "must": {
+        "match": {
+          "title": "Java讲义"
+        }
+      },
+      "should": {
+        "match": {
+          "content": "语言"
+        }
+      },
+      "must_not": {
+        "range": {
+          "id": {
+            "gte": 2
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+# 五、聚合查询
+
+## 1.指标聚合
+
+### 1).max 和 min
+
+```json
+{
+    "size": 0,
+    "aggs": {
+        "max_id_demo": {
+            "max": {
+                "field": "id"
+            }
+        }
+    }
+}
+```
+
+### 2).avg 和 sum
+
+- avg和sum用法类似
+
+```json
+{
+    "size": 0,
+    "aggs": {
+        "avg_id_demo": {
+            "avg": {
+                "field": "id"
+            }
+        }
+    }
+}
+```
+
+### 3).cardinality
+
+- 类似distinct
+
+```json
+{
+    "size": 0,
+    "aggs": {
+        "all_title_demo": {
+              "cardinality": {
+                  "field": "title"
+              }
+        }
+    }
+}
+```
+
+### 4).stats 和 extended_stats
+
+- 会统计count，max，min，avg，sum，必须是整型，两者用法基本类似，后者返回更多的统计条目如平方差
+
+```json
+{
+    "size": 0,
+    "aggs": {
+        "extended_id_demo": {
+              "stats": {
+                  "field": "id"
+              }
+        }
+    }
+}
+```
+
+### 5).percentiles
+
+- 百分位统计
+
+```json
+{
+    "size": 0,
+    "aggs": {
+        "percentiles_id_demo": {
+              "percentiles": {
+                  "field": "id"
+              }
+        }
+    }
+}
+```
+
+### 6).percentiles
+
+- 按字段统计文档数量
+
+```json
+{
+    "size": 0,
+    "aggs": {
+        "count_id_demo": {
+              "value_count": {
+                  "field": "id"
+              }
+        }
+    }
+}
+```
+
+## 2.分组聚合
+
+### 1).terms aggregation
+
+- 按id分组，统计每组的数量
+
+```json
+{
+  "size": 0,
+  "aggs": {
+    "per_count_demo": {
+      "terms": {
+        "field": "id"
+      }
+    }
+  }
+}
+```
+
+- 按id分组，统计每组的数量和每组平均价格
+
+```json
+{
+    "size": 0,
+    "aggs": {
+        "per_count_demo": {
+              "terms": {
+                  "field": "id"
+              },
+              "aggs": {
+                  "avg_id_test": {
+                      "avg": {
+                          "field": "id"
+                      }
+                  }
+              }
+        }
+    }
+}
+```
+
+### 2).filter aggregation
+
+- filter过滤符合条件的文档，将这些文档分到一个桶中，然后统计
+
+```json
+{
+  "size": 0,
+  "aggs": {
+    "avg_id_demo": {
+      "filter": {
+        "term": {
+          "content": "前端"
+        }
+      },
+      "aggs": {
+        "avg_id_test": {
+          "avg": {
+            "field": "id"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+- filters过滤符合条件的文档，将这些文档分到不同的filter桶中，然后统计
+
+```json
+{
+  "size": 0,
+  "aggs": {
+    "avg_id_demo": {
+      "filters": {
+        "filters": [
+          {
+            "match": {
+              "id": 1
+            }
+          },
+          {
+            "match": {
+              "id": 2
+            }
+          },
+          {
+            "match": {
+              "id": 3
+            }
+          }
+        ]
+      },
+      "aggs": {
+        "avg_id_test": {
+          "avg": {
+            "field": "id"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+### 3).range aggregation
+
+- 数值类型的范围聚合，反映数据的分布情况
+
+```json
+{
+    "size": 0,
+    "aggs": {
+        "range_id_demo": {
+            "range": {
+                "field": "id",
+                "ranges": [
+                    {
+                        "to": 2
+                    },
+                    {
+                        "from": 2,
+                        "to": 3
+                    },
+                    {
+                        "from": 3
+                    }
+                ]
+            }
+        }
+    }
+}
+```
+
+- 日期类型的范围聚合
+
+```json
+{
+  "size": 0,
+  "aggs": {
+    "range_date_demo": {
+      "date_range": {
+        "field": "postTime",
+        "format": "yyyy-MM-dd HH:mm:ss",
+        "ranges": [
+          {
+            "to": "2018-01-01 16:00:00"
+          },
+          {
+            "from": "2018-01-01 16:00:00",
+            "to": "2019-01-01 16:00:00"
+          },
+          {
+            "from": "2019-01-01 16:00:00"
+          }
+        ]
+      }
+    }
+  }
+}
+```
diff --git a/doc/elastic/elastic-search-setup.md b/doc/elastic/elastic-search-setup.md
new file mode 100644
index 00000000..3d123482
--- /dev/null
+++ b/doc/elastic/elastic-search-setup.md
@@ -0,0 +1,83 @@
+# 一、下载和安装
+
+- 安装Java环境
+- 安装ElasticSearch，在安装Java JDK，配置JAVA_HOME环境变量之后，就可以安装Elasticsearch全文搜索引擎了，首先需要下载特定版本的ES安装文件。
+- 下载ElasticSearch最新版本，这里是7.x.x，从官方下载中心 ElasticSearch Download 下载ElasticSearch安装包，并解压
+
+# 二、启动
+
+## 1.Windows下ES安装
+
+- 进入解压的bin/目录，双击执行 elasticsearch.bat，该脚本文件执行 ElasticSearch 安装程序
+- 稍等片刻，打开浏览器，输入 http://localhost:9200，有回应说明安装成功
+
+## 2.Linux下ES安装
+
+- 将elasticsearch-7.7.0-linux-x86_64.tar.gz下载到/usr/local/目录，这个目录是专门放下载的第三方软件的目录
+- tar -xzvf elasticsearch-7.7.0-linux-x86_64.tar.gz 解压
+- elasticsearch.yml做如下配置：
+
+```
+node.name: node-1
+network.host: 0.0.0.0
+http.port: 9200
+cluster.initial_master_nodes: ["node-1"]
+
+path.data: /data/es/db
+path.logs: /data/es/logs
+```
+
+- mkdir -p /data/es/db /data/es/logs
+
+- jvm.options做如下配置，单机部署可不用设置：
+
+```
+-Xms4g
+-Xmx4g
+```
+
+- Elasticsearch 不允许root用户直接运行，所以要创建新用户，在root用户中创建新用户,执行如下命令：
+
+```
+useradd sun  # 新增sun用户
+passwd  sun  # 为sun用户设置密码
+chown -R sun:sun /usr/local/elasticsearch-7.7.0 # 为新用户授权，文件夹所有者
+chown -R sun:sun /data
+su sun
+./elasticsearch &
+```
+
+- 可能遇到的问题
+
+```
+启动内存国小，设置jvm内存-xms设置为内存的一半
+
+max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
+vim /etc/sysctl.conf        # 在最后一行上加上
+vm.max_map_count=262144     # 保存退出，加载配置，启动容器，就能够访问了
+sysctl -p                   # 使修改生效
+```
+
+- 验证是否生效，curl localhost:9200
+
+# 三、插件和工具
+
+## 1.elasticsearch-analysis-ik插件
+
+- ik是一个对中文比较友好的分词插件
+
+```
+download pre-build package from here: https://github.com/medcl/elasticsearch-analysis-ik/releases
+create plugin folder cd your-es-root/plugins/ && mkdir ik
+unzip plugin to folder your-es-root/plugins/ik
+```
+
+## 2.kibanba安装
+
+- kibanba是es中的数据可视化工具
+
+```
+https://www.elastic.co/cn/downloads/kibanasu sun
+
+直接下载解压，运行bin目录下的可执行文件kibana.bat
+```
diff --git a/doc/elastic/elastic-search.md b/doc/elastic/elastic-search.md
new file mode 100644
index 00000000..ac2c1bbd
--- /dev/null
+++ b/doc/elastic/elastic-search.md
@@ -0,0 +1,14 @@
+# ElasticSearch
+
+## 一、简介
+
+ES是一个基于Lucene的分布式全文搜索服务器，和SQL Server的全文索引（Fulltext Index）有点类似，都是基于分词和分段的全文搜索引擎， 具有分词，同义词，词干查询的功能，但是ES天生具有分布式和实时的属性。
+ElasticSearch官网：http://www.elasticsearch.org
+
+[官方Github](https://github.com/elastic/elasticsearch)
+
+[官方中文权威指南](https://www.elastic.co/guide/cn/elasticsearch/guide/current/index.html)
+
+[官方Java API文档](https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/index.html)
+
+[Elasticsearch Java API 手册](https://es.quanke.name/)
\ No newline at end of file
diff --git a/orm/src/test/java/com/zfoo/orm/cache/guava/GuavaCacheTest.java b/orm/src/test/java/com/zfoo/orm/cache/guava/GuavaCacheTest.java
new file mode 100644
index 00000000..83aad343
--- /dev/null
+++ b/orm/src/test/java/com/zfoo/orm/cache/guava/GuavaCacheTest.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2020 The zfoo Authors
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and limitations under the License.
+ *
+ */
+
+package com.zfoo.orm.cache.guava;
+
+import com.google.common.cache.*;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.zfoo.util.ThreadUtils;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author jaysunxiao
+ * @version 1.0
+ * @since 2019-08-13 11:15
+ */
+@Ignore
+public class GuavaCacheTest {
+
+    @Test
+    public void sizeRemovedTest() {
+        RemovalListener<String, String> listener = new RemovalListener<String, String>() {
+            public void onRemoval(RemovalNotification<String, String> notification) {
+                System.out.println("[" + notification.getKey() + ":" + notification.getValue() + "] is removed!");
+
+                // 采用LRU算法，所以最后会移除key1
+                Assert.assertEquals(notification.getKey(), "key1");
+                Assert.assertEquals(notification.getValue(), "value1");
+            }
+        };
+        Cache<String, String> cache = CacheBuilder.newBuilder()
+                .maximumSize(3)
+                .removalListener(listener)
+                .build();
+        cache.put("key1", "value1");
+        cache.put("key2", "value2");
+        cache.put("key3", "value3");
+        cache.put("key4", "value4");
+    }
+
+    @Test
+    public void timeRemovedTest() {
+        // 基于过期时间的策略不会回调这个remove方法
+        Cache<String, String> cache = CacheBuilder.newBuilder()
+                .expireAfterAccess(10, TimeUnit.SECONDS)
+                .refreshAfterWrite(5, TimeUnit.SECONDS)
+                .removalListener(new RemovalListener<String, String>() {
+                    public void onRemoval(RemovalNotification<String, String> notification) {
+                        System.out.println("[" + notification.getKey() + ":" + notification.getValue() + "] is removed!");
+                    }
+                })
+                .build(new CacheLoader<String, String>() {
+                    public String load(String key) {
+                        System.out.println("load by " + Thread.currentThread().getName());
+                        return "newValue";
+                    }
+
+                    @Override
+                    public ListenableFuture<String> reload(String key, String oldValue) {
+                        System.out.println("reload by " + Thread.currentThread().getName());
+                        return Futures.immediateFuture("newValue");
+                    }
+                });
+
+        cache.put("key1", "value1");
+
+        Assert.assertEquals(cache.getIfPresent("key1"), "value1");
+        ThreadUtils.sleep(5000);
+        Assert.assertEquals(cache.getIfPresent("key1"), "newValue");
+    }
+
+
+}