安装efk日志采集和开启search-guard 安全认证

安装ansible

pip install ansible

复制代码

安装elastsearch集群

下载role

ansible-galaxy install elastic.elasticsearch,7.6.2
复制代码

配置主机信息

cd /root/.ansible/roles
[root@iZ1bb01rvthes8gbrnyixjZ roles]# cat inventory.yml
[elk-master-nodes]
master_ip  ansible_user=root ansible_ssh_pass=xx ansible_port=22
[elk-data-nodes]
data_node_ip  ansible_user=root ansible_ssh_pass=xx ansible_port=22
复制代码

配置playbook

[root@iZ1bb01rvthes8gbrnyixjZ roles]# cat elk.yml
- hosts: elk-master-nodes
  roles:
    - role: elastic.elasticsearch
  vars:
    es_enable_xpack: false
    es_data_dirs:
      - "/data/elasticsearch/data"
    es_log_dir: "/data/elasticsearch/logs"
    es_java_install: true
    es_heap_size: "8g"
    es_config:
      cluster.name: "elk-cluster"
      cluster.initial_master_nodes: "master1:9300,master2:9300,master3:9300"
      discovery.seed_hosts: "master1:9300,master2:9300,master3:9300"
      http.port: 9200
      node.data: false
      node.master: true
      bootstrap.memory_lock: false
      network.host: '0.0.0.0'
# master 与data node同节点
- hosts: elk-data-nodes
  roles:
    - role: elastic.elasticsearch
  vars:
    es_enable_xpack: false
    es_data_dirs:
      - "/data/elasticsearch/data"
    es_log_dir: "/data/elasticsearch/logs"
    es_java_install: true
    es_heap_size: "8g"
    es_config:
      cluster.name: "elk-cluster"
      cluster.initial_master_nodes: "master1:9300,master2:9300,master3:9300"
      discovery.seed_hosts: "master1:9300,master2:9300,master3:9300"
      http.port: 9200
      node.data: true
      node.master: false
      bootstrap.memory_lock: false
      network.host: '0.0.0.0'
复制代码

运行playbook

ansible-playbook  -i inventory.yml elk.yml
复制代码

验证集群

curl http://master1:9200/_cluster/health?pretty

复制代码

如果报错，修改配置文件

[root@iZ1bb01rvthes8gbrnyixjZ roles]# cat /etc/elasticsearch/elasticsearch.yml
cluster.name: es-cluster
node.name: master1
node.master: true
node.data: true
bootstrap.memory_lock: true
network.host: 0.0.0.0
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping.unicast.hosts: ["master1", "master2", "master3"]
cluster.initial_master_nodes: master1
#################################### Paths ####################################

# Path to directory containing configuration (this file and logging.yml):

path.data: /data/elasticsearch/data

path.logs: /data/elasticsearch/logs



action.auto_create_index: true
# 重启es
systemctl restart elasticsearch
复制代码

安装kibana

# 版本需要跟es版本对应为 oss 7.6.2
wget https://artifacts.elastic.co/downloads/kibana/kibana-oss-7.6.2-x86_64.rpm
yum install kibana-oss-7.6.2-x86_64.rpm
systemctl start kibana
# 修改配置为监听0.0.0.0 端口
cat /etc/kibana/kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:9200"]
复制代码

安装search-guard 安全认证插件

安装 search-guard es 插件

#插件版本与es版本需对应
# 下载地址
https://docs.search-guard.com/latest/search-guard-versions
wget https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/7.6.2-41.0.0/search-guard-suite-plugin-7.6.2-41.0.0.zip
# 安装文档
https://docs.search-guard.com/latest/search-guard-installation

复制代码

安装步骤

1. 关闭分片分配
curl -Ss -XPUT 'http://localhost:9200/_cluster/settings?pretty' -H 'Content-Type: application/json' -d'
{
  "persistent": {
    "cluster.routing.allocation.enable": "none"
  }
}
'
2. 停掉es
systemctl stop elasticsearch
3. 安装插件
/usr/share/elasticsearch/bin/elasticsearch-plugin install file:///root/search-guard-suite-plugin-7.6.2-41.0.0.zip
4. 增加 tls 配置
cd /usr/share/elasticsearch/plugins/search-guard-7/tools/
sh install_demo_configuration.sh 
# 连按三个Y
#cat /etc/elasticsearch/elasticsearch.yml 可以看到增加了tls配置
#### 三台机都要做以上操作，不然es开启分片分配会报错
4. 启动es集群
systemctl start elasticsearch
5. 开启分片分配
./sgadmin.sh --enable-shard-allocation -cert /etc/elasticsearch/kirk.pem -key /etc/elasticsearch/kirk-key.pem -cacert /etc/elasticsearch/root-ca.pem
6. 初始化search guard
sh sgadmin_demo.sh
7. 验证
curl http://localhost:9200/_searchguard/health -k
{"message":null,"mode":"strict","status":"UP"}[
复制代码

安装 search-guard kibana 插件

# 安装文档：https://docs.search-guard.com/latest/kibana-plugin-installation
wget https://maven.search-guard.com/search-guard-kibana-plugin-release/com/floragunn/search-guard-kibana-plugin/7.6.2-41.1.0/search-guard-kibana-plugin-7.6.2-41.1.0.zip
/usr/share/kibana/bin/kibana-plugin install file:///root/search-guard-kibana-plugin-7.6.2-41.1.0.zip --allow-root
# 修改配置文件
#改es 账号密码，默认 admin/admin
复制代码

修改默认密码

cd /usr/share/elasticsearch/plugins/search-guard-7/tools/
[root@iZ1bb01rvthes8gbrnyixjZ tools]# sh hash.sh -p password
WARNING: JAVA_HOME not set, will use /usr/bin/java
$2y$12$5bv2H9KAeLn3gMbO.4Hr6.eT9AW.hHpJZ3GOxycKWe4D0ixs7zGia
把hash值填写到 
vi /usr/share/elasticsearch/plugins/search-guard-7/sgconfig/sg_internal_users.yml 
admin:
  #hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG"
  hash: "$2y$12$GCUMF8L8AE7mSsCoD/zoI.1E2FiHppedwFnKZPBwcQOYzx2yygPlq"
  reserved: true
  backend_roles:
  - "admin"
  description: "Demo admin user"
# 重新初始化或者在初始化之前修改密码
sh sgadmin_demo.sh
复制代码

开启http访问

vi /etc/elasticsearch/elasticsearch.yml
searchguard.ssl.http.enabled: false
复制代码

安装flutend日志采集

# https://github.com/fluent/fluentd-kubernetes-daemonset
# 修改对应参数
[root@yunqi-master1 fluentd-kubernetes-daemonset]# cat fluentd-daemonset-elasticsearch-rbac.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: kube-system

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd
  namespace: kube-system
rules:
- apiGroups:
  - ""
  resources:
  - pods
  - namespaces
  verbs:
  - get
  - list
  - watch

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: fluentd
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: kube-system
  labels:
    k8s-app: fluentd-logging
    version: v1
spec:
  selector:
    matchLabels:
      k8s-app: fluentd-logging
      version: v1
  template:
    metadata:
      labels:
        k8s-app: fluentd-logging
        version: v1
    spec:
      serviceAccount: fluentd
      serviceAccountName: fluentd
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      containers:
      - name: fluentd
        image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch
        env:
          - name:  FLUENT_ELASTICSEARCH_HOST
            value: "master1"
          - name:  FLUENT_ELASTICSEARCH_PORT
            value: "9200"
          - name: FLUENT_ELASTICSEARCH_SCHEME
            value: "http"
          # Option to configure elasticsearch plugin with self signed certs
          # ================================================================
          - name: FLUENT_ELASTICSEARCH_SSL_VERIFY
            value: "true"
          # Option to configure elasticsearch plugin with tls
          # ================================================================
          - name: FLUENT_ELASTICSEARCH_SSL_VERSION
            value: "TLSv1_2"
          # X-Pack Authentication
          # =====================
          - name: FLUENT_ELASTICSEARCH_USER
            value: "admin"
          - name: FLUENT_ELASTICSEARCH_PASSWORD
            value: "admin"
          - name: FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE
            value: "15m"
        resources:
          limits:
            memory: 400Mi
          requests:
            cpu: 100m
            memory: 200Mi
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /data/docker/containers
          readOnly: true
      terminationGracePeriodSeconds: 30
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /data/docker/containers
kubectl apply -f fluentd-daemonset-elasticsearch-rbac.yaml

复制代码

flutend 踩坑点

flutend报错之一

2016-12-01T22:41:01.814337960Z 2016-12-01 22:41:01 +0000 [warn]: /var/log/containers/kubernetes-dashboard-qcmyh_kube-system_kubernetes-dashboard-c98d8d1296b691493ac2a8ce7f30a42dbe8107d2e425cd12f681021144f1895f.log unreadable. It is excluded and would be examined next time.
2016-12-01T22:41:01.814351093Z 2016-12-01 22:41:01 +0000 [warn]: /var/log/containers/kube-dns-v20-zr41g_kube-system_POD-22aaa1f29033d57e6f1a40a2c87c6d3624815824adf9e5192d4ec611e1ab1909.log unreadable. It is excluded and would be examined next time.
2016-12-01T22:41:01.814364345Z 2016-12-01 22:41:01 +0000 [warn]: /var/log/containers/kubernetes-dashboard-qcmyh_kube-system_POD-465a601853d4a418984da54c1a5d7868368b6f221ba936988930cd22b87d4685.log unreadable. It is excluded and would be examined next time.
2016-12-01T22:41:01.814374217Z 2016-12-01 22:41:01 +0000 [warn]: /var/log/containers/kube-addon-manager-minikube_kube-system_kube-addon-manager-0813bd6a91f83b7d5171b8f291a1a353ce3c4672f5494319c61ba0d78c13f046.log unreadable. It is excluded and would be examined next time.
2016-12-01T22:41:01.814381784Z 2016-12-01 22:41:01 +0000 [warn]: /var/log/containers/kube-addon-manager-minikube_kube-system_POD-a894a06883d52c19b5e0f62bff14465f70525363d26bc66652cc31c36b266b67.log unreadable. It is excluded and would be examined next time.
复制代码

排查

# 发现是软链接
ll /var/log/containers
# 需要修改yaml文件为docker 路径，docker info 可以查看路径
docker info
# 参考issue
https://github.com/kubernetes/minikube/issues/876#issuecomment-264676096
复制代码

flutend报错之二

chunk bytes limit exceeds for an emitted event stream:
复制代码

排查

默认chuck bytes size不够大
修改yaml文件注入容器环境变量修改默认chunk byte size
- name: FLUENT_ELASTICSEARCH_BUFFER_CHUNK_LIMIT_SIZE
  value: "15m"
复制代码