（241018）
.env
# 项目名
COMPOSE_PROJECT_NAME=es
# elastic用户（至少6个数字）
ELASTIC_PASSWORD=
# kibana用户（至少6个数字）
KIBANA_PASSWORD=
# 版本号，一定要填写需求版本
# https://www.elastic.co/downloads/past-releases#elasticsearch
STACK_VERSION=8.15.1
# 集群名
CLUSTER_NAME=docker-cluster
# elastic的使用版本
LICENSE=basic
# elastic端口
ES_PORT=9200
# kibana端口
KIBANA_PORT=5601
# fleet端口
FLEET_PORT=8220
# apm端口
APMSERVER_PORT=8200
# APM令牌，k8s环境使用Kubernetes Secrets管理，也可以linux环境配置
ELASTIC_APM_SECRET_TOKEN=
# es、kibana、logstash最大内存量，可以换写成byte（3073741824、1073741824）
ES_MEM_LIMIT=3g
KB_MEM_LIMIT=1g
LS_MEM_LIMIT=1g# 密钥，k8s环境使用Kubernetes Secrets管理，也可以linux环境配置
ENCRYPTION_KEY=
docker-compose.yml
x${ELASTIC_PASSWORD} == x和x${KIBANA_PASSWORD} == x检测变量是否存在
! -f config/certs/ca.zip和! -f config/certs/certs.zip检测证书是否存在
bootstrap.memory_lock启用内存锁定，防止交换
xpack.license.self_generated.type指定许可证类型
（端口只适用于本地开发，生产环境更改端口）
volumes:certs:driver: localesdata01:driver: localkibanadata:driver: localmetricbeatdata01:driver: localfilebeatdata01:driver: locallogstashdata01:driver: localfleetserverdata:driver: localnetworks:default:name: es-networkexternal: falseservices:setup:image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}volumes:- certs:/usr/share/elasticsearch/config/certsuser: "0"command: >bash -c 'if [ x${ELASTIC_PASSWORD} == x ]; thenecho "Set the ELASTIC_PASSWORD environment variable in the .env file";exit 1;elif [ x${KIBANA_PASSWORD} == x ]; thenecho "Set the KIBANA_PASSWORD environment variable in the .env file";exit 1;fi;if [ ! -f config/certs/ca.zip ]; thenecho "Creating CA";bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;unzip config/certs/ca.zip -d config/certs;fi;if [ ! -f config/certs/certs.zip ]; thenecho "Creating certs";echo -ne \"instances:\n"\" - name: es01\n"\" dns:\n"\" - es01\n"\" - localhost\n"\" ip:\n"\" - 127.0.0.1\n"\" - name: kibana\n"\" dns:\n"\" - kibana\n"\" - localhost\n"\" ip:\n"\" - 127.0.0.1\n"\> config/certs/instances.yml;bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;unzip config/certs/certs.zip -d config/certs;fi;echo "Setting file permissions"chown -R root:root config/certs;find . -type d -exec chmod 750 \{\} \;;find . -type f -exec chmod 640 \{\} \;;echo "Waiting for Elasticsearch availability";until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;echo "Setting kibana_system password";until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;echo "Done";'healthcheck:test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]interval: 1stimeout: 5sretries: 120es01:depends_on:setup:condition: service_healthyimage: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}labels:co.elastic.logs/module: elasticsearchvolumes:- certs:/usr/share/elasticsearch/config/certs- esdata01:/usr/share/elasticsearch/dataports:- ${ES_PORT}:9200environment:- node.name=es01- cluster.name=${CLUSTER_NAME}- discovery.type=single-node- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}- bootstrap.memory_lock=true- xpack.security.enabled=true- xpack.security.http.ssl.enabled=true- xpack.security.http.ssl.key=certs/es01/es01.key- xpack.security.http.ssl.certificate=certs/es01/es01.crt- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt- xpack.security.transport.ssl.enabled=true- xpack.security.transport.ssl.key=certs/es01/es01.key- xpack.security.transport.ssl.certificate=certs/es01/es01.crt- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt- xpack.security.transport.ssl.verification_mode=certificate- xpack.license.self_generated.type=${LICENSE}mem_limit: ${ES_MEM_LIMIT}ulimits:memlock:soft: -1hard: -1healthcheck:test:["CMD-SHELL","curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",]interval: 10stimeout: 10sretries: 120kibana:depends_on:es01:condition: service_healthyimage: docker.elastic.co/kibana/kibana:${STACK_VERSION}labels:co.elastic.logs/module: kibanavolumes:- certs:/usr/share/kibana/config/certs- kibanadata:/usr/share/kibana/data- ./kibana.yml:/usr/share/kibana/config/kibana.yml:roports:- ${KIBANA_PORT}:5601environment:- SERVERNAME=kibana- ELASTICSEARCH_HOSTS=https://es01:9200- ELASTICSEARCH_USERNAME=kibana_system- ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}- ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt- XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}- XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}- XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}- XPACK_REPORTING_KIBANASERVER_HOSTNAME=localhost- SERVER_SSL_ENABLED=true- SERVER_SSL_CERTIFICATE=config/certs/kibana/kibana.crt- SERVER_SSL_KEY=config/certs/kibana/kibana.key- SERVER_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt- ELASTIC_APM_SECRET_TOKEN=${ELASTIC_APM_SECRET_TOKEN}mem_limit: ${KB_MEM_LIMIT}healthcheck:test:["CMD-SHELL","curl -I -s --cacert config/certs/ca/ca.crt https://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",]interval: 10stimeout: 10sretries: 120metricbeat01:depends_on:es01:condition: service_healthykibana:condition: service_healthyimage: docker.elastic.co/beats/metricbeat:${STACK_VERSION}user: rootvolumes:- certs:/usr/share/metricbeat/certs- metricbeatdata01:/usr/share/metricbeat/data- "./metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro"- "/var/run/docker.sock:/var/run/docker.sock:ro"- "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro"- "/proc:/hostfs/proc:ro"- "/:/hostfs:ro"environment:- ELASTIC_USER=elastic- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}- ELASTIC_HOSTS=https://es01:9200- KIBANA_HOSTS=https://kibana:5601- LOGSTASH_HOSTS=http://logstash01:9600- CA_CERT=certs/ca/ca.crt- ES_CERT=certs/es01/es01.crt- ES_KEY=certs/es01/es01.key- KB_CERT=certs/kibana/kibana.crt- KB_KEY=certs/kibana/kibana.keycommand:-strict.perms=falsefilebeat01:depends_on:es01:condition: service_healthyimage: docker.elastic.co/beats/filebeat:${STACK_VERSION}user: rootvolumes:- certs:/usr/share/filebeat/certs- filebeatdata01:/usr/share/filebeat/data- "./filebeat_ingest_data/:/usr/share/filebeat/ingest_data/"- "./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro"- "/var/lib/docker/containers:/var/lib/docker/containers:ro"- "/var/run/docker.sock:/var/run/docker.sock:ro"environment:- ELASTIC_USER=elastic- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}- ELASTIC_HOSTS=https://es01:9200- KIBANA_HOSTS=https://kibana:5601- LOGSTASH_HOSTS=http://logstash01:9600- CA_CERT=certs/ca/ca.crtcommand:-strict.perms=falselogstash01:depends_on:es01:condition: service_healthykibana:condition: service_healthyimage: docker.elastic.co/logstash/logstash:${STACK_VERSION}labels:co.elastic.logs/module: logstashuser: rootvolumes:- certs:/usr/share/logstash/certs- logstashdata01:/usr/share/logstash/data- "./logstash_ingest_data/:/usr/share/logstash/ingest_data/"- "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro"environment:- xpack.monitoring.enabled=false- ELASTIC_USER=elastic- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}- ELASTIC_HOSTS=https://es01:9200fleet-server:depends_on:kibana:condition: service_healthyes01:condition: service_healthyimage: docker.elastic.co/beats/elastic-agent:${STACK_VERSION}volumes:- certs:/certs- fleetserverdata:/usr/share/elastic-agent- "/var/lib/docker/containers:/var/lib/docker/containers:ro"- "/var/run/docker.sock:/var/run/docker.sock:ro"- "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro"- "/proc:/hostfs/proc:ro"- "/:/hostfs:ro"ports:- ${FLEET_PORT}:8220- ${APMSERVER_PORT}:8200user: rootenvironment:- SSL_CERTIFICATE_AUTHORITIES=/certs/ca/ca.crt- CERTIFICATE_AUTHORITIES=/certs/ca/ca.crt- FLEET_CA=/certs/ca/ca.crt- FLEET_ENROLL=1- FLEET_INSECURE=true- FLEET_SERVER_ELASTICSEARCH_CA=/certs/ca/ca.crt- FLEET_SERVER_ELASTICSEARCH_HOST=https://es01:9200- FLEET_SERVER_ELASTICSEARCH_INSECURE=true- FLEET_SERVER_ENABLE=1- FLEET_SERVER_CERT=/certs/fleet-server/fleet-server.crt- FLEET_SERVER_CERT_KEY=/certs/fleet-server/fleet-server.key- FLEET_SERVER_INSECURE_HTTP=true- FLEET_SERVER_POLICY_ID=fleet-server-policy- FLEET_URL=https://fleet-server:8220- KIBANA_FLEET_CA=/certs/ca/ca.crt- KIBANA_FLEET_SETUP=1- KIBANA_FLEET_USERNAME=elastic- KIBANA_FLEET_PASSWORD=${ELASTIC_PASSWORD}- KIBANA_HOST=https://kibana:5601webapp:build:context: appvolumes:- "/var/lib/docker/containers:/var/lib/docker/containers:ro"- "/var/run/docker.sock:/var/run/docker.sock:ro"- "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro"- "/proc:/hostfs/proc:ro"- "/:/hostfs:ro"ports:- 8000:8000
metricbeat.yml
metricbeat.config.modules:path: ${path.config}/modules.d/*.ymlreload.enabled: falsemetricbeat.modules:
- module: elasticsearchxpack.enabled: trueperiod: 10shosts: ${ELASTIC_HOSTS}username: ${ELASTIC_USER}password: ${ELASTIC_PASSWORD}ssl:enabled: truecertificate_authorities: ${CA_CERT}- module: logstashxpack.enabled: trueperiod: 10shosts: ${LOGSTASH_HOSTS}- module: kibanametricsets: - statsperiod: 10shosts: ${KIBANA_HOSTS}username: ${ELASTIC_USER}password: ${ELASTIC_PASSWORD}xpack.enabled: truessl:enabled: truecertificate_authorities: ${CA_CERT}- module: dockermetricsets:- "container"- "cpu"- "diskio"- "healthcheck"- "info"#- "image"- "memory"- "network"hosts: ["unix:///var/run/docker.sock"]period: 10senabled: trueprocessors:- add_host_metadata: ~- add_docker_metadata: ~output.elasticsearch:hosts: ${ELASTIC_HOSTS}username: ${ELASTIC_USER}password: ${ELASTIC_PASSWORD}ssl:enabled: truecertificate_authorities: ${CA_CERT}
filebeat.yml
filebeat.inputs:
- type: filestreamid: default-filestreampaths:- ingest_data/*.logfilebeat.autodiscover:providers:- type: dockerhints.enabled: trueprocessors:
- add_docker_metadata: ~setup.kibana:host: ${KIBANA_HOSTS}username: ${ELASTIC_USER}password: ${ELASTIC_PASSWORD}output.elasticsearch:hosts: ${ELASTIC_HOSTS}username: ${ELASTIC_USER}password: ${ELASTIC_PASSWORD}ssl:enabled: truecertificate_authorities: ${CA_CERT}
logstash.conf
input {file {#https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html#default is TAIL which assumes more data will come into the file.#change to mode => "read" if the file is a compelte file. by default, the file will be removed once reading is complete -- backup your files if you need them.mode => "tail"path => "/usr/share/logstash/ingest_data/*"}
}filter {
}output {elasticsearch {index => "logstash-%{+YYYY.MM.dd}"hosts=> "${ELASTIC_HOSTS}"user=> "${ELASTIC_USER}"password=> "${ELASTIC_PASSWORD}"cacert=> "certs/ca/ca.crt"}
}
kibana.yml
elastic:apm:active: trueserverUrl: "http://fleet-server:8200"secretToken: ${ELASTIC_APM_SECRET_TOKEN}
server.host: "0.0.0.0"
telemetry.enabled: "true"
xpack.fleet.packages:- name: fleet_serverversion: latest- name: systemversion: latest- name: elastic_agentversion: latest- name: apmversion: latest
xpack.fleet.agentPolicies:- name: Fleet-Server-Policyid: fleet-server-policynamespace: defaultmonitoring_enabled: - logs- metricspackage_policies:- name: fleet_server-1package:name: fleet_server- name: system-1package:name: system- name: elastic_agent-1package:name: elastic_agent- name: apm-1package:name: apminputs:- type: apmenabled: truevars:- name: hostvalue: 0.0.0.0:8200- name: secret_tokenvalue: ${ELASTIC_APM_SECRET_TOKEN}
app/dockerfile（仅演示文件）
# syntax=docker/dockerfile:1FROM python:3.9-slim-busterWORKDIR /appCOPY requirements.txt requirements.txtRUN pip3 install -r requirements.txtCOPY main.py main.pyCMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--log-level", "info", "--workers", "1"]
app/main.py（仅演示文件）
from elasticapm.contrib.starlette import ElasticAPM, make_apm_client
from fastapi import FastAPI
from nicegui import ui
from typing import Callable
import asyncio
import functools
import httpx as r
#import psutiltry:apm = make_apm_client({'SERVICE_NAME': 'my_python_service','SECRET_TOKEN': 'supersecrettoken',# SERVER_URL must be set to "fleet-server" if running as a docker container.# if running as a local python script, then set the url to "LOCALHOST"'SERVER_URL': 'http://fleet-server:8200','ENVIRONMENT': 'development'})
except Exception as e:print('failed to create client')app = FastAPI()try:app.add_middleware(ElasticAPM, client=apm)
except Exception as e:print('failed to add APM Middleware')@app.get("/custom_message/{message}")
async def custom_message(message: str):apm.capture_message(f"Custom Message: {message}")return {"message": f"Custom Message: {message}"}@app.get("/error")
async def throw_error():try:1 / 0except Exception as e:apm.capture_exception()return {"message": "Failed Successfully :)"}def init(fastapi_app: FastAPI) -> None:@ui.page('/', title="APM Demo App")async def show():with ui.header(elevated=True).style('background-color: #3874c8').classes('items-center justify-between'):ui.markdown('### APM DEMO')ui.button(on_click=lambda: right_drawer.toggle(), icon='menu').props('flat color=white')with ui.right_drawer(fixed=False).style('background-color: #ebf1fa').props('bordered') as right_drawer:ui.chat_message('Hello Elastic Stack User!',name='APM Robot',stamp='now',avatar='https://robohash.org/apm_robot')ui.chat_message('This app is powered by NICEGUI and FastAPI with Elastic APM Instrumentation :)',name='APM Robot',stamp='now',avatar='https://robohash.org/apm_robot')ui.chat_message('Please click a button to trigger an APM event.',name='APM Robot',stamp='now',avatar='https://robohash.org/apm_robot')with ui.footer().style('background-color: #3874c8'):ui.label('APM DEMO PAGE')with ui.card():ui.label('Generate Error - Python')ui.button('Generate', on_click=python_error)with ui.card():ui.label('Generate Error - JS')ui.button('Generate', on_click=js_error)with ui.card():ui.label('Generate Custom Message')custom_message_text = ui.input(placeholder='Message')ui.button('Generate').on('click', handler=lambda: gen_custom_message(custom_message_text.value))ui.run_with(fastapi_app,storage_secret='supersecret', # NOTE setting a secret is optional but allows for persistent storage per user)async def io_bound(callback: Callable, *args: any, **kwargs: any):'''Makes a blocking function awaitable; pass function as first parameter and its arguments as the rest'''return await asyncio.get_event_loop().run_in_executor(None, functools.partial(callback, *args, **kwargs))async def python_error():try:res = await io_bound(r.get, 'http://localhost:8000/error')ui.notify(res.text)except Exception as e:apm.capture_exception()ui.notify(f'{e}')async def js_error():try:res = await ui.run_javascript('fetch("http://localhost:8000/error")')ui.notify(f'Message: Failed Successfully :)')except Exception as e:apm.capture_exception()ui.notify(f'{e}')async def gen_custom_message(text_message):try:res = await io_bound(r.get, 'http://localhost:8000/custom_message/' + str(text_message))ui.notify(res.text)except Exception as e:apm.capture_exception()ui.notify(f'{e}')init(app)try:apm.capture_message('App Loaded, Hello World!')
except Exception as e:print('error: ' + e)if __name__ == '__main__':print('Please start the app with the "uvicorn" command as shown in the start.sh script')
golang版本（仅演示文件）
# syntax=docker/dockerfile:1FROM golang:1.20 AS builderWORKDIR /appCOPY go.mod go.sum ./RUN go mod downloadCOPY . .RUN go build -o main .FROM gcr.io/distroless/baseWORKDIR /appCOPY --from=builder /app/main .EXPOSE 8000CMD ["./main"]
package mainimport ("fmt""net/http""github.com/gin-gonic/gin""github.com/elastic/apm-agent-go/v2/apm""github.com/elastic/apm-agent-go/v2/module/apmhttp"
)func main() {// Initialize the Elastic APM agenttracer, err := apm.NewTracer("my_go_service", "development")if err != nil {fmt.Println("Failed to create APM tracer:", err)return}defer tracer.Close()// Create a Gin routerr := gin.Default()// Middleware to capture APM transactionsr.Use(func(c *gin.Context) {tx := tracer.StartTransaction(c.Request.URL.Path, "request")defer tx.End()c.Next()// Capture errors if anyif len(c.Errors) > 0 {for _, err := range c.Errors {apm.CaptureError(err.Err).Send()}}})// Custom message endpointr.GET("/custom_message/:message", func(c *gin.Context) {message := c.Param("message")apm.CaptureMessage(fmt.Sprintf("Custom Message: %s", message)).Send()c.JSON(http.StatusOK, gin.H{"message": fmt.Sprintf("Custom Message: %s", message)})})// Error endpointr.GET("/error", func(c *gin.Context) {defer func() {if r := recover(); r != nil {apm.CaptureError(fmt.Errorf("recovered from panic: %v", r)).Send()c.JSON(http.StatusInternalServerError, gin.H{"message": "Failed Successfully :)"})}}()// This will cause a panic_ = 1 / 0})// Start the serverif err := r.Run(":8000"); err != nil {fmt.Println("Failed to start server:", err)}
}
get-cert-details.sh
docker cp es-es01-1:/usr/share/elasticsearch/config/certs/ca/ca.crt /tmp/.
cat /tmp/ca.crt
echo 'ssl:';
echo ' certificate_authorities:';
echo ' - |';
cat /tmp/ca.crt | sed 's/^/ /'
openssl x509 -fingerprint -sha256 -noout -in /tmp/ca.crt | awk -F"=" {' print $2 '} | sed s/://g
