Содержание

promtail

Справка:

docker run --rm grafana/promtail -help

Установка

Windows:

powershell

powershell

function promtail{
  $URL="https://github.com/grafana/loki/releases/download/v2.8.2/promtail-windows-amd64.exe.zip"
  $PROMTAILPATH="$env:PROGRAMFILES\promtail"
  $Filename = [System.IO.Path]::GetFileName($URL)
  if (!(Test-Path "$env:Temp\$Filename")){
    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
    $dl = New-Object net.webclient
    $dl.Downloadfile($URL, "$env:Temp\$Filename")
  }
 
  New-Item "$PROMTAILPATH" -type directory -Force
  Add-Type -AssemblyName System.IO.Compression.FileSystem
  [IO.Compression.ZipFile]::ExtractToDirectory("$env:Temp\$Filename","$PROMTAILPATH")
 
  $URL="https://www.nssm.cc/ci/nssm-2.24-101-g897c7ad.zip"
  $Filename = [System.IO.Path]::GetFileName($URL)
  if (!(Test-Path "$env:Temp\$Filename")){
    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
    $dl = New-Object net.webclient
    $dl.Downloadfile($URL, "$env:Temp\$Filename")
  }
  Expand-Archive "$env:Temp\$Filename" -DestinationPath "$env:Temp"
  Copy-Item $env:Temp\nssm-*\win64\nssm.exe $PROMTAILPATH
  .$PROMTAILPATH\nssm.exe install promtail "$PROMTAILPATH\promtail-windows-amd64.exe" "-config.file" '""C:\Program Files\promtail\config.yaml""'
  .$PROMTAILPATH\nssm.exe set promtail AppStderr $PROMTAILPATH\promtail.log
  .$PROMTAILPATH\nssm.exe set promtail AppStdout $PROMTAILPATH\promtail-nssm.log
}
promtail


Конфигурация с импортом json логов из C:/Logs/*/*.json:

config.yaml

config.yaml

server:
  http_listen_port: 9080
  grpc_listen_port: 0
positions:
  filename: "C:\\Program Files\\Promtail\\positions.yaml"
clients:
  - url: http://loki/loki/api/v1/push
scrape_configs:
- job_name: system
  static_configs:
  - targets:
      - localhost
    labels:
      job: filejson
      __path__: L:\Logs\*\*.json
      host: I01
      label: app.web
  pipeline_stages:
  - match:
      selector: '{job="filejson"}'
      stages:
      - regex:
          source: filename
          expression: ^L:\\Logs\\(?P<lastpath>.*)\\.*.json$
      - labels:
          lastpath:


Установить службу с помощью WinSW:

WinSW-x64.xml

WinSW-x64.xml

<service>
  <id>promtail</id>
  <name>Promtail (powered by WinSW)</name>
  <description>Promtail (powered by WinSW)</description>
  <env name="PROGRAMFILES" value="%BASE%"/>
  <executable>%BASE%\promtail-windows-amd64.exe</executable>
  <arguments>-config.file "%BASE%\config.yaml"</arguments>
  <logpath>%BASE%\logs</logpath>
</service>


Тестовый запуск:

& "C:\Program Files\promtail\promtail-windows-amd64.exe" "-config.file" "C:\Program Files\promtail\config.yaml" -dry-run

Запуск службы:

Start-Service -Name promtail

https://github.com/grafana/loki/releases - Дистрибутивы

https://grafana.com/docs/loki/latest/clients/promtail/configuration/

Примеры

Отправить json логи в loki (с назначением меток из имени директории):

/data/logs/promtail-docker-config.yaml

/data/logs/promtail-docker-config.yaml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
- job_name: system
  static_configs:
  - targets:
      - localhost
    labels:
      job: filejson
      __path__: /data/logs/*/*/*/*.json
  pipeline_stages:
    - match:
        selector: '{job="filejson"}'
        stages:
        - regex:
           source: filename
           expression: "^/data/logs/(?P<label>[^/].*)/(?P<host>[^/].*)/(?P<lastpath>[^/].*)/.*.json$"
        - labels:
             label:
             host:
             lastpath:

docker-compose.yaml

docker-compose.yaml

version: "3.7"

services:
  loki:
    image: grafana/loki:2.8.2
    command: |
      -config.file=/etc/loki/local-config.yaml
      -distributor.ingestion-burst-size-mb=32
      -distributor.ingestion-rate-limit-mb=16
      -server.grpc-max-recv-msg-size-bytes=16777216
      -ingester.per-stream-rate-limit=16MB
      -ingester.max-chunk-age=24h
      -validation.reject-old-samples.max-age=24h
      -validation.reject-old-samples=false
    ports:
      - mode: host
        protocol: tcp
        published: 3100
        target: 3100

  promtail:
    image: grafana/promtail:2.8.2
    volumes:
      - /data/logs:/data/logs
    command: -config.file=/data/logs/promtail-docker-config.yaml

/data/logs/app.web/w02/admin.domain.com/log20230516.json

/data/logs/app.web/w02/admin.domain.com/log20230516.json

{"timestamp":"2022-09-30T00:03:29.6413608Z","level":"info","msg":"Request starting HTTP/1.1 GET http://admin.w02.domain.com/signin?ReturnUrl=%2F - -","protocol":"HTTP/1.1","method":"GET","contentType":null,"contentLength":null,"scheme":"http","host":"admin.w02.domain.com","pathBase":"","path":"/signin","queryString":"?ReturnUrl=%2F","hostingRequestStartingLog":"Request starting HTTP/1.1 GET http://admin.w02.domain.com/signin?ReturnUrl=%2F - -","eventId":{"Id":1,"Name":"RequestStarting"},"sourceContext":"Microsoft.AspNetCore.Hosting.Diagnostics","requestId":"8000309a-0006-e500-b63f-84710c7967bb","requestPath":"/signin"}
{"timestamp":"2022-09-30T00:03:29.6422548Z","level":"info","msg":"Request finished HTTP/1.1 GET http://admin.w02.domain.com/signin?ReturnUrl=%2F - - - 200 - text/html;+charset=utf-8 0.8760ms","elapsedMilliseconds":0.876,"statusCode":200,"contentType":"text/html; charset=utf-8","contentLength":null,"protocol":"HTTP/1.1","method":"GET","scheme":"http","host":"admin.w02.domain.com","pathBase":"","path":"/signin","queryString":"?ReturnUrl=%2F","hostingRequestFinishedLog":"Request finished HTTP/1.1 GET http://admin.w02.domain.com/signin?ReturnUrl=%2F - - - 200 - text/html;+charset=utf-8 0.8760ms","eventId":{"Id":2,"Name":"RequestFinished"},"sourceContext":"Microsoft.AspNetCore.Hosting.Diagnostics","requestId":"8000309a-0006-e500-b63f-84710c7967bb","requestPath":"/signin"}
{"timestamp":"2022-09-30T00:03:31.8757443Z","level":"info","msg":"Request starting HTTP/1.1 GET http://admin.w02.domain.com/hc - -","protocol":"HTTP/1.1","method":"GET","contentType":null,"contentLength":null,"scheme":"http","host":"admin.w02.domain.com","pathBase":"","path":"/hc","queryString":"","hostingRequestStartingLog":"Request starting HTTP/1.1 GET http://admin.w02.domain.com/hc - -","eventId":{"Id":1,"Name":"RequestStarting"},"sourceContext":"Microsoft.AspNetCore.Hosting.Diagnostics","requestId":"80003274-0005-fc00-b63f-84710c7967bb","requestPath":"/hc"}
{"timestamp":"2022-09-30T00:03:31.8765985Z","level":"info","msg":"Executed DbCommand (\"0\"ms) [Parameters=[\"\"], CommandType='Text', CommandTimeout='30']\"\r\n\"\"select 1\"","elapsed":"0","parameters":"","commandType":"Text","commandTimeout":30,"newLine":"\r\n","commandText":"select 1","eventId":{"Id":20101,"Name":"Microsoft.EntityFrameworkCore.Database.Command.CommandExecuted"},"sourceContext":"Microsoft.EntityFrameworkCore.Database.Command","healthCheckName":"db_health_check","requestId":"80003274-0005-fc00-b63f-84710c7967bb","requestPath":"/hc"}
{"timestamp":"2022-09-30T00:03:31.8768549Z","level":"info","msg":"Request finished HTTP/1.1 GET http://admin.w02.domain.com/hc - - - 200 - application/json 1.1182ms","elapsedMilliseconds":1.1182,"statusCode":200,"contentType":"application/json","contentLength":null,"protocol":"HTTP/1.1","method":"GET","scheme":"http","host":"admin.domain.com","pathBase":"","path":"/hc","queryString":"","hostingRequestFinishedLog":"Request finished HTTP/1.1 GET http://admin.w02.domain.com/hc - - - 200 - application/json 1.1182ms","eventId":{"Id":2,"Name":"RequestFinished"},"sourceContext":"Microsoft.AspNetCore.Hosting.Diagnostics","requestId":"80003274-0005-fc00-b63f-84710c7967bb","requestPath":"/hc"}