Fluent Bit

Установка

Windows

https://docs.fluentbit.io/manual/installation/windows

if (Get-Service | Select-Object Name | Select-String FluentBit){Stop-Service -Name "FluentBit"}
$FLUENTBIT_VERSION="2.0.9"
function fluentbit{
  $URL="https://releases.fluentbit.io/$(echo $FLUENTBIT_VERSION |  %{$_ -replace '(\d+)(.\d+)?(.\d+)','$1$2'})/fluent-bit-$FLUENTBIT_VERSION-win64.exe"
  $Filename = [System.IO.Path]::GetFileName($URL)
  if (!(Test-Path "$env:Temp\$Filename")){
    [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
    $dl = New-Object net.webclient
    $dl.Downloadfile($URL, "$env:Temp\$Filename")
  }
  ."$env:Temp\$Filename" /S
}
fluentbit
if (Get-Service | Select-Object Name | Select-String FluentBit){Start-Service -Name "FluentBit"}

Создать службу:

New-Service -Name "FluentBit" -BinaryPathName '"C:\Program Files\fluent-bit\bin\fluent-bit.exe" -c "C:\Program Files\fluent-bit\conf\fluentbit.conf"' -StartupType Automatic

или

sc create FluentBit binpath= "\"C:\Program Files\fluent-bit\fluent-bit.exe\" -c \"C:\Program Files\fluent-bit\conf\fluentbit.conf\""

Удалить службу:

(Get-WmiObject win32_service -Filter "name='FluentBit'").delete()

или

sc delete FluentBit

Запустить службу:

Start-Service FluentBit

или

sc start FluentBit

Примеры

Отправить json логи в loki (с назначением меток из имени директории):

C:\Program Files\fluent-bit\conf\fluentbit.conf

[SERVICE]
    flush        2
    daemon       Off
    log_level    info
    log_file     fluent-bit.log
    parsers_file parsers_custom.conf
    plugins_file plugins.conf
    http_server  On
    http_listen  0.0.0.0
    http_port    2020

[INPUT]
    name tail
    path C:\Logs\*\*.json
    path_key filename
    parser idscan_app
    db C:\Program Files\fluent-bit\app.web.db
    mem_buf_limit 128m
    buffer_chunk_size 1m
    buffer_max_size 128m
    tag app.web

[INPUT]
    name tail
    path C:\services\Logs\*\*.json
    path_key filename
    parser idscan_app
    db C:\Program Files\fluent-bit\app.console.db
    mem_buf_limit 128m
    buffer_chunk_size 1m
    buffer_max_size 128m
    tag app.console

[OUTPUT]
    name  loki
    match app.*
    host 172.16.4.5
    labels label=$label, lastpath=$lastpath
    remove_keys label,lastpath
    workers 3

[FILTER]
    name parser
    match app.*
    key_Name filename
    parser get_lastpath
    preserve_Key true
    reserve_Data true

[FILTER]
    Name modify
    Match app.console
    Add label app.console

[FILTER]
    Name modify
    Match app.web
    Add label app.web

[FILTER]
    name modify
    match   *
    add hostname ${HOSTNAME}

C:\Program Files\fluent-bit\conf\parsers_custom.conf

[PARSER]
    Name         app
    Format       json
    Time_Key     timestamp
    Time_Format  %Y-%m-%dT%H:%M:%S.%L%z

[PARSER]
    Name get_lastpath
    Format regex
    Regex ^(.*)\\(?<lastpath>.*)\\(.*\.json)$

C:\Logs\application\app.01.json

{"timestamp":"2022-09-30T00:03:29.6413608Z","level":"info","msg":"Request starting HTTP/1.1 GET http://admin.w02.domain.com/signin?ReturnUrl=%2F - -","protocol":"HTTP/1.1","method":"GET","contentType":null,"contentLength":null,"scheme":"http","host":"admin.w02.domain.com","pathBase":"","path":"/signin","queryString":"?ReturnUrl=%2F","hostingRequestStartingLog":"Request starting HTTP/1.1 GET http://admin.w02.domain.com/signin?ReturnUrl=%2F - -","eventId":{"Id":1,"Name":"RequestStarting"},"sourceContext":"Microsoft.AspNetCore.Hosting.Diagnostics","requestId":"8000309a-0006-e500-b63f-84710c7967bb","requestPath":"/signin"}
{"timestamp":"2022-09-30T00:03:29.6422548Z","level":"info","msg":"Request finished HTTP/1.1 GET http://admin.w02.domain.com/signin?ReturnUrl=%2F - - - 200 - text/html;+charset=utf-8 0.8760ms","elapsedMilliseconds":0.876,"statusCode":200,"contentType":"text/html; charset=utf-8","contentLength":null,"protocol":"HTTP/1.1","method":"GET","scheme":"http","host":"admin.w02.domain.com","pathBase":"","path":"/signin","queryString":"?ReturnUrl=%2F","hostingRequestFinishedLog":"Request finished HTTP/1.1 GET http://admin.w02.domain.com/signin?ReturnUrl=%2F - - - 200 - text/html;+charset=utf-8 0.8760ms","eventId":{"Id":2,"Name":"RequestFinished"},"sourceContext":"Microsoft.AspNetCore.Hosting.Diagnostics","requestId":"8000309a-0006-e500-b63f-84710c7967bb","requestPath":"/signin"}
{"timestamp":"2022-09-30T00:03:31.8757443Z","level":"info","msg":"Request starting HTTP/1.1 GET http://admin.w02.domain.com/hc - -","protocol":"HTTP/1.1","method":"GET","contentType":null,"contentLength":null,"scheme":"http","host":"admin.w02.domain.com","pathBase":"","path":"/hc","queryString":"","hostingRequestStartingLog":"Request starting HTTP/1.1 GET http://admin.w02.domain.com/hc - -","eventId":{"Id":1,"Name":"RequestStarting"},"sourceContext":"Microsoft.AspNetCore.Hosting.Diagnostics","requestId":"80003274-0005-fc00-b63f-84710c7967bb","requestPath":"/hc"}
{"timestamp":"2022-09-30T00:03:31.8765985Z","level":"info","msg":"Executed DbCommand (\"0\"ms) [Parameters=[\"\"], CommandType='Text', CommandTimeout='30']\"\r\n\"\"select 1\"","elapsed":"0","parameters":"","commandType":"Text","commandTimeout":30,"newLine":"\r\n","commandText":"select 1","eventId":{"Id":20101,"Name":"Microsoft.EntityFrameworkCore.Database.Command.CommandExecuted"},"sourceContext":"Microsoft.EntityFrameworkCore.Database.Command","healthCheckName":"db_health_check","requestId":"80003274-0005-fc00-b63f-84710c7967bb","requestPath":"/hc"}
{"timestamp":"2022-09-30T00:03:31.8768549Z","level":"info","msg":"Request finished HTTP/1.1 GET http://admin.w02.domain.com/hc - - - 200 - application/json 1.1182ms","elapsedMilliseconds":1.1182,"statusCode":200,"contentType":"application/json","contentLength":null,"protocol":"HTTP/1.1","method":"GET","scheme":"http","host":"admin.domain.com","pathBase":"","path":"/hc","queryString":"","hostingRequestFinishedLog":"Request finished HTTP/1.1 GET http://admin.w02.domain.com/hc - - - 200 - application/json 1.1182ms","eventId":{"Id":2,"Name":"RequestFinished"},"sourceContext":"Microsoft.AspNetCore.Hosting.Diagnostics","requestId":"80003274-0005-fc00-b63f-84710c7967bb","requestPath":"/hc"}

Ссылки

https://fluentbit.io
https://releases.fluentbit.io
https://github.com/fluent/fluent-bit-docs/blob/master/installation/windows.md
https://habr.com/ru/post/675728/ - многопоточность и файловый буфер