Grafana, Loki, and Vector

Grafana is an open-source monitoring and observability platform for querying, visualising, and alerting on data from multiple sources.

This compose file deploys a complete log aggregation stack: Loki ingests and stores logs with TSDB indexing and 28-day retention, Vector collects UDP syslogs on port 514 from UniFi network devices and forwards them to Loki, and Grafana provides the query and dashboard interface. All persistent data is stored on NFS volumes. Grafana and Loki are proxied through a Traefik reverse proxy with Cloudflare TLS certificates. Example Loki and Vector configuration files are included below.

For more information, check out the official Grafana documentation.

Docker Compose

# compose.yaml

services:
  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    restart: unless-stopped
    # ports:
    #  - 3000:3000
    networks:
      - grafana
      - grafana_proxy
    volumes:
      - type: volume
        source: docker-nfs
        target: /var/lib/grafana
        volume:
          subpath: grafana/grafana/data
      - type: volume
        source: docker-nfs
        target: /usr/share/grafana/conf
        volume:
          subpath: grafana/grafana/config
    environment:
      GF_SERVER_ROOT_URL: https://grafana.${TRAEFIK_BASE_URL}
      TZ: Europe/London
    labels:
      - "traefik.enable=true"
      - "traefik.docker.network=grafana_proxy"

      - "traefik.http.services.grafana.loadbalancer.server.port=3000"

      - "traefik.http.routers.grafana.rule=Host(`grafana.${TRAEFIK_BASE_URL}`)"
      - "traefik.http.routers.grafana.entrypoints=websecure"
      - "traefik.http.routers.grafana.tls.certresolver=cloudflare"

  loki:
    image: grafana/loki:latest
    container_name: loki
    restart: unless-stopped
    # ports:
    #  - 3100:3100
    networks:
      - grafana
      - grafana_proxy
    volumes:
      - type: volume
        source: docker-nfs
        target: /etc/loki
        volume:
          subpath: grafana/loki/config
      - type: volume
        source: docker-nfs
        target: /loki-data
        volume:
          subpath: grafana/loki/data
    command: -config.file=/etc/loki/loki-config.yaml
    environment:
      TZ: Europe/London
    labels:
      - "traefik.enable=true"
      - "traefik.docker.network=grafana_proxy"

      - "traefik.http.services.loki.loadbalancer.server.port=3100"

      - "traefik.http.routers.loki.rule=Host(`loki.${TRAEFIK_BASE_URL}`)"
      - "traefik.http.routers.loki.entrypoints=websecure"
      - "traefik.http.routers.loki.tls.certresolver=cloudflare"

  vector:
    image: timberio/vector:latest-debian
    container_name: vector
    restart: unless-stopped
    ports:
      - 514:514/udp
    networks:
      - grafana
    volumes:
      - type: volume
        source: docker-nfs
        target: /etc/vector
        volume:
          subpath: vector/config
    command: --config /etc/vector/vector.toml
    environment:
      TZ: Europe/London

networks:
  grafana:
    name: grafana
  grafana_proxy:
    name: grafana_proxy

volumes:
  docker-nfs:
    driver: local
    driver_opts:
      type: nfs
      o: addr=xxx.xxx.xxx.xxx,nolock,soft,rw,nfsvers=4.2
      device: :/mnt/nfs-volume

Environment Variables

# .env

TRAEFIK_BASE_URL=example.com

Configuration

Loki Configuration

The Loki and Vector configuration files are located in the mounted config directories.

For Loki, you can use the following example configuration as a starting point: 

# loki-config.yaml

auth_enabled: false

limits_config:
  allow_structured_metadata: true
  volume_enabled: true
  max_query_lookback: 672h # 28 days
  retention_period: 672h # 28 days

server:
  http_listen_port: 3100

common:
  ring:
    instance_addr: 0.0.0.0
    kvstore:
      store: inmemory
  replication_factor: 1
  path_prefix: /loki-data

schema_config:
  configs:
    - from: 2020-05-15
      store: tsdb
      object_store: filesystem
      schema: v13
      index:
        prefix: index_
        period: 24h

storage_config:
  tsdb_shipper:
    active_index_directory: /loki-data/index
    cache_location: /loki-data/index_cache
  filesystem:
    directory: /loki-data/chunks

pattern_ingester:
  enabled: true

analytics:
  reporting_enabled: false

compactor:
  working_directory: /loki-data/retention
  delete_request_store: filesystem
  retention_enabled: true
  compaction_interval: 10m
  retention_delete_delay: 24h
  retention_delete_worker_count: 50

Vector Configuration

For Vector, you can use the following example configuration as a starting point: 

# vector.toml

[sources.source_syslog]
  type = "syslog"
  address = "0.0.0.0:514"
  max_length = 102_400
  mode = "udp"
  path = "/var/log/syslog_udp"

[sinks.sink_loki]
  type = "loki"
  inputs = ["source_syslog"]
  endpoint = "http://loki:3100"
  out_of_order_action = "rewrite_timestamp"
  encoding.codec = "json"

  [sinks.sink_loki.labels]
    application = "unifi"
    host = "gateway-fiber"

Traefik

Modify your Traefik compose.yaml to enable access to the grafana_proxy network. 

# compose.yaml (excerpt)

services:
  traefik:
    image: traefik:latest
    container_name: traefik
    ...
    networks:
      - traefik
      # here
      - grafana_proxy
    ...

networks:
  # here
  grafana_proxy:
    name: grafana_proxy