vector/config/examples/docs_example.toml at master · IAC-InfrastructureAsCode/vector · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# Set global options
data_dir = "/var/lib/vector"

# Ingest data by tailing one or more files
[sources.apache_logs]
type         = "file"
include      = ["/var/log/apache2/*.log"]    # supports globbing
ignore_older = 86400                         # 1 day

# Structure and parse the data
[transforms.apache_parser]
inputs       = ["apache_logs"]
type         = "remap"
drop_on_error = false
source       = '''
. = parse_apache_log!(.message)
'''

# Sample the data to save on cost
[transforms.apache_sample]
inputs       = ["apache_parser"]
type         = "sample"
rate         = 2                            # only keep 50% (1/`rate`)

# Send structured data to a short-term storage
[sinks.es_cluster]
inputs       = ["apache_sample"]            # only take sampled data
type         = "elasticsearch"
endpoint         = "http://79.12.221.222:9200"   # local or external host
[sinks.es_cluster.bulk]
index        = "vector-%Y-%m-%d"             # daily indices

# Send structured data to a cost-effective long-term storage
[sinks.s3_archives]
inputs         = ["apache_parser"]             # don't sample for S3
type           = "aws_s3"
region         = "us-east-1"
bucket         = "my-log-archives"
key_prefix     = "date=%Y-%m-%d"               # daily partitions, hive friendly format
compression    = "gzip"                        # compress final objects
framing.method = "newline_delimited"           # new line delimited...
encoding.codec = "json"                        # ...JSON
[sinks.s3_archives.batch]
max_bytes      = 10000000                      # 10mb uncompressed