Elastic cloud ingest pipeline template

  1[
  2  {
  3    "grok": {
  4      "field": "message",
  5      "patterns": [
  6        "%{IPORHOST:remote_addr} - %{USERNAME:remote_user} \\[%{HTTPDATE:time_local}\\] \\\"%{DATA:request}\\\" %{INT:status} %{NUMBER:bytes_sent} \\\"%{DATA:http_referer}\\\" \\\"%{DATA:http_user_agent}\\\" \\\"%{DATA:http_x_forwarded_for}\\\" rt=\\\"(?:%{NUMBER:request_time}|-)\\\" uct=\\\"(?:%{NUMBER:upstream_connect_time}|-)\\\" uht=\\\"(?:%{NUMBER:upstream_header_time}|-)\\\" urt=\\\"(?:%{NUMBER:upstream_response_time}|-)\\\""
  7      ],
  8      "ignore_missing": true,
  9      "if": "ctx.source.toLowerCase().contains('nginx')",
 10      "ignore_failure": true,
 11      "description": "nginx"
 12    }
 13  },
 14  {
 15    "grok": {
 16      "field": "source",
 17      "patterns": [
 18        "^/var/www/webapp/logs/(?<app_id>[^/]+)"
 19      ],
 20      "ignore_missing": true,
 21      "if": "ctx.source =~ /^\\/var\\/www\\/webapp\\//",
 22      "ignore_failure": true,
 23      "description": "java app_id"
 24    }
 25  },
 26  {
 27    "grok": {
 28      "field": "source",
 29      "patterns": [
 30        "^/var/applog/(?<app_id>[^/]+)"
 31      ],
 32      "ignore_missing": true,
 33      "if": "ctx.source =~ /^\\/var\\/applog\\//",
 34      "ignore_failure": true,
 35      "description": "php app_id"
 36    }
 37  },
 38  {
 39    "grok": {
 40      "field": "message",
 41      "patterns": [
 42        "^%{TIMESTAMP_ISO8601:Timestamp} (\\[)(?<IP>(?:[0-9\\.\\-]+))(\\])(\\[)(?<UserID>(?:[0-9\\-]+))(\\])(\\[)(?<SessionID>(?:[0-9a-zA-Z\\-]+))(\\])(\\[)(?<RequestID>(?:[0-9a-zA-Z\\-]+))(\\])(\\[)(?<SeverityLevel>(?:[a-zA-Z\\s]+))(\\])(\\[)(?<Category>(?:[0-9a-zA-Z\\\\_:]+))(\\])\\s+(?<Message>(?m:.*))$"
 43      ],
 44      "ignore_missing": true,
 45      "if": "ctx.source =~ /^\\/var\\/applog\\//",
 46      "ignore_failure": true,
 47      "description": "php message"
 48    }
 49  },
 50  {
 51    "grok": {
 52      "field": "message",
 53      "patterns": [
 54        "%{TIMESTAMP_ISO8601:log_time} (?<log_level>[^\\s]+)\\s+\\[(?<ecs_cluster>[^\\s]+)\\](.*)?"
 55      ],
 56      "ignore_missing": true,
 57      "if": "ctx.source =~ /^\\/var\\/www\\/webapp\\//",
 58      "ignore_failure": true,
 59      "description": "java message body extract"
 60    }
 61  },
 62  {
 63    "date": {
 64      "field": "log_time",
 65      "formats": [
 66        "ISO8601"
 67      ],
 68      "target_field": "log_timestamp",
 69      "timezone": "Atlantic/Stanley",
 70      "locale": "en",
 71      "if": "ctx.source =~ /^\\/var\\/www\\/webapp\\// ",
 72      "ignore_failure": true,
 73      "description": "java log_time"
 74    }
 75  },
 76  {
 77    "lowercase": {
 78      "field": "app_id",
 79      "ignore_missing": true
 80    }
 81  },
 82  {
 83    "lowercase": {
 84      "field": "type",
 85      "ignore_missing": true,
 86      "ignore_failure": true
 87    }
 88  },
 89  {
 90    "convert": {
 91      "field": "request_time",
 92      "type": "float",
 93      "ignore_missing": true,
 94      "ignore_failure": true
 95    }
 96  },
 97  {
 98    "convert": {
 99      "field": "upstream_connect_time",
100      "type": "float",
101      "ignore_missing": true,
102      "ignore_failure": true
103    }
104  },
105  {
106    "convert": {
107      "field": "upstream_response_time",
108      "type": "float",
109      "ignore_missing": true,
110      "ignore_failure": true
111    }
112  },
113  {
114    "convert": {
115      "field": "upstream_header_time",
116      "type": "float",
117      "ignore_missing": true,
118      "ignore_failure": true
119    }
120  },
121  {
122    "remove": {
123      "field": "host",
124      "ignore_missing": true,
125      "ignore_failure": true,
126      "description": "remove old host field"
127    }
128  },
129  {
130    "set": {
131      "field": "host",
132      "value": "{{beat.hostname}}",
133      "ignore_empty_value": true,
134      "description": "set new host field"
135    }
136  },
137  {
138    "set": {
139      "field": "env",
140      "value": "dev",
141      "override": false,
142      "ignore_empty_value": true,
143      "if": "ctx.beat.hostname.toLowerCase().contains('1-1')",
144      "description": "add dev env tag"
145    }
146  },
147  {
148    "set": {
149      "field": "env",
150      "value": "qa",
151      "ignore_empty_value": true,
152      "if": "ctx.beat.hostname.toLowerCase().contains('1-2')",
153      "description": "add qa env tag"
154    }
155  },
156  {
157    "set": {
158      "field": "env",
159      "value": "uat",
160      "ignore_empty_value": true,
161      "if": "ctx.beat.hostname.toLowerCase().contains('1-3')",
162      "description": "add uat env tag"
163    }
164  },
165  {
166    "set": {
167      "field": "env",
168      "value": "prod",
169      "ignore_empty_value": true,
170      "if": "ctx.beat.hostname.toLowerCase().contains('1-4')",
171      "description": "add prod env tag"
172    }
173  },
174  {
175    "date_index_name": {
176      "field": "@timestamp",
177      "date_rounding": "M",
178      "index_name_prefix": "{{env}}-{{app_id}}-",
179      "index_name_format": "yyyy-MM",
180      "date_formats": [
181        "ISO8601"
182      ]
183    }
184  }
185]