Extend logs v2 intake API to support ECS logging fields (#9349)

Author: lahsivjar

changelogs/head.asciidoc

@@ -17,6 +17,7 @@ Set error.id for OpenTelemetry exception span events {pull}9372[9372]
 
 [float]
 ==== Intake API Changes
+- experimental:[] Extend logs v2 intake API to support ECS logging fields {pull}9349[9349]
 - experimental:[] Add support for string timestamp format(`2006-01-02T15:04:05.999-0700`) {pull}9376[9376]
 
 [float]

docs/spec/v2/log.json

@@ -42,6 +42,14 @@ type Error struct {
 	// This may be set when a stack trace cannot be parsed.
 	StackTrace string
 
+	// Message holds an error message.
+	//
+	// Message is the ECS field equivalent of the APM field `error.log.message`.
+	Message string
+
+	// Type holds the type of the error.
+	Type string
+
 	Exception *Exception
 	Log       *ErrorLog
 }
@@ -72,6 +80,8 @@ func (e *Error) fields() mapstr.M {
 		exceptionFields := e.Exception.appendFields(nil, 0)
 		errorFields.set("exception", exceptionFields)
 	}
+	errorFields.maybeSetString("message", e.Message)
+	errorFields.maybeSetString("type", e.Type)
 	errorFields.maybeSetMapStr("log", e.logFields())
 	errorFields.maybeSetString("culprit", e.Culprit)
 	errorFields.maybeSetMapStr("custom", customFields(e.Custom))

internal/model/error.go

@@ -213,6 +213,16 @@ func TestErrorFields(t *testing.T) {
 				},
 			},
 		},
+		"withLogMessageAndType": {
+			Error: Error{
+				Message: "error log message",
+				Type:    "IllegalArgumentException",
+			},
+			Output: mapstr.M{
+				"message": "error log message",
+				"type":    "IllegalArgumentException",
+			},
+		},
 	}
 
 	for name, tc := range tests {

internal/model/error_test.go

@@ -38,14 +38,20 @@ type Event struct {
 	// Severity holds the numeric severity of the event for log events.
 	Severity int64
 
-	// Severity holds the action captured by the event for log events.
+	// Action holds the action captured by the event for log events.
 	Action string
+
+	// Dataset holds the the dataset which produces the events. If an event
+	// source publishes more than one type of log or events (e.g. access log,
+	// error log), the dataset is used to specify which one the event comes from.
+	Dataset string
 }
 
 func (e *Event) fields() mapstr.M {
 	var fields mapStr
 	fields.maybeSetString("outcome", e.Outcome)
 	fields.maybeSetString("action", e.Action)
+	fields.maybeSetString("dataset", e.Dataset)
 	if e.Severity > 0 {
 		fields.set("severity", e.Severity)
 	}

internal/model/event.go

@@ -0,0 +1,87 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package model
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/elastic-agent-libs/mapstr"
+)
+
+func TestEventFieldsEmpty(t *testing.T) {
+	event := APMEvent{}
+	beatEvent := event.BeatEvent()
+	assert.Empty(t, beatEvent.Fields)
+}
+
+func TestEventFields(t *testing.T) {
+	tests := map[string]struct {
+		Event  Event
+		Output mapstr.M
+	}{
+		"withOutcome": {
+			Event: Event{Outcome: "success"},
+			Output: mapstr.M{
+				"outcome": "success",
+			},
+		},
+		"withAction": {
+			Event: Event{Action: "process-started"},
+			Output: mapstr.M{
+				"action": "process-started",
+			},
+		},
+		"withDataset": {
+			Event: Event{Dataset: "access-log"},
+			Output: mapstr.M{
+				"dataset": "access-log",
+			},
+		},
+		"withSeverity": {
+			Event: Event{Severity: 1},
+			Output: mapstr.M{
+				"severity": int64(1),
+			},
+		},
+		"withDuration": {
+			Event: Event{Duration: time.Minute},
+			Output: mapstr.M{
+				"duration": int64(60000000000),
+			},
+		},
+		"withOutcomeActionDataset": {
+			Event: Event{Outcome: "success", Action: "process-started", Dataset: "access-log"},
+			Output: mapstr.M{
+				"outcome": "success",
+				"action":  "process-started",
+				"dataset": "access-log",
+			},
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			event := APMEvent{Event: tc.Event}
+			beatEvent := event.BeatEvent()
+			assert.Equal(t, tc.Output, beatEvent.Fields["event"])
+		})
+	}
+}

internal/model/event_test.go

@@ -34,10 +34,33 @@ var (
 type Log struct {
 	// Level holds the log level of the log event.
 	Level string
+	// Logger holds the name of the logger instance.
+	Logger string
+	Origin LogOrigin
+}
+
+// LogOrigin holds information about the origin of the log.
+type LogOrigin struct {
+	File         LogOriginFile
+	FunctionName string
+}
+
+// LogOriginFile holds information about the file and the line of the origin of the log.
+type LogOriginFile struct {
+	Name string
+	Line int
 }
 
 func (e Log) fields() mapstr.M {
-	var fields mapStr
+	var fields, origin, file mapStr
 	fields.maybeSetString("level", e.Level)
+	fields.maybeSetString("logger", e.Logger)
+	origin.maybeSetString("function", e.Origin.FunctionName)
+	file.maybeSetString("name", e.Origin.File.Name)
+	if e.Origin.File.Line > 0 {
+		file.set("line", e.Origin.File.Line)
+	}
+	origin.maybeSetMapStr("file", mapstr.M(file))
+	fields.maybeSetMapStr("origin", mapstr.M(origin))
 	return mapstr.M(fields)
 }

internal/model/log.go

@@ -36,10 +36,36 @@ func TestLogTransform(t *testing.T) {
 		},
 		{
 			Log: Log{
-				Level: "warn",
+				Level:  "warn",
+				Logger: "bootstrap",
 			},
 			Output: mapstr.M{
-				"level": "warn",
+				"level":  "warn",
+				"logger": "bootstrap",
+			},
+		},
+		{
+			Log: Log{
+				Level:  "warn",
+				Logger: "bootstrap",
+				Origin: LogOrigin{
+					File: LogOriginFile{
+						Name: "testFile",
+						Line: 12,
+					},
+					FunctionName: "testFunc",
+				},
+			},
+			Output: mapstr.M{
+				"level":  "warn",
+				"logger": "bootstrap",
+				"origin": mapstr.M{
+					"function": "testFunc",
+					"file": mapstr.M{
+						"name": "testFile",
+						"line": 12,
+					},
+				},
 			},
 		},
 	}

internal/model/log_test.go

@@ -53,7 +53,7 @@ func generateV2() {
 		panic(err)
 	}
 	generateCode(p, pkg, parsed, []string{"metadataRoot", "errorRoot", "metricsetRoot", "spanRoot", "transactionRoot", "logRoot"})
-	generateJSONSchema(p, pkg, parsed, []string{"metadata", "errorEvent", "metricset", "span", "transaction", "log"})
+	generateJSONSchema(p, pkg, parsed, []string{"metadata", "errorEvent", "metricset", "span", "transaction"})
 }
 
 func generateV3RUM() {