Add support for Schema Processor in Logs Pipelines

.generator/schemas/v1/openapi.yaml

@@ -6331,6 +6331,7 @@ components:
       - $ref: '#/components/schemas/LogsSpanRemapper'
       - $ref: '#/components/schemas/LogsArrayProcessor'
       - $ref: '#/components/schemas/LogsDecoderProcessor'
+      - $ref: '#/components/schemas/LogsSchemaProcessor'
     LogsQueryCompute:
       description: Define computation for a log query.
       properties:
@@ -6392,6 +6393,243 @@ components:
             periods.
           type: string
       type: object
+    LogsSchemaCategoryMapper:
+      description: "Use the Schema Category Mapper to categorize log event into enum
+        fields.\nIn the case of OCSF, they can be used to map sibling fields which
+        are composed of an ID and a name.\n\n**Notes**:\n\n- The syntax of the query
+        is the one of Logs Explorer search bar.\n  The query can be done on any log
+        attribute or tag, whether it is a facet or not.\n  Wildcards can also be used
+        inside your query.\n- Categories are executed in order and processing stops
+        at the first match.\n  Make sure categories are properly ordered in case a
+        log could match multiple queries.\n- Sibling fields always have a numerical
+        ID field and a human-readable string name.\n- A fallback section handles cases
+        where the name or ID value matches a specific value.\n  If the name matches
+        \"Other\" or the ID matches 99, the value of the sibling name field will be
+        pulled from a source field from the original log."
+      properties:
+        categories:
+          description: 'Array of filters to match or not a log and their
+
+            corresponding `name` to assign a custom value to the log.'
+          example:
+          - filter:
+              query: '@eventName:(ConsoleLogin OR ExternalIdPDirectoryLogin OR UserAuthentication
+                OR Authenticate)'
+            id: 1
+            name: Logon
+          - filter:
+              query: '@eventName:*'
+            id: 99
+            name: Other
+          items:
+            $ref: '#/components/schemas/LogsSchemaCategoryMapperCategory'
+          type: array
+        fallback:
+          $ref: '#/components/schemas/LogsSchemaCategoryMapperFallback'
+        name:
+          description: Name of the logs schema category mapper.
+          example: activity_id and activity_name
+          type: string
+        targets:
+          $ref: '#/components/schemas/LogsSchemaCategoryMapperTargets'
+        type:
+          $ref: '#/components/schemas/LogsSchemaCategoryMapperType'
+      required:
+      - categories
+      - targets
+      - type
+      - name
+      type: object
+    LogsSchemaCategoryMapperCategory:
+      description: Object describing the logs filter with corresponding category ID
+        and name assignment.
+      properties:
+        filter:
+          $ref: '#/components/schemas/LogsFilter'
+        id:
+          description: ID to inject into the category.
+          example: 1
+          format: int64
+          type: integer
+        name:
+          description: Value to assign to target schema field.
+          example: Password Change
+          type: string
+      required:
+      - filter
+      - id
+      - name
+      type: object
+    LogsSchemaCategoryMapperFallback:
+      description: Used to override hardcoded category values with a value pulled
+        from a source attribute on the log.
+      properties:
+        sources:
+          additionalProperties:
+            items:
+              type: string
+            type: array
+          description: Fallback sources used to populate value of field.
+          example: {}
+          type: object
+        values:
+          additionalProperties:
+            type: string
+          description: Values that define when the fallback is used.
+          example: {}
+          type: object
+      type: object
+    LogsSchemaCategoryMapperTargets:
+      description: Name of the target attributes which value is defined by the matching
+        category.
+      properties:
+        id:
+          description: ID of the field to map log attributes to.
+          example: ocsf.activity_id
+          type: string
+        name:
+          description: Name of the field to map log attributes to.
+          example: ocsf.activity_name
+          type: string
+      type: object
+    LogsSchemaCategoryMapperType:
+      description: Type of logs schema category mapper.
+      enum:
+      - schema-category-mapper
+      example: schema-category-mapper
+      type: string
+      x-enum-varnames:
+      - SCHEMA_CATEGORY_MAPPER
+    LogsSchemaData:
+      description: Configuration of the schema data to use.
+      properties:
+        class_name:
+          description: Class name of the schema to use.
+          example: Account Change
+          type: string
+        class_uid:
+          description: Class UID of the schema to use.
+          example: 3001
+          format: int64
+          type: integer
+        profiles:
+          description: Optional list of profiles to modify the schema.
+          example:
+          - security_control
+          - host
+          items:
+            type: string
+          type: array
+        schema_type:
+          description: Type of schema to use.
+          example: ocsf
+          type: string
+        version:
+          description: Version of the schema to use.
+          example: 1.5.0
+          type: string
+      required:
+      - schema_type
+      - version
+      - class_uid
+      - class_name
+      type: object
+    LogsSchemaMapper:
+      description: Configuration of the schema processor mapper to use.
+      oneOf:
+      - $ref: '#/components/schemas/LogsSchemaRemapper'
+      - $ref: '#/components/schemas/LogsSchemaCategoryMapper'
+    LogsSchemaProcessor:
+      description: A processor that has additional validations and checks for a given
+        schema. Currently supported schema types include OCSF.
+      properties:
+        is_enabled:
+          default: false
+          description: Whether or not the processor is enabled.
+          type: boolean
+        mappers:
+          description: The `LogsSchemaProcessor` `mappers`.
+          example:
+          - name: Map userIdentity to ocsf.user.uid
+            sources:
+            - userIdentity.principalId
+            target: ocsf.user.uid
+            type: schema-remapper
+          items:
+            $ref: '#/components/schemas/LogsSchemaMapper'
+          type: array
+        name:
+          description: Name of the processor.
+          example: Map additionalEventData.LoginTo to ocsf.dst_endpoint.svc_name
+          type: string
+        schema:
+          $ref: '#/components/schemas/LogsSchemaData'
+        type:
+          $ref: '#/components/schemas/LogsSchemaProcessorType'
+      required:
+      - name
+      - mappers
+      - type
+      - schema
+      type: object
+    LogsSchemaProcessorType:
+      default: schema-processor
+      description: Type of logs schema processor.
+      enum:
+      - schema-processor
+      example: schema-processor
+      type: string
+      x-enum-varnames:
+      - SCHEMA_PROCESSOR
+    LogsSchemaRemapper:
+      description: The schema remapper maps source log fields to their correct fields.
+      properties:
+        name:
+          description: Name of the logs schema remapper.
+          example: Map userIdentity.principalId, responseElements.role.roleId, responseElements.user.userId
+            to ocsf.user.uid
+          type: string
+        override_on_conflict:
+          default: false
+          description: Override or not the target element if already set.
+          type: boolean
+        preserve_source:
+          default: false
+          description: Remove or preserve the remapped source element.
+          type: boolean
+        sources:
+          description: Array of source attributes.
+          example:
+          - userIdentity.principalId
+          - responseElements.role.roleId
+          - responseElements.user.userId
+          items:
+            description: Attribute used as a source to remap its value to the target
+              attribute.
+            type: string
+          type: array
+        target:
+          description: Target field to map log source field to.
+          example: ocsf.user.uid
+          type: string
+        target_format:
+          $ref: '#/components/schemas/TargetFormatType'
+        type:
+          $ref: '#/components/schemas/LogsSchemaRemapperType'
+      required:
+      - name
+      - sources
+      - target
+      - type
+      type: object
+    LogsSchemaRemapperType:
+      description: Type of logs schema remapper.
+      enum:
+      - schema-remapper
+      example: schema-remapper
+      type: string
+      x-enum-varnames:
+      - SCHEMA_REMAPPER
     LogsServiceRemapper:
       description: 'Use this processor if you want to assign one or more attributes
         as the official service.

cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Schema-Processor-and-preserve_source-false-returns-OK-response_4156521119/frozen.json

@@ -0,0 +1 @@
+"2025-10-22T19:11:58.774Z"

cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Schema-Processor-and-preserve_source-false-returns-OK-response_4156521119/recording.har

@@ -0,0 +1,110 @@
+{
+  "log": {
+    "_recordingName": "Logs Pipelines/Create a pipeline with Schema Processor and preserve_source false returns \"OK\" response",
+    "creator": {
+      "comment": "persister:fs",
+      "name": "Polly.JS",
+      "version": "6.0.5"
+    },
+    "entries": [
+      {
+        "_id": "98435b4b38f66ccde06a72100d3c817d",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 3314,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "application/json"
+            },
+            {
+              "_fromType": "array",
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 575,
+          "httpVersion": "HTTP/1.1",
+          "method": "POST",
+          "postData": {
+            "mimeType": "application/json",
+            "params": [],
+            "text": "{\"filter\":{\"query\":\"source:python\"},\"name\":\"testSchemaProcessor\",\"processors\":[{\"is_enabled\":true,\"mappers\":[{\"categories\":[{\"filter\":{\"query\":\"@eventName:(*Create*)\"},\"id\":1,\"name\":\"Create\"},{\"filter\":{\"query\":\"@eventName:(ChangePassword OR PasswordUpdated)\"},\"id\":3,\"name\":\"Password Change\"},{\"filter\":{\"query\":\"@eventName:(*Attach*)\"},\"id\":7,\"name\":\"Attach Policy\"},{\"filter\":{\"query\":\"@eventName:(*Detach* OR *Remove*)\"},\"id\":8,\"name\":\"Detach Policy\"},{\"filter\":{\"query\":\"@eventName:(*Delete*)\"},\"id\":6,\"name\":\"Delete\"},{\"filter\":{\"query\":\"@eventName:*\"},\"id\":99,\"name\":\"Other\"}],\"fallback\":{\"sources\":{\"ocsf.activity_name\":[\"eventName\"]},\"values\":{\"ocsf.activity_id\":\"99\",\"ocsf.activity_name\":\"Other\"}},\"name\":\"activity_id and activity_name\",\"targets\":{\"id\":\"ocsf.activity_id\",\"name\":\"ocsf.activity_name\"},\"type\":\"schema-category-mapper\"},{\"categories\":[{\"filter\":{\"query\":\"-@errorCode:*\"},\"id\":1,\"name\":\"Success\"},{\"filter\":{\"query\":\"@errorCode:*\"},\"id\":2,\"name\":\"Failure\"}],\"name\":\"status\",\"targets\":{\"id\":\"ocsf.status_id\",\"name\":\"ocsf.status\"},\"type\":\"schema-category-mapper\"},{\"categories\":[{\"filter\":{\"query\":\"@eventName:*\"},\"id\":1,\"name\":\"Informational\"}],\"name\":\"Set default severity\",\"targets\":{\"id\":\"ocsf.severity_id\",\"name\":\"ocsf.severity\"},\"type\":\"schema-category-mapper\"},{\"name\":\"Map userIdentity to ocsf.user.uid\",\"preserve_source\":false,\"sources\":[\"userIdentity.principalId\",\"responseElements.role.roleId\",\"responseElements.user.userId\"],\"target\":\"ocsf.user.uid\",\"type\":\"schema-remapper\"},{\"name\":\"Map userName to ocsf.user.name\",\"preserve_source\":false,\"sources\":[\"requestParameters.userName\",\"responseElements.role.roleName\",\"requestParameters.roleName\",\"responseElements.user.userName\"],\"target\":\"ocsf.user.name\",\"type\":\"schema-remapper\"},{\"name\":\"Map api to ocsf.api\",\"preserve_source\":false,\"sources\":[\"api\"],\"target\":\"ocsf.api\",\"type\":\"schema-remapper\"},{\"name\":\"Map user to ocsf.user\",\"preserve_source\":false,\"sources\":[\"user\"],\"target\":\"ocsf.user\",\"type\":\"schema-remapper\"},{\"name\":\"Map actor to ocsf.actor\",\"preserve_source\":false,\"sources\":[\"actor\"],\"target\":\"ocsf.actor\",\"type\":\"schema-remapper\"},{\"name\":\"Map cloud to ocsf.cloud\",\"preserve_source\":false,\"sources\":[\"cloud\"],\"target\":\"ocsf.cloud\",\"type\":\"schema-remapper\"},{\"name\":\"Map http_request to ocsf.http_request\",\"preserve_source\":false,\"sources\":[\"http_request\"],\"target\":\"ocsf.http_request\",\"type\":\"schema-remapper\"},{\"name\":\"Map metadata to ocsf.metadata\",\"preserve_source\":false,\"sources\":[\"metadata\"],\"target\":\"ocsf.metadata\",\"type\":\"schema-remapper\"},{\"name\":\"Map time to ocsf.time\",\"preserve_source\":false,\"sources\":[\"time\"],\"target\":\"ocsf.time\",\"type\":\"schema-remapper\"},{\"name\":\"Map src_endpoint to ocsf.src_endpoint\",\"preserve_source\":false,\"sources\":[\"src_endpoint\"],\"target\":\"ocsf.src_endpoint\",\"type\":\"schema-remapper\"},{\"name\":\"Map severity to ocsf.severity\",\"preserve_source\":false,\"sources\":[\"severity\"],\"target\":\"ocsf.severity\",\"type\":\"schema-remapper\"},{\"name\":\"Map severity_id to ocsf.severity_id\",\"preserve_source\":false,\"sources\":[\"severity_id\"],\"target\":\"ocsf.severity_id\",\"type\":\"schema-remapper\"}],\"name\":\"Apply OCSF schema for 3001\",\"schema\":{\"class_name\":\"Account Change\",\"class_uid\":3001,\"profiles\":[\"cloud\",\"datetime\"],\"schema_type\":\"ocsf\",\"version\":\"1.5.0\"},\"type\":\"schema-processor\"}],\"tags\":[]}"
+          },
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines"
+        },
+        "response": {
+          "bodySize": 3843,
+          "content": {
+            "mimeType": "application/json",
+            "size": 3843,
+            "text": "{\"id\":\"-qkKiJPYTne-113i8XJ_Nw\",\"type\":\"pipeline\",\"name\":\"testSchemaProcessor\",\"is_enabled\":false,\"is_read_only\":false,\"filter\":{\"query\":\"source:python\"},\"processors\":[{\"name\":\"Apply OCSF schema for 3001\",\"is_enabled\":true,\"mappers\":[{\"name\":\"activity_id and activity_name\",\"categories\":[{\"filter\":{\"query\":\"@eventName:(*Create*)\"},\"name\":\"Create\",\"id\":1},{\"filter\":{\"query\":\"@eventName:(ChangePassword OR PasswordUpdated)\"},\"name\":\"Password Change\",\"id\":3},{\"filter\":{\"query\":\"@eventName:(*Attach*)\"},\"name\":\"Attach Policy\",\"id\":7},{\"filter\":{\"query\":\"@eventName:(*Detach* OR *Remove*)\"},\"name\":\"Detach Policy\",\"id\":8},{\"filter\":{\"query\":\"@eventName:(*Delete*)\"},\"name\":\"Delete\",\"id\":6},{\"filter\":{\"query\":\"@eventName:*\"},\"name\":\"Other\",\"id\":99}],\"targets\":{\"name\":\"ocsf.activity_name\",\"id\":\"ocsf.activity_id\"},\"fallback\":{\"values\":{\"ocsf.activity_id\":\"99\",\"ocsf.activity_name\":\"Other\"},\"sources\":{\"ocsf.activity_name\":[\"eventName\"]}},\"type\":\"schema-category-mapper\"},{\"name\":\"status\",\"categories\":[{\"filter\":{\"query\":\"-@errorCode:*\"},\"name\":\"Success\",\"id\":1},{\"filter\":{\"query\":\"@errorCode:*\"},\"name\":\"Failure\",\"id\":2}],\"targets\":{\"name\":\"ocsf.status\",\"id\":\"ocsf.status_id\"},\"fallback\":{\"values\":{},\"sources\":{}},\"type\":\"schema-category-mapper\"},{\"name\":\"Set default severity\",\"categories\":[{\"filter\":{\"query\":\"@eventName:*\"},\"name\":\"Informational\",\"id\":1}],\"targets\":{\"name\":\"ocsf.severity\",\"id\":\"ocsf.severity_id\"},\"fallback\":{\"values\":{},\"sources\":{}},\"type\":\"schema-category-mapper\"},{\"name\":\"Map userIdentity to ocsf.user.uid\",\"sources\":[\"userIdentity.principalId\",\"responseElements.role.roleId\",\"responseElements.user.userId\"],\"target\":\"ocsf.user.uid\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map userName to ocsf.user.name\",\"sources\":[\"requestParameters.userName\",\"responseElements.role.roleName\",\"requestParameters.roleName\",\"responseElements.user.userName\"],\"target\":\"ocsf.user.name\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map api to ocsf.api\",\"sources\":[\"api\"],\"target\":\"ocsf.api\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map user to ocsf.user\",\"sources\":[\"user\"],\"target\":\"ocsf.user\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map actor to ocsf.actor\",\"sources\":[\"actor\"],\"target\":\"ocsf.actor\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map cloud to ocsf.cloud\",\"sources\":[\"cloud\"],\"target\":\"ocsf.cloud\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map http_request to ocsf.http_request\",\"sources\":[\"http_request\"],\"target\":\"ocsf.http_request\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map metadata to ocsf.metadata\",\"sources\":[\"metadata\"],\"target\":\"ocsf.metadata\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map time to ocsf.time\",\"sources\":[\"time\"],\"target\":\"ocsf.time\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map src_endpoint to ocsf.src_endpoint\",\"sources\":[\"src_endpoint\"],\"target\":\"ocsf.src_endpoint\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map severity to ocsf.severity\",\"sources\":[\"severity\"],\"target\":\"ocsf.severity\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"},{\"name\":\"Map severity_id to ocsf.severity_id\",\"sources\":[\"severity_id\"],\"target\":\"ocsf.severity_id\",\"preserve_source\":false,\"override_on_conflict\":false,\"type\":\"schema-remapper\"}],\"schema\":{\"schema_type\":\"ocsf\",\"version\":\"1.5.0\",\"class_name\":\"Account Change\",\"class_uid\":3001,\"extensions\":[],\"profiles\":[\"cloud\",\"datetime\"]},\"type\":\"schema-processor\"}],\"tags\":[]}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 703,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2025-10-22T19:11:58.776Z",
+        "time": 190
+      },
+      {
+        "_id": "b2bb1a967f84f27ff9e0bbf774a5d296",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 0,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "*/*"
+            }
+          ],
+          "headersSize": 533,
+          "httpVersion": "HTTP/1.1",
+          "method": "DELETE",
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v1/logs/config/pipelines/-qkKiJPYTne-113i8XJ_Nw"
+        },
+        "response": {
+          "bodySize": 3,
+          "content": {
+            "mimeType": "application/json",
+            "size": 3,
+            "text": "{}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 677,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2025-10-22T19:11:58.972Z",
+        "time": 217
+      }
+    ],
+    "pages": [],
+    "version": "1.2"
+  }
+}

cassettes/v1/Logs-Pipelines_3794283453/Create-a-pipeline-with-Schema-Processor-and-preserve_source-true-returns-OK-response_2340010008/frozen.json

@@ -0,0 +1 @@
+"2025-10-22T19:11:59.195Z"