Logs, Telemetry Events

Telemetry is a specification to instrument all the key events. Using this specification reference applications & services will generate telemetry events. For more info refer Sunbird-Telemetry documentation here. Also to know how telemetry is processed refer to Telemetry Processing Documentation in https://lern.sunbird.org/learn/telemetry-processing.

Logs also use telemetry specification and request tracing/ tracking is accomplished by passing the request context into the logs and telemetry. This helps in troubleshooting issues. Request context has below details:

uid //user id from user authentication token in header (x-authenticated-user-token)
did //device id from request header (x-device-id)
sid //session id from request header (x-session-id)
reqId //request id from request header (x-request-id)

This request id also set to API response as msgid. This helps in tracking once request that invokes multiple APIs and flows.

Configuration

Telemetry has request context information and it is set during the telemetry logging.

Below are the properties set by the application to add the producer information into the telemetry data:

#Telemetry producer related info

telemetry_pdata_id=local.sunbird.learning.service
telemetry_pdata_pid=learning-service
telemetry_pdata_ver=5.1.0

Telemetry is logged and also send to SB-Obsrv data-pipeline using logback in notification service. logback.xml configuration has a kafka appender which will send the telemetry generated by the code to a kafka topic.

 <topic>${ENV_NAME}.telemetry.raw</topic>

Logback.xml configuration:

 <appender name="kafka-appender" class="com.github.danielwegener.logback.kafka.KafkaAppender">
    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
      <pattern>%msg</pattern>
    </encoder>

    <topic>${ENV_NAME}.telemetry.raw</topic>
    <!-- ensure that every message sent by the executing host is partitioned to the same partition strategy -->
    <keyingStrategy class="com.github.danielwegener.logback.kafka.keying.NoKeyKeyingStrategy" />
    <!-- block the logging application thread if the kafka appender cannot keep up with sending the log messages -->
    <deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" />

    <!-- each <producerConfig> translates to regular kafka-client config (format: key=value) -->
    <!-- producer configs are documented here: https://kafka.apache.org/documentation.html#newproducerconfigs -->
    <!-- bootstrap.servers is the only mandatory producerConfig -->
    <producerConfig>bootstrap.servers=${SUNBIRD_KAFKA_URL}</producerConfig>
    <!-- don't wait for a broker to ack the reception of a batch.  -->
    <producerConfig>acks=0</producerConfig>
    <!-- wait up to 1000ms and collect log messages before sending them as a batch -->
    <producerConfig>linger.ms=15000</producerConfig>
    <!-- even if the producer buffer runs full, do not block the application but start to drop messages -->
    <producerConfig>max.block.ms=0</producerConfig>
    <!-- define a client-id that you use to identify yourself against the kafka broker -->
    <producerConfig>client.id=${HOSTNAME}-${CONTEXT_NAME}-logback-relaxed</producerConfig>

    <!-- there is no fallback <appender-ref>. If this appender cannot deliver, it will drop its messages. -->

  </appender>

  <logger name="TelemetryEventLogger" level="INFO">
    <appender-ref ref="kafka-appender" />
  </logger>

List of Events

Audit Event

{
   "eid":"AUDIT",
   "ets":1649247985143,
   "ver":"3.0",
   "mid":"d808691c-e253-43a6-a8a0-aa03bc67b6ce",
   "actor":{
      "id":"50792198-c6d7-4964-8d4c-da6891ceed0a",
      "type":"User"
   },
   "context":{
      "channel":"0126796199493140480",
      "pdata":{
         "id":"staging.sunbird.learning.service",
         "pid":"learner-service",
         "ver":"4.7.0"
      },
      "env":"User",
      "cdata":[
         {
            "id":"d808691c-e253-43a6-a8a0-aa03bc67b6ce",
            "type":"Request"
         }
      ],
      "rollup":{
         "l1":"0126796199493140480"
      }
   },
   "object":{
      "type":"User"
   },
   "edata":{
      "state":"Update",
      "props":[
         "identifier",
         "tncAcceptedOn",
         "id",
         "tncAcceptedVersion"
      ]
   }
}

Error Event

{
   "eid":"ERROR",
   "ets":1649248112302,
   "ver":"3.0",
   "mid":"31eab671-1395-4135-8723-15ffa5d349cb",
   "actor":{
      "id":"internal",
      "type":"Consumer"
   },
   "context":{
      "channel":"0126796199493140480",
      "pdata":{
         "id":"staging.sunbird.learning.service",
         "pid":"learner-service",
         "ver":"4.7.0"
      },
      "env":"Organisation",
      "cdata":[
         {
            "id":"31eab671-1395-4135-8723-15ffa5d349cb",
            "type":"Request"
         }
      ],
      "rollup":{
         
      }
   },
   "edata":{
      "err":"UOS_ORGSER0017",
      "stacktrace":"Invalid value null for parameter hashTagId. Please provide a valid value. org.sunbird.validator.BaseRequestValidator.lambda$validateListValues$6(BaseRequestValidator.java:291)java.base/java.util.ArrayList.forEach(ArrayList.java:1541)org.sunbird.validator.BaseRequestValidator.validateListValues",
      "errtype":"api_access",
      "requestid":"31eab671-1395-4135-8723-15ffa5d349cb"
   }
}

Log Event

{
   "eid":"LOG",   
   "actor":{
      "id":"internal",
      "type":"Consumer"
   },
   "edata":{
      "level":"info",
      "type":"Api_access",
      "message":"",
      "params":[
         {
            "method":"POST"
         },
         {
            "url":"/v1/org/search"
         },
         {
            "duration":0
         },
         {
            "status":"OK"
         }
      ]
   },
   "ver":"3.0",
   "syncts":1649247488365,
   "@timestamp":"2022-04-06T12:18:08.365Z",
   "ets":1649247476273,
   "context":{
      "channel":"0126796199493140480",
      "pdata":{
         "id":"staging.sunbird.learning.service",
         "pid":"learner-service",
         "ver":"4.7.0"
      },
      "env":"Organisation",
      "cdata":[
         {
            "id":"9c158007-345b-44d7-a128-6c36f7a42cfb",
            "type":"Request"
         }
      ],
      "rollup":{
         
      }
   },
   "flags":{
      "pp_validation_processed":true
   },
   "mid":"9c158007-345b-44d7-a128-6c36f7a42cfb",
   "type":"events"
}

Sample Telemetry Data

Audit Event Data

{
  "eid": "AUDIT",
  "ets": 1566563420660,
  "ver": "3.0",
  "mid": "1566563420660.f46c14d1-8c9a-417f-82b8-f125ba32b828",
  "actor": {
    "id": "internal",
    "type": "Consumer"
  },
  "context": {
    "channel": "0128220189818880000",
    "pdata": {
      "id": "staging.diksha.learning.service", // Producer ID.
      "ver": "1.15", // Version of the App
      "pid": "learner-service"// Optional. In case the component is distributed, then which instance of that component
    },
    "env": "User",
    "cdata": [
      {
        "id": "4e2afe8e-fb44-4788-9f49-0ef61c5c808b",
        "type": "User"
      },
      {
        "id": "11166",
        "type": "Certificate"
      }
    ],
    "rollup": {
      "l1": "0128220189818880000"
    }
  },
  "object": {
    "id": "11166",
    "type": "Certificate"
  },
  "edata": {
    "state": "Create", // defines the state i.e: Mergecert, Mergeuser
    "props": [
      "certId", // certificate Id
      "userId"  // user Id
    ]
  }
}