feat: improved handling of data-related events in the job logs

Fetching from s3 to local storage:
- Data: try remote get
- Data: done remote get

Pushing from local storage to plasma:
- Data: try cache put
- Data: done cache put

Fetching from plasma:
- Data: try cache get
- Data: done cache get

Uncompressing, etc.:
- Data: try unpack
- Data: done unpack

This PR also adds a pulse animation to the InProgress events. This has worked with with Kui's Grid view.

Author: starpit

plugins/plugin-codeflare/src/controller/events/torch.ts

@@ -16,7 +16,16 @@
 
 import Event from "./Event"
 
-type EventType = "Data Fetch" | "Data Uncompress" | "Evaluation" | "EvaluationStep" | "Epoch" | "Iteration" | "Marker"
+type EventType =
+  | "Data Fetch from Upstream"
+  | "Data Store in Cache"
+  | "Data Fetch from Cache"
+  | "Data Uncompress"
+  | "Evaluation"
+  | "EvaluationStep"
+  | "Epoch"
+  | "Iteration"
+  | "Marker"
 type Detail = { epoch: number; step: number; nSteps: number; ip: string }
 export type TorchEvent = Event<EventType, Detail>
 
@@ -78,13 +87,36 @@ export function collateEvent(M: TorchEvent[], line: string) {
   }
 
   // Data fetch/uncompress events
-  const hackMatch = line.match(/ip=([\d.]+)\)\s+(\d+-\d+-\d+\s+\d+:\d+:\d+)\s+(getting data|unpacking)/)
+  const hackMatch = line.match(
+    /ip=([\d.]+)\)\s+(\d+-\d+-\d+\s+\d+:\d+:\d+)\s+Data:\s+(try|done)\s+(remote get|cache put|cache get|unpack)/
+  )
   if (hackMatch) {
     const ip = hackMatch[1]
     const timestamp = new Date(hackMatch[2]).getTime()
     const name = `Torch Training on ${ip}`
-    const type: EventType = hackMatch[3] === "unpacking" ? "Data Uncompress" : "Data Fetch"
-    M.push(new TorchEventImpl(name, ip, type, 1, 1, 1, timestamp, "Done", line.slice(line.indexOf(hackMatch[3]))))
+
+    const protoState = hackMatch[3]
+    const state = protoState === "try" ? "InProgress" : "Done"
+    const protoType = hackMatch[4]
+    const type: EventType =
+      protoType === "remote get"
+        ? "Data Fetch from Upstream"
+        : protoType === "cache put"
+        ? "Data Store in Cache"
+        : protoType === "cache get"
+        ? "Data Fetch from Cache"
+        : "Data Uncompress"
+
+    if (state === "InProgress") {
+      M.push(new TorchEventImpl(name, ip, type, 1, 1, 1, timestamp, state, line.slice(line.indexOf(protoState))))
+    } else {
+      const prev = findPrevious(M, ip, type, "InProgress")
+      if (prev) {
+        prev.state = state
+      } else {
+        console.error("Missing Data: begin event for this Data: end event", line)
+      }
+    }
   }
 
   // Torch Events

plugins/plugin-codeflare/web/scss/components/Dashboard/Grid.scss

@@ -99,6 +99,8 @@ $fullWidth: 1em; /* $large * ($unit + $rgap) - $rgap */
     }
     @include State("InProgress") {
       @include Color(var(--color-base0A), !important);
+      filter: brightness(1.1);
+      animation: var(--animation-infinite-repeating-pulse);
     }
     @include State("Pending") {
       @include Color(var(--color-base04), !important);
@@ -146,7 +148,18 @@ $fullWidth: 1em; /* $large * ($unit + $rgap) - $rgap */
     @include StepUI;
   }
 
-  @include CFCell(DataFetch) {
+  @include CFCell(DataFetchfromUpstream) {
+    grid-column: span $small;
+    @include Color(var(--color-base0E));
+  }
+
+  @include CFCell(DataStoreinCache) {
+    grid-column: span $small;
+    @include StepUI;
+    @include Color(var(--color-base0E));
+  }
+
+  @include CFCell(DataFetchfromCache) {
     grid-column: span $small;
     @include Color(var(--color-base0E));
   }

tests/plugin-codeflare/dashboard/inputs/2/README.md

@@ -12,7 +12,7 @@
   "Choose the bucket that contains your model and glue data.madwizard/apriori/platform": "Darwin",
   "expand(kubectl config get-contexts -o name, Kubernetes contexts)": "default/api-codeflare-train-v11-codeflare-openshift-com:6443/kube:admin",
   "expand([ -z ${KUBE_CONTEXT} ] && exit 1 || kubectl --context ${KUBE_CONTEXT} get ns -o name | grep -Ev 'openshift|kube-' | sed 's#namespace/##', Kubernetes namespaces)####Create a namespace": "nvidia-gpu-operator",
-  "Number of CPUs####Number of GPUs####Minimum Workers####Maximum Workers####Worker Memory####Head Memory": "{\"Number of CPUs\":\"1\",\"Number of GPUs\":\"1\",\"Minimum Workers\":\"5\",\"Maximum Workers\":\"5\",\"Worker Memory\":\"32Gi\",\"Head Memory\":\"32Gi\"}",
+  "Number of CPUs####Number of GPUs####Minimum Workers####Maximum Workers####Worker Memory####Head Memory": "{\"Number of CPUs\":\"1\",\"Number of GPUs\":\"1\",\"Minimum Workers\":\"4\",\"Maximum Workers\":\"4\",\"Worker Memory\":\"32Gi\",\"Head Memory\":\"32Gi\"}",
   "Choose the bucket that contains your model and glue data.expand([ -n \"$MC_CONFIG_DIR\" ] && mc -q --config-dir ${MC_CONFIG_DIR} ls s3 | awk '{print substr($NF, 1, length($NF) - 1)}', S3 Buckets)####separator####📁 Create a new bucket": "browsey",
   "Choose your Model File.expand([ -n \"$MC_CONFIG_DIR\" ] && [ -n \"$S3_FILEPATH\" ] && [ -n \"$S3_FILEPATH${S3_BUCKET_SUFFIX}\" ] && mc -q --config-dir ${MC_CONFIG_DIR} ls \"s3/$S3_FILEPATH${S3_BUCKET_SUFFIX}\" | awk '{print $NF}', S3 Objects)": "roberta-base",
   "Choose your Glue Data File.expand([ -n \"$MC_CONFIG_DIR\" ] && [ -n \"$S3_FILEPATH\" ] && [ -n \"$S3_FILEPATH${S3_BUCKET_SUFFIX}\" ] && mc -q --config-dir ${MC_CONFIG_DIR} ls \"s3/$S3_FILEPATH${S3_BUCKET_SUFFIX}\" | awk '{print $NF}', S3 Objects)": "glue_data",

tests/plugin-codeflare/dashboard/inputs/2/choices.json

@@ -1,13 +0,0 @@
-LAST SEEN   TYPE     REASON      OBJECT                              MESSAGE
-0s          Normal   Scheduled   pod/mycluster-ray-head-type-krlr4   Successfully assigned nvidia-gpu-operator/mycluster-ray-head-type-krlr4 to ip-10-0-128-169.ec2.internal
-0s          Normal   AddedInterface   pod/mycluster-ray-head-type-krlr4   Add eth0 [10.128.44.144/23] from openshift-sdn
-0s          Normal   Pulling          pod/mycluster-ray-head-type-krlr4   Pulling image "rayproject/ray-ml:1.13.0-py37-gpu"
-0s          Normal   Pulled           pod/mycluster-ray-head-type-krlr4   Successfully pulled image "rayproject/ray-ml:1.13.0-py37-gpu" in 6m48.700535275s
-0s          Normal   Created          pod/mycluster-ray-head-type-krlr4   Created container ray-node
-0s          Normal   Started          pod/mycluster-ray-head-type-krlr4   Started container ray-node
-0s          Normal   Scheduled        pod/mycluster-ray-worker-type-6r7hp   Successfully assigned nvidia-gpu-operator/mycluster-ray-worker-type-6r7hp to ip-10-0-133-106.ec2.internal
-0s          Normal   AddedInterface   pod/mycluster-ray-worker-type-6r7hp   Add eth0 [10.131.42.42/23] from openshift-sdn
-0s          Normal   Pulling          pod/mycluster-ray-worker-type-6r7hp   Pulling image "rayproject/ray-ml:1.13.0-py37-gpu"
-0s          Normal   Pulled           pod/mycluster-ray-worker-type-6r7hp   Successfully pulled image "rayproject/ray-ml:1.13.0-py37-gpu" in 6m14.380152399s
-0s          Normal   Created          pod/mycluster-ray-worker-type-6r7hp   Created container ray-node
-0s          Normal   Started          pod/mycluster-ray-worker-type-6r7hp   Started container ray-node

tests/plugin-codeflare/dashboard/inputs/2/events/kubernetes.txt

@@ -1 +1 @@
-11f4cebe-c012-4467-b5d6-b3cf1fd69269
+f6c365c4-5fec-41b1-a811-3594c1c0f91d