Hitl dashboard - extend API to support different pipeline, added traceback list view component

droidlet/tools/hitl/dashboard_app/README.MD

@@ -1,9 +1,17 @@
 # Dashboard App for HITL
-Updated July 22 by Chuxi.
+Updated July 29 by Chuxi.
 
 This is a Dashboard app prototype for the HITL system.
 
 ## Update Note
+- July 29:
+    - Extend components / backend APIs to be able to display different pipeline.
+    - Added traceback list component.
+    - Demo:
+        - View different pipeline
+            - ![demo_different_pipeline_extension](https://user-images.githubusercontent.com/51009396/181843461-f27f9fe8-b575-440c-b14d-12f57f1807b2.gif)
+        - View traceback list
+            - ![demo_traceback_list](https://user-images.githubusercontent.com/51009396/181843326-3c367e8d-999f-497b-8011-956d885db886.gif)
 - July 22:
     - Updated frontend component to show the model line graph of loss and accuracy vs epoch.
     - Demo:
@@ -37,7 +45,8 @@ This is a Dashboard app prototype for the HITL system.
     - Added view session Log feature.
     - Demo:
         - ![demo_view_sesion_log](https://user-images.githubusercontent.com/51009396/178044751-8d099829-cc2f-4353-8bfd-e81e9a23b63e.gif)
-- July 6: ![demo_change_dataset_ver](https://user-images.githubusercontent.com/51009396/178807740-b90ada63-573d-47ba-845e-8ab84033d184.gif)
+
+- July 6: 
     - Added detail page to show detail infomation of a pipeline run (specified by the batch_id)
     - Added a get all sessions related to a batch id backend API. 
     - Demo:
@@ -57,6 +66,12 @@ This is a Dashboard app prototype for the HITL system.
 - Frontend: a react based frontend - including navigation and NLU pipeline overview page. Default run on localhost:3000.
 Please note this dashboard app is currently only in development build and is not optimized.
 
+## Dependency
+- Dependency required for backend server (in addition to the droidelet setting, please make sure you have the updated version as below):
+Flask==2.1.2
+Flask-SocketIO==5.2.0
+- Dependency for frontend is specified in the `fairo/droidlet/tools/hitl/dashboard_app/dashboard_frontend/package.json` file.
+
 ## How to run
 - Set up environment variables
 ```
@@ -80,10 +95,10 @@ npm start
 
 ## Supported APIs
 APIs are based on socket event, the following APIs are supported currently: 
-- get_job_list:
-    - get a list of jobs stored on AWS that has been run in the past. 
-    - input: no parameter input.
-    - output: a list of batch ids of the jobs.
+- get_run_list:
+    - get a list of jobs stored on AWS that has been run in the past.
+    - input: pipeline name in lowercase (nlu, tao, vision etc.)
+    - output: a list of batch ids of the jobs of the pipeline.
 - get_traceback_by_id:
     - get traceback record by id.
     - input: a batch id.

droidlet/tools/hitl/dashboard_app/backend/dashboard_aws_helper.py

@@ -6,20 +6,20 @@
 and preparing proper response for APIs the server provides.
 """
 
+import ast
 import json
 import tarfile
 import boto3
 import botocore
 import os
 import re
+import pandas as pd
 
 from droidlet.tools.hitl.dashboard_app.backend.dashboard_model_utils import load_model
 from droidlet.tools.hitl.utils.read_model_log import read_model_log_to_list
 
 
-PIPELINE_DATASET_MAPPING = {
-    "NLU": "nsp_data",
-}
+PIPELINE_DATASET_MAPPING = {"NLU": "nsp_data", "TAO": "nsp_data"}
 
 S3_BUCKET_NAME = "droidlet-hitl"
 S3_ROOT = "s3://droidlet-hitl"
@@ -70,22 +70,20 @@ def _read_file(fname: str):
     return content
 
 
-def get_job_list():
+def get_run_list(pipeline: str):
     """
-    helper method for preparing get_job_list api's response
+    helper method for preparing get_run_list api's response
     """
-    job_list = []
-    # list object from s3 bucket
-    res = s3.meta.client.get_paginator("list_objects").paginate(
-        Bucket=S3_BUCKET_NAME, Delimiter="/"
-    )
-    # pattern of YYYYMMDDHHMMSS (batch id pattern)
-    pattern = r"([0-9]{4})(0[1-9]|1[0-2])(0[1-9]|[1-2][0-9]|3[0-1])(2[0-3]|[01][0-9])([0-5][0-9])([0-5][0-9])"
+    # download run list file
+    local_fname = _download_file(f"{pipeline}_run_list")
+    if local_fname is None:
+        return f"cannot find run list for pipeline {pipeline}", 404
 
-    for prefix in res.search("CommonPrefixes"):
-        if re.match(pattern, prefix.get("Prefix")):
-            job_list.append(int(prefix.get("Prefix")[:-1]))
-    return job_list
+    run_list = _read_file(local_fname).split("\n")
+    run_list = list(filter(lambda line: len(line), run_list))  # filter out empty strings if there
+    run_list = [int(bid) for bid in run_list]
+
+    return run_list, None
 
 
 def get_traceback_by_id(batch_id: int):
@@ -95,7 +93,26 @@ def get_traceback_by_id(batch_id: int):
     local_fname = _download_file(f"{batch_id}/log_traceback.csv")
     if local_fname is None:
         return f"cannot find traceback with id {batch_id}", 404
-    return _read_file(local_fname), None
+    traceback_df = pd.read_csv(local_fname)
+    traceback_df.chat_content = traceback_df.chat_content.map(
+        lambda x: ast.literal_eval(x)
+    )  # parsse array
+
+    def _get_freq(x):
+        freq_dict = {}
+        for o in x:
+            if o not in freq_dict:
+                freq_dict[o] = 0
+            freq_dict[o] += 1
+        return freq_dict
+
+    traceback_df.chat_content = traceback_df.chat_content.map(
+        lambda ls: [o for o in ls if len(o)]
+    )  # filter out empty ones
+    traceback_df.chat_content = traceback_df.chat_content.map(
+        lambda x: _get_freq(x)
+    )  # get frequency
+    return traceback_df.to_json(orient="records"), None
 
 
 def get_run_info_by_id(batch_id: int):

droidlet/tools/hitl/dashboard_app/backend/dashboard_server.py

@@ -15,8 +15,8 @@
     get_dataset_version_list_by_pipeline,
     get_interaction_session_log_by_id,
     get_interaction_sessions_by_id,
-    get_job_list,
     get_model_by_id,
+    get_run_list,
     get_run_info_by_id,
     get_traceback_by_id,
 )
@@ -38,7 +38,7 @@ class DASHBOARD_EVENT(Enum):
     server supported event types, i.e. API types
     """
 
-    GET_RUNS = "get_job_list"
+    GET_RUNS = "get_run_list"
     GET_TRACEBACK = "get_traceback_by_id"
     GET_RUN_INFO = "get_run_info_by_id"
     GET_INTERACTION_SESSIONS = "get_interaction_sessions_by_id"
@@ -57,16 +57,20 @@ class DASHBOARD_EVENT(Enum):
 
 
 @socketio.on(DASHBOARD_EVENT.GET_RUNS.value)
-def get_jobs():
+def get_runs(pipeline: str):
     """
     get a list of jobs stored on AWS that has been run in the past.
-    - input: no parameter input.
+    - input: pipeline name in lowercase (nlu, tao, vision etc.)
     - output: a list of batch ids of the jobs.
     """
-    print(f"Request received: {DASHBOARD_EVENT.GET_RUNS.value}")
-    job_list = get_job_list()
-    print(f"Job list reterived from aws, sending job list (length:{len(job_list)}) to client")
-    emit(DASHBOARD_EVENT.GET_RUNS.value, job_list)
+    print(f"Request received: {DASHBOARD_EVENT.GET_RUNS.value}, pipeline = {pipeline}")
+    job_list, error_code = get_run_list(pipeline)
+    if error_code:
+        print(job_list)
+        emit(DASHBOARD_EVENT.GET_RUNS.value, error_code)
+    else:
+        print(f"Job list reterived from aws, sending job list (length:{len(job_list)}) to client")
+        emit(DASHBOARD_EVENT.GET_RUNS.value, job_list)
 
 
 @socketio.on(DASHBOARD_EVENT.GET_TRACEBACK.value)

droidlet/tools/hitl/dashboard_app/backend/tests/dashboard_aws_helper_test.py

@@ -4,7 +4,7 @@
 import json
 
 from droidlet.tools.hitl.dashboard_app.backend.dashboard_aws_helper import (
-    get_job_list,
+    get_run_list,
     get_run_info_by_id,
     get_traceback_by_id,
 )
@@ -41,8 +41,8 @@ def setUp(self):
         s3.Object(S3_BUCKET_NAME, self._info_fname).put(Body=json_content)
         s3.Object(S3_BUCKET_NAME, self._traceback_fname).put(Body="1, 2 \n1, 2")
 
-    def test_get_job_list(self):
-        res = get_job_list()
+    def test_get_run_list(self):
+        res = get_run_list()
         self.assertGreater(len(res), 0)
 
     def test_get_traceback_by_id_valid(self):
@@ -55,12 +55,12 @@ def test_get_traceback_by_id_invalid(self):
         self.assertEqual(res, f"cannot find traceback with id {INVALID_ID}")
 
     def test_get_run_info_by_id_valid(self):
-        res = get_run_info_by_id(VALID_ID)
+        res, _ = get_run_info_by_id(VALID_ID)
         self.assertIsNotNone(res)
         self.assertNotEqual(res, f"cannot find run info with id {VALID_ID}")
 
     def test_get_run_info_by_id_inalid(self):
-        res = get_run_info_by_id(INVALID_ID)
+        res, _ = get_run_info_by_id(INVALID_ID)
         self.assertEqual(res, f"cannot find run info with id {INVALID_ID}")
 
     def tearDown(self):
@@ -70,7 +70,6 @@ def tearDown(self):
         # remove from local temp directory as well
         local_info_fname = os.path.join(HITL_TMP_DIR, self._info_fname)
         local_traceback_fname = os.path.join(HITL_TMP_DIR, self._traceback_fname)
-
         if os.path.exists(local_info_fname):
             os.remove(local_info_fname)
         if os.path.exists(local_traceback_fname):

droidlet/tools/hitl/dashboard_app/dashboard_frontend/package.json

@@ -12,6 +12,7 @@
     "@testing-library/user-event": "^13.5.0",
     "antd": "^4.21.3",
     "moment": "^2.29.3",
+    "papaparse": "^5.3.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-router-dom": "^6.3.0",

droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/App.js

@@ -13,8 +13,8 @@ import { SocketContext, socket } from './context/socket';
 import Main from './component/main';
 import { Routes, Route, BrowserRouter } from "react-router-dom";
 import NavBar from './component/navbar';
-import { SUBPATHS } from './constants/subpaths';
-import { BackTop, Layout, Typography } from 'antd';
+import { PIPELINES, SUBPATHS } from './constants/subpaths';
+import { BackTop, Layout } from 'antd';
 import PipelinePanel from './component/pipeline/panel';
 import DetailPage from './component/pipeline/detail/detailPage';
 import JobInfoCard from './component/pipeline/detail/job/jobInfoCard';
@@ -36,15 +36,17 @@ function App() {
             {/* Routes for different pipeline */}
             <Layout>
               <Routes>
-                <Route path={SUBPATHS.HOME.key} element={<div><Typography.Title>Welcome to Droidlet HiTL Dashboard</Typography.Title></div>} />
-                <Route path={SUBPATHS.NLU.key} element={<PipelinePanel pipelineType={SUBPATHS.NLU} />}>
-                  <Route path=":batch_id" element={<DetailPage pipelineType={SUBPATHS.NLU.label} />}>
-                    <Route path=":job" element={<JobInfoCard />} />
+                <Route path={SUBPATHS.HOME.key} element={<Main />} />
+                {PIPELINES.map((pipeline) =>
+                  <Route path={pipeline.key} element={<PipelinePanel pipelineType={pipeline} />}>
+                    <Route path=":batch_id" element={<DetailPage pipelineType={pipeline.label} />}>
+                      <Route path=":job" element={<JobInfoCard />} />
+                    </Route>
                   </Route>
-                </Route>
-                <Route path="dataset" element={<DatasetPageContainer />}> 
+                )}
+                <Route path="dataset" element={<DatasetPageContainer />}>
                   <Route path=":pipeline" element={<DatasetDetailPage />} />
-                </Route>
+                </Route>               
                 <Route path="*" element={<NotFoundPage />} />
               </Routes>
               <BackTop />

droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/main.js

@@ -20,7 +20,7 @@ const Main = () => {
     }, []);
 
     const getJobList = () => {
-        socket.emit("get_job_list");
+        socket.emit("get_run_list");
         setLoading(true);
     } 
 
@@ -43,7 +43,7 @@ const Main = () => {
     }
 
     useEffect(() => {
-        socket.on("get_job_list", (data) => handleReceivedJobList(data));
+        socket.on("get_run_list", (data) => handleReceivedJobList(data));
         socket.on("get_traceback_by_id", (data) => handleReciveTraceback(data));
         socket.on("get_run_info_by_id", (data) => handleReciveInfo(data));
     }, [socket, handleReceivedJobList, handleReciveTraceback, handleReciveInfo]);

droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/asset/modelCard.js

@@ -130,10 +130,6 @@ const ModelCard = (props) => {
         setAttrModalOpen(true);
     }
 
-    const handleViewModelLossAndAcc = (lossAccType) => {
-        alert(lossAccData[lossAccType].map((o) => (`loss: ${o.loss},  acc: ${o.acc}`)));
-    }
-
     return (
         <div style={{ width: '70%' }}>
             <Card title="Model" loading={loadingKeys || loadingArgs || loadingLossAcc}>

droidlet/tools/hitl/dashboard_app/dashboard_frontend/src/component/pipeline/detail/detailPage.js

@@ -24,13 +24,15 @@ import { SocketContext } from '../../../context/socket';
 import { JOB_TYPES } from "../../../constants/runContants";
 import ModelCard from "./asset/modelCard";
 import DatasetCard from "./asset/datasetCard";
+import TracebackList from "./tao/tracebackList";
 
 const { Title } = Typography;
 const { Panel } = Collapse;
 
 const DetailPage = (props) => {
     const socket = useContext(SocketContext);
     const pipelineType = props.pipelineType;
+
     const batch_id = useParams().batch_id;
     const [runInfo, setRunInfo] = useState(null);
     const [jobs, setJobs] = useState(null);
@@ -102,13 +104,14 @@ const DetailPage = (props) => {
                     >
                         <MetaInfoDescription metaInfo={runInfo} />
                         <Divider />
-                        <div style={{ 'display': 'flex'}}>
+                        <div style={{ 'display': 'flex' }}>
                             <DatasetCard batchId={batch_id} pipelineType={pipelineType} />
                             <ModelCard batchId={batch_id} pipelineType={pipelineType} />
                         </div>
                     </Panel>
                 </Collapse>
 
+
                 <Divider />
                 <div style={{ 'display': 'flex', "padding": "0 32px 0 32px" }}>
                     <div style={{ 'width': '160px' }}>
@@ -124,9 +127,15 @@ const DetailPage = (props) => {
                     </div>
                     <Outlet context={{ metaInfo: runInfo }} />
                 </div>
+                {
+                    pipelineType === "TAO" && <>
+                        <Divider />
+                        <TracebackList batchId={batch_id} />
+                    </>
+                }
                 <div style={{ "paddingTop": "18px" }}>
                     <Button type="primary">
-                        <Link to="../" state={{ label: TAB_ITEMS.RUNS.label, key: TAB_ITEMS.RUNS.key }}>
+                        <Link to="../." state={{ label: TAB_ITEMS.RUNS.label, key: TAB_ITEMS.RUNS.key }}>
                             Back to View All
                         </Link>
                     </Button>