New filters by Job Type and Rehearse

backend/app/api/v1/commons/ocp.py

@@ -1,4 +1,4 @@
-from datetime import datetime, date
+from datetime import date
 import pandas as pd
 from app.services.search import ElasticService
 
@@ -32,9 +32,31 @@ async def getData(start_datetime: date, end_datetime: date):
     if len(jobs) == 0:
         return jobs
 
+    jobs["benchmark"] = jobs.apply(updateBenchmark, axis=1)
+    jobs["jobType"] = jobs.apply(jobType, axis=1)
+    jobs["isRehearse"] = jobs.apply(isRehearse, axis=1)
+
     cleanJobs = jobs[jobs['platform'] != ""]
 
     jbs = cleanJobs
     jbs['shortVersion'] = jbs['ocpVersion'].str.slice(0, 4)
 
     return jbs
+
+
+def updateBenchmark(row):
+    if row["upstreamJob"].__contains__("upgrade"):
+        return "upgrade-" + row["benchmark"]
+    return row["benchmark"]
+
+
+def jobType(row):
+    if row["upstreamJob"].__contains__("periodic"):
+        return "periodic"
+    return "pull request"
+
+
+def isRehearse(row):
+    if row["upstreamJob"].__contains__("rehearse"):
+        return "True"
+    return "False"

backend/app/api/v1/endpoints/cpt/cptJobs.py

@@ -24,8 +24,8 @@
                 422: response_422(),
             },)
 async def jobs(start_date: date = Query(None, description="Start date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-10"]),
-                end_date: date = Query(None, description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"]),
-                pretty: bool = Query(False, description="Output contet in pretty format.")):
+               end_date: date = Query(None, description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"]),
+               pretty: bool = Query(False, description="Output contet in pretty format.")):
     if start_date is None:
         start_date = datetime.utcnow().date()
         start_date = start_date - timedelta(days=5)

backend/app/api/v1/endpoints/cpt/maps/ocp.py

@@ -1,6 +1,7 @@
 from ....commons.ocp import getData
 from datetime import date
 
+
 ################################################################
 # This will return a DataFrame from OCP required by the CPT
 #  endpoint, it contians the following columns:
@@ -27,11 +28,13 @@ async def ocpMapper(start_datetime: date, end_datetime: date):
 
 def dropColumns(df):
     df = df.drop(columns=['shortVersion', 'benchmark', 'platform', 'clusterType', 'masterNodesCount',
-                            'workerNodesCount', 'infraNodesCount', 'masterNodesType', 'workerNodesType',
-                            'infraNodesType', 'totalNodesCount', 'clusterName', 'ocpVersion', 'networkType',
-                            'buildTag', 'upstreamJob', 'upstreamJobBuild', 'executionDate', 'jobDuration', 'timestamp'])
+                          'workerNodesCount', 'infraNodesCount', 'masterNodesType', 'workerNodesType',
+                          'infraNodesType', 'totalNodesCount', 'clusterName', 'ocpVersion', 'networkType',
+                          'buildTag', 'upstreamJob', 'upstreamJobBuild', 'executionDate', 'jobDuration',
+                          'timestamp', 'jobType', 'isRehearse'])
     return df
 
+
 def getReleaseStream(row):
     if row["releaseStream"].__contains__("fast"):
         return "Fast"

backend/app/api/v1/endpoints/ocp/results.py

@@ -1,20 +1,16 @@
-import asyncio
-from typing import Dict, Iterable
-
-import httpx
-from fastapi import APIRouter, Request
-from fastapi.param_functions import Query, Path
+from fastapi import APIRouter
+from fastapi.param_functions import Path
 
 from app.services.search import ElasticService
 
 router = APIRouter()
 
+
 @router.get('/api/ocp/v1/jobs/{ci}/{job_id}',
             summary="Returns the details of a specified Job.")
 async def results_for_job(
-                ci: str = Path(..., description="Name of the CI system tha tthe job belongs to.", examples=["PROW", "JENKINS"]),
-                job_id: str = Path(..., description="Unique identifier of the Jon, normally the UUID.", examples=["8b671d0b-8638-4423-b453-cc54b1caf529"]),
-                ):
+        ci: str = Path(..., description="Name of the CI system tha tthe job belongs to.", examples=["PROW", "JENKINS"]),
+        job_id: str = Path(..., description="Unique identifier of the Jon, normally the UUID.", examples=["8b671d0b-8638-4423-b453-cc54b1caf529"])):
     query = {
         "query": {
             "query_string": {
@@ -24,43 +20,8 @@ async def results_for_job(
         }
     }
 
-    airflow = False
-    if ci == "AIRFLOW":
-        airflow = True
-
     es = ElasticService(configpath="ocp.elasticsearch")
     response = await es.post(query)
     await es.close()
     tasks = [item['_source'] for item in response["hits"]["hits"]]
-    if ci == "AIRFLOW":
-        tasks_states = await async_tasks_states(tasks)
-
-        for task in tasks:
-            task['job_status'] = tasks_states[task['build_tag']]
-
     return tasks
-
-
-async def async_tasks_states(tasks: Iterable) -> Dict[str, str]:
-    async with airflow_service.httpx_client() as session:
-        tasks_states = await asyncio.gather(
-            *[call_url(session, task) for task in tasks]
-        )
-    return {
-        task: state for task_state in tasks_states
-        for task, state in task_state.items()
-    }
-
-
-async def call_url(session: httpx.AsyncClient, task) -> Dict[str, str]:
-    path = (
-        f"{airflow_service.base_url}/api/v1"
-        f"/dags/{task['upstream_job']}"
-        f"/dagRuns/{task['upstream_job_build']}"
-        f"/taskInstances/{task['build_tag']}"
-    )
-    resp = await session.get(path)
-    resp.raise_for_status()
-    return {
-        task['build_tag']: resp.json()['state']
-    }

frontend/src/components/OCP/OCPHome.js

@@ -32,6 +32,8 @@ export function OCPHome() {
     const [workerCount, setWorkerCount] = useState(searchParams.get("workerCount") || ocpJobs.selectedWorkerCount)
     const [networkType, setNetworkType] = useState(searchParams.get("networkType") || ocpJobs.selectedNetworkType)
     const [version, setVersion] = useState(searchParams.get("version") || ocpJobs.selectedVersion)
+    const [jobType, setJobType] = useState(searchParams.get("jobType") || ocpJobs.selectedJobType)
+    const [isRehearse, setRehearse] = useState(searchParams.get("isRehearse") || ocpJobs.selectedRehearse)
     const [startDate, setStartDate] = useState(searchParams.get("startDate")  || ocpJobs.startDate) || ""
     const [endDate, setEndDate] = useState(searchParams.get("endDate") || ocpJobs.endDate) || ""
 
@@ -43,6 +45,8 @@ export function OCPHome() {
         {name: "Workers Count", onChange: setWorkerCount, value: workerCount, options:ocpJobs.workers },
         {name: "Network Type", onChange: setNetworkType, value: networkType, options:ocpJobs.networkTypes },
         {name: "Versions", onChange: setVersion, value: version, options: ocpJobs.versions},
+        {name: "Job Type", onChange: setJobType, value: jobType, options: ocpJobs.jobTypes},
+        {name: "Rehearse", onChange: setRehearse, value: isRehearse, options: ocpJobs.rehearses},
     ]
 
     useEffect(() => {
@@ -53,15 +57,17 @@ export function OCPHome() {
         if(version !== '') buildParams += `&version=${version}`
         if(workerCount !== '') buildParams += `&workerCount=${workerCount}`
         if(networkType !== '') buildParams += `&networkType=${networkType}`
+        if(jobType !== '') buildParams += `&jobType=${jobType}`
+        if(isRehearse !== '') buildParams += `&isRehearse=${isRehearse}`
         if(startDate !== '') buildParams += `&startDate=${startDate}`
         if(endDate !== '') buildParams += `&endDate=${endDate}`
         history.push(`/ocp?${buildParams.substring(1)}`, { replace: true });
 
     }, [history, ciSystem, platform, benchmark, version, workerCount, networkType, startDate, endDate])
 
     useEffect( ()=>{
-        dispatch(updateOCPDataFilter({ciSystem, platform, benchmark, version, workerCount, networkType}))
-    }, [ ciSystem, platform, benchmark, version, workerCount, networkType, dispatch ])
+        dispatch(updateOCPDataFilter({ciSystem, platform, benchmark, version, workerCount, networkType, jobType, isRehearse}))
+    }, [ ciSystem, platform, benchmark, version, workerCount, networkType, , jobType, isRehearse, dispatch ])
 
     useEffect(() => {
         if(startDate || endDate){

frontend/src/store/Actions/ActionCreator.js

@@ -57,10 +57,12 @@ export const fetchOCPJobsData = (startDate = '', endDate='') => async dispatch =
                 const workers = GetWorkers(results)
                 const networkTypes = GetNetworkTypes(results)
                 const ciSystems = GetCiSystems(results)
+                const jobTypes = GetJobType(results)
+                const rehearses = ["TRUE", "FALSE"]
                 const updatedTime = new Date().toLocaleString().replace(', ', ' ').toString();
                 await dispatch(getOCPJobsData({
                     data: results, benchmarks, versions, waitForUpdate: false, platforms, workers, networkTypes,
-                    updatedTime, ciSystems, startDate: api_data.startDate, endDate: api_data.endDate
+                    updatedTime, ciSystems, jobTypes, rehearses, startDate: api_data.startDate, endDate: api_data.endDate
                 }))
                 await dispatch(updateOCPMetaData({data: results}))
             }
@@ -167,3 +169,7 @@ const GetTestNames = (api_data) => {
         else return item.testName.toLowerCase().trim()
     }))).sort()
 }
+
+const GetJobType = (api_data) => {
+    return Array.from(new Set(api_data.map(item => item.jobType.toUpperCase().trim()))).sort()
+}

frontend/src/store/reducers/InitialData.js

@@ -11,12 +11,16 @@ export const OCP_INITIAL_DATA = {
     workers: ["All"],
     ciSystems: ["All"],
     networkTypes: ["All"],
+    jobTypes: ["All"],
+    rehearses: ["All"],
     selectedBenchmark: "All",
     selectedVersion: "All",
     selectedPlatform: "All",
     selectedWorkerCount: "All",
     selectedNetworkType: "All",
     selectedCiSystem: "All",
+    selectedJobType: "All",
+    selectedRehearse: "All",
     waitForUpdate: false,
     platforms: ["All"],
     copyData: [],

frontend/src/store/reducers/OCPJobsReducer.js

@@ -17,11 +17,12 @@ const getFilteredData = (data, selectedValue, keyValue) => {
     return data.filter(item => equalIgnoreCase(item[keyValue], selectedValue) || equalIgnoreCase(selectedValue, 'all') )
 }
 
-const GetUpdatedData = (data, platform, benchmark, version, workerCount, networkType, ciSystem) => {
+const GetUpdatedData = (data, platform, benchmark, version, workerCount, networkType, ciSystem, jobType, isRehearse) => {
     const filterValues = {
         "platform": platform, "benchmark": benchmark,
         "shortVersion": version, "workerNodesCount": workerCount,
-        "networkType": networkType, "ciSystem": ciSystem
+        "networkType": networkType, "ciSystem": ciSystem,
+        "jobType": jobType, "isRehearse": isRehearse,
     }
     let filteredData = data
     for (let [keyName, value] of Object.entries(filterValues))
@@ -63,26 +64,31 @@ const jobsSlice = createSlice({
             state.workers = ["All", ...action.payload.workers]
             state.networkTypes = ["All", ...action.payload.networkTypes]
             state.ciSystems = ["All", ...action.payload.ciSystems]
+            state.jobTypes = ["All", ...action.payload.jobTypes]
+            state.rehearses = ["All", ...action.payload.rehearses]
             state.updatedTime = action.payload.updatedTime
             state.error = null
             Object.assign(state,  GetSummary(state.data))
             state.startDate = action.payload.startDate
             state.endDate = action.payload.endDate
         },
         updateOCPDataFilter: (state, action) => {
-            const {ciSystem, platform, benchmark, version, workerCount, networkType} = action.payload
+            const {ciSystem, platform, benchmark, version, workerCount, networkType, jobType, isRehearse} = action.payload
             state.selectedBenchmark = benchmark
             state.selectedVersion = version
             state.selectedPlatform = platform
             state.selectedNetworkType = networkType
             state.selectedWorkerCount = workerCount
             state.selectedCiSystem = ciSystem
-            state.data = GetUpdatedData(original(state.copyData), platform, benchmark, version, workerCount, networkType, ciSystem)
+            state.selectedJobType = jobType
+            state.selectedRehearse = isRehearse
+            state.data = GetUpdatedData(original(state.copyData), platform, benchmark, version, workerCount, networkType, ciSystem, jobType, isRehearse)
             Object.assign(state,  GetSummary(state.data))
         },
         updateOCPMetaData: (state, action) => {
             state.data = GetUpdatedData(action.payload.data, state.selectedPlatform, state.selectedBenchmark,
-                state.selectedVersion, state.selectedWorkerCount, state.selectedNetworkType, state.selectedCiSystem)
+                state.selectedVersion, state.selectedWorkerCount, state.selectedNetworkType, state.selectedCiSystem,
+                state.selectedJobType, state.selectedRehearse)
             Object.assign(state,  GetSummary(state.data))
         },
         setWaitForOCPUpdate: (state, action) => {