From d07dc16598e408cb9ff03d2ddd609b04061e734f Mon Sep 17 00:00:00 2001 From: Vicente Zepeda Mas Date: Tue, 28 Nov 2023 17:07:54 +0100 Subject: [PATCH] New filters by Job Type and Rehearse - Adds columns to DF - Adds filters to the ocp Home Page Signed-off-by: Vicente Zepeda Mas --- backend/app/api/v1/commons/ocp.py | 24 ++++++++- backend/app/api/v1/endpoints/cpt/cptJobs.py | 4 +- backend/app/api/v1/endpoints/cpt/maps/ocp.py | 9 ++-- backend/app/api/v1/endpoints/ocp/results.py | 49 ++----------------- frontend/src/components/OCP/OCPHome.js | 10 +++- frontend/src/store/Actions/ActionCreator.js | 8 ++- frontend/src/store/reducers/InitialData.js | 4 ++ frontend/src/store/reducers/OCPJobsReducer.js | 16 ++++-- 8 files changed, 66 insertions(+), 58 deletions(-) diff --git a/backend/app/api/v1/commons/ocp.py b/backend/app/api/v1/commons/ocp.py index addefc27..2f409422 100644 --- a/backend/app/api/v1/commons/ocp.py +++ b/backend/app/api/v1/commons/ocp.py @@ -1,4 +1,4 @@ -from datetime import datetime, date +from datetime import date import pandas as pd from app.services.search import ElasticService @@ -32,9 +32,31 @@ async def getData(start_datetime: date, end_datetime: date): if len(jobs) == 0: return jobs + jobs["benchmark"] = jobs.apply(updateBenchmark, axis=1) + jobs["jobType"] = jobs.apply(jobType, axis=1) + jobs["isRehearse"] = jobs.apply(isRehearse, axis=1) + cleanJobs = jobs[jobs['platform'] != ""] jbs = cleanJobs jbs['shortVersion'] = jbs['ocpVersion'].str.slice(0, 4) return jbs + + +def updateBenchmark(row): + if row["upstreamJob"].__contains__("upgrade"): + return "upgrade-" + row["benchmark"] + return row["benchmark"] + + +def jobType(row): + if row["upstreamJob"].__contains__("periodic"): + return "periodic" + return "pull request" + + +def isRehearse(row): + if row["upstreamJob"].__contains__("rehearse"): + return "True" + return "False" diff --git a/backend/app/api/v1/endpoints/cpt/cptJobs.py b/backend/app/api/v1/endpoints/cpt/cptJobs.py index bd0eb2ab..16cd4c88 100644 --- a/backend/app/api/v1/endpoints/cpt/cptJobs.py +++ b/backend/app/api/v1/endpoints/cpt/cptJobs.py @@ -24,8 +24,8 @@ 422: response_422(), },) async def jobs(start_date: date = Query(None, description="Start date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-10"]), - end_date: date = Query(None, description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"]), - pretty: bool = Query(False, description="Output contet in pretty format.")): + end_date: date = Query(None, description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"]), + pretty: bool = Query(False, description="Output contet in pretty format.")): if start_date is None: start_date = datetime.utcnow().date() start_date = start_date - timedelta(days=5) diff --git a/backend/app/api/v1/endpoints/cpt/maps/ocp.py b/backend/app/api/v1/endpoints/cpt/maps/ocp.py index 4d578f2d..4193b094 100644 --- a/backend/app/api/v1/endpoints/cpt/maps/ocp.py +++ b/backend/app/api/v1/endpoints/cpt/maps/ocp.py @@ -1,6 +1,7 @@ from ....commons.ocp import getData from datetime import date + ################################################################ # This will return a DataFrame from OCP required by the CPT # endpoint, it contians the following columns: @@ -27,11 +28,13 @@ async def ocpMapper(start_datetime: date, end_datetime: date): def dropColumns(df): df = df.drop(columns=['shortVersion', 'benchmark', 'platform', 'clusterType', 'masterNodesCount', - 'workerNodesCount', 'infraNodesCount', 'masterNodesType', 'workerNodesType', - 'infraNodesType', 'totalNodesCount', 'clusterName', 'ocpVersion', 'networkType', - 'buildTag', 'upstreamJob', 'upstreamJobBuild', 'executionDate', 'jobDuration', 'timestamp']) + 'workerNodesCount', 'infraNodesCount', 'masterNodesType', 'workerNodesType', + 'infraNodesType', 'totalNodesCount', 'clusterName', 'ocpVersion', 'networkType', + 'buildTag', 'upstreamJob', 'upstreamJobBuild', 'executionDate', 'jobDuration', + 'timestamp', 'jobType', 'isRehearse']) return df + def getReleaseStream(row): if row["releaseStream"].__contains__("fast"): return "Fast" diff --git a/backend/app/api/v1/endpoints/ocp/results.py b/backend/app/api/v1/endpoints/ocp/results.py index b895f815..91cd6c2d 100644 --- a/backend/app/api/v1/endpoints/ocp/results.py +++ b/backend/app/api/v1/endpoints/ocp/results.py @@ -1,20 +1,16 @@ -import asyncio -from typing import Dict, Iterable - -import httpx -from fastapi import APIRouter, Request -from fastapi.param_functions import Query, Path +from fastapi import APIRouter +from fastapi.param_functions import Path from app.services.search import ElasticService router = APIRouter() + @router.get('/api/ocp/v1/jobs/{ci}/{job_id}', summary="Returns the details of a specified Job.") async def results_for_job( - ci: str = Path(..., description="Name of the CI system tha tthe job belongs to.", examples=["PROW", "JENKINS"]), - job_id: str = Path(..., description="Unique identifier of the Jon, normally the UUID.", examples=["8b671d0b-8638-4423-b453-cc54b1caf529"]), - ): + ci: str = Path(..., description="Name of the CI system tha tthe job belongs to.", examples=["PROW", "JENKINS"]), + job_id: str = Path(..., description="Unique identifier of the Jon, normally the UUID.", examples=["8b671d0b-8638-4423-b453-cc54b1caf529"])): query = { "query": { "query_string": { @@ -24,43 +20,8 @@ async def results_for_job( } } - airflow = False - if ci == "AIRFLOW": - airflow = True - es = ElasticService(configpath="ocp.elasticsearch") response = await es.post(query) await es.close() tasks = [item['_source'] for item in response["hits"]["hits"]] - if ci == "AIRFLOW": - tasks_states = await async_tasks_states(tasks) - - for task in tasks: - task['job_status'] = tasks_states[task['build_tag']] - return tasks - - -async def async_tasks_states(tasks: Iterable) -> Dict[str, str]: - async with airflow_service.httpx_client() as session: - tasks_states = await asyncio.gather( - *[call_url(session, task) for task in tasks] - ) - return { - task: state for task_state in tasks_states - for task, state in task_state.items() - } - - -async def call_url(session: httpx.AsyncClient, task) -> Dict[str, str]: - path = ( - f"{airflow_service.base_url}/api/v1" - f"/dags/{task['upstream_job']}" - f"/dagRuns/{task['upstream_job_build']}" - f"/taskInstances/{task['build_tag']}" - ) - resp = await session.get(path) - resp.raise_for_status() - return { - task['build_tag']: resp.json()['state'] - } diff --git a/frontend/src/components/OCP/OCPHome.js b/frontend/src/components/OCP/OCPHome.js index 56ab8072..dc91af55 100644 --- a/frontend/src/components/OCP/OCPHome.js +++ b/frontend/src/components/OCP/OCPHome.js @@ -32,6 +32,8 @@ export function OCPHome() { const [workerCount, setWorkerCount] = useState(searchParams.get("workerCount") || ocpJobs.selectedWorkerCount) const [networkType, setNetworkType] = useState(searchParams.get("networkType") || ocpJobs.selectedNetworkType) const [version, setVersion] = useState(searchParams.get("version") || ocpJobs.selectedVersion) + const [jobType, setJobType] = useState(searchParams.get("jobType") || ocpJobs.selectedJobType) + const [isRehearse, setRehearse] = useState(searchParams.get("isRehearse") || ocpJobs.selectedRehearse) const [startDate, setStartDate] = useState(searchParams.get("startDate") || ocpJobs.startDate) || "" const [endDate, setEndDate] = useState(searchParams.get("endDate") || ocpJobs.endDate) || "" @@ -43,6 +45,8 @@ export function OCPHome() { {name: "Workers Count", onChange: setWorkerCount, value: workerCount, options:ocpJobs.workers }, {name: "Network Type", onChange: setNetworkType, value: networkType, options:ocpJobs.networkTypes }, {name: "Versions", onChange: setVersion, value: version, options: ocpJobs.versions}, + {name: "Job Type", onChange: setJobType, value: jobType, options: ocpJobs.jobTypes}, + {name: "Rehearse", onChange: setRehearse, value: isRehearse, options: ocpJobs.rehearses}, ] useEffect(() => { @@ -53,6 +57,8 @@ export function OCPHome() { if(version !== '') buildParams += `&version=${version}` if(workerCount !== '') buildParams += `&workerCount=${workerCount}` if(networkType !== '') buildParams += `&networkType=${networkType}` + if(jobType !== '') buildParams += `&jobType=${jobType}` + if(isRehearse !== '') buildParams += `&isRehearse=${isRehearse}` if(startDate !== '') buildParams += `&startDate=${startDate}` if(endDate !== '') buildParams += `&endDate=${endDate}` history.push(`/ocp?${buildParams.substring(1)}`, { replace: true }); @@ -60,8 +66,8 @@ export function OCPHome() { }, [history, ciSystem, platform, benchmark, version, workerCount, networkType, startDate, endDate]) useEffect( ()=>{ - dispatch(updateOCPDataFilter({ciSystem, platform, benchmark, version, workerCount, networkType})) - }, [ ciSystem, platform, benchmark, version, workerCount, networkType, dispatch ]) + dispatch(updateOCPDataFilter({ciSystem, platform, benchmark, version, workerCount, networkType, jobType, isRehearse})) + }, [ ciSystem, platform, benchmark, version, workerCount, networkType, , jobType, isRehearse, dispatch ]) useEffect(() => { if(startDate || endDate){ diff --git a/frontend/src/store/Actions/ActionCreator.js b/frontend/src/store/Actions/ActionCreator.js index 81e118be..ebd78ed7 100644 --- a/frontend/src/store/Actions/ActionCreator.js +++ b/frontend/src/store/Actions/ActionCreator.js @@ -57,10 +57,12 @@ export const fetchOCPJobsData = (startDate = '', endDate='') => async dispatch = const workers = GetWorkers(results) const networkTypes = GetNetworkTypes(results) const ciSystems = GetCiSystems(results) + const jobTypes = GetJobType(results) + const rehearses = ["TRUE", "FALSE"] const updatedTime = new Date().toLocaleString().replace(', ', ' ').toString(); await dispatch(getOCPJobsData({ data: results, benchmarks, versions, waitForUpdate: false, platforms, workers, networkTypes, - updatedTime, ciSystems, startDate: api_data.startDate, endDate: api_data.endDate + updatedTime, ciSystems, jobTypes, rehearses, startDate: api_data.startDate, endDate: api_data.endDate })) await dispatch(updateOCPMetaData({data: results})) } @@ -167,3 +169,7 @@ const GetTestNames = (api_data) => { else return item.testName.toLowerCase().trim() }))).sort() } + +const GetJobType = (api_data) => { + return Array.from(new Set(api_data.map(item => item.jobType.toUpperCase().trim()))).sort() +} \ No newline at end of file diff --git a/frontend/src/store/reducers/InitialData.js b/frontend/src/store/reducers/InitialData.js index 366b624f..53e31d50 100644 --- a/frontend/src/store/reducers/InitialData.js +++ b/frontend/src/store/reducers/InitialData.js @@ -11,12 +11,16 @@ export const OCP_INITIAL_DATA = { workers: ["All"], ciSystems: ["All"], networkTypes: ["All"], + jobTypes: ["All"], + rehearses: ["All"], selectedBenchmark: "All", selectedVersion: "All", selectedPlatform: "All", selectedWorkerCount: "All", selectedNetworkType: "All", selectedCiSystem: "All", + selectedJobType: "All", + selectedRehearse: "All", waitForUpdate: false, platforms: ["All"], copyData: [], diff --git a/frontend/src/store/reducers/OCPJobsReducer.js b/frontend/src/store/reducers/OCPJobsReducer.js index 18c6dc42..4d225004 100644 --- a/frontend/src/store/reducers/OCPJobsReducer.js +++ b/frontend/src/store/reducers/OCPJobsReducer.js @@ -17,11 +17,12 @@ const getFilteredData = (data, selectedValue, keyValue) => { return data.filter(item => equalIgnoreCase(item[keyValue], selectedValue) || equalIgnoreCase(selectedValue, 'all') ) } -const GetUpdatedData = (data, platform, benchmark, version, workerCount, networkType, ciSystem) => { +const GetUpdatedData = (data, platform, benchmark, version, workerCount, networkType, ciSystem, jobType, isRehearse) => { const filterValues = { "platform": platform, "benchmark": benchmark, "shortVersion": version, "workerNodesCount": workerCount, - "networkType": networkType, "ciSystem": ciSystem + "networkType": networkType, "ciSystem": ciSystem, + "jobType": jobType, "isRehearse": isRehearse, } let filteredData = data for (let [keyName, value] of Object.entries(filterValues)) @@ -63,6 +64,8 @@ const jobsSlice = createSlice({ state.workers = ["All", ...action.payload.workers] state.networkTypes = ["All", ...action.payload.networkTypes] state.ciSystems = ["All", ...action.payload.ciSystems] + state.jobTypes = ["All", ...action.payload.jobTypes] + state.rehearses = ["All", ...action.payload.rehearses] state.updatedTime = action.payload.updatedTime state.error = null Object.assign(state, GetSummary(state.data)) @@ -70,19 +73,22 @@ const jobsSlice = createSlice({ state.endDate = action.payload.endDate }, updateOCPDataFilter: (state, action) => { - const {ciSystem, platform, benchmark, version, workerCount, networkType} = action.payload + const {ciSystem, platform, benchmark, version, workerCount, networkType, jobType, isRehearse} = action.payload state.selectedBenchmark = benchmark state.selectedVersion = version state.selectedPlatform = platform state.selectedNetworkType = networkType state.selectedWorkerCount = workerCount state.selectedCiSystem = ciSystem - state.data = GetUpdatedData(original(state.copyData), platform, benchmark, version, workerCount, networkType, ciSystem) + state.selectedJobType = jobType + state.selectedRehearse = isRehearse + state.data = GetUpdatedData(original(state.copyData), platform, benchmark, version, workerCount, networkType, ciSystem, jobType, isRehearse) Object.assign(state, GetSummary(state.data)) }, updateOCPMetaData: (state, action) => { state.data = GetUpdatedData(action.payload.data, state.selectedPlatform, state.selectedBenchmark, - state.selectedVersion, state.selectedWorkerCount, state.selectedNetworkType, state.selectedCiSystem) + state.selectedVersion, state.selectedWorkerCount, state.selectedNetworkType, state.selectedCiSystem, + state.selectedJobType, state.selectedRehearse) Object.assign(state, GetSummary(state.data)) }, setWaitForOCPUpdate: (state, action) => {