Merge pull request #53 from chentex/api-refactor

Refactoring and organizing the Dashboard and API
cloud-bulldozer · Nov 28, 2023 · 36bcf16 · 36bcf16
2 parents 140d466 + c8723d5
commit 36bcf16
Show file tree

Hide file tree

Showing 62 changed files with 1,939 additions and 2,133 deletions.
diff --git a/backend/README.md b/backend/README.md
@@ -9,7 +9,7 @@
 * pipenv
 
 Add `app` directory as a module that can be found by the Python importer.
-  
+
 ```shell
 export PYTHONPATH=$PWD
 ```
@@ -36,4 +36,4 @@ Start backend application with hot reload.
 
 ```shell
 ./scripts/start-reload.sh
-```
+```
diff --git a/backend/app/api/api.py b/backend/app/api/api.py
@@ -1,20 +1,16 @@
 from fastapi import APIRouter
 
-from app.api.v1.endpoints import results
-from app.api.v1.endpoints import airflow
-from app.api.v1.endpoints import jobs
-from app.api.v1.endpoints import jenkins
-from app.api.v1.endpoints import graph
-from app.api.v2.endpoints import jobs as jobsv2
+from app.api.v1.endpoints.ocp import results
+from app.api.v1.endpoints.ocp import ocpJobs
+from app.api.v1.endpoints.ocp import graph
+from app.api.v1.endpoints.cpt import cptJobs
 
-api_router = APIRouter()
+router = APIRouter()
 
-# v1 endopoints
-api_router.include_router(results.router, tags=['perfscale'])
-api_router.include_router(airflow.router, tags=['perfscale'])
-api_router.include_router(jobs.router, tags=['perfscale'])
-api_router.include_router(jenkins.router, tags=['perfscale'])
-api_router.include_router(graph.router, tags=['perfscale'])
+# OCP endopoints
+router.include_router(ocpJobs.router, tags=['ocp'])
+router.include_router(results.router, tags=['ocp'])
+router.include_router(graph.router, tags=['ocp.graphs'])
 
-# v2 endpoints
-api_router.include_router(jobsv2.router, tags=['perfscale', 'v2'])
+# CPT endopoints
+router.include_router(cptJobs.router, tags=['cpt'])
diff --git a/backend/app/api/v1/commons/example_responses.py b/backend/app/api/v1/commons/example_responses.py
@@ -0,0 +1,119 @@
+def response_200(example):
+    return {
+                "content": {
+                    "application/json": {
+                        "example": example,
+                    }
+                },
+            }
+
+def response_422():
+    return {
+                "content": {
+                    "application/json": {
+                        "example": {"error": "invalid date format, start_date must be less than end_date"},
+                    }
+                },
+            }
+
+ocp_response_example ={
+    "startDate": "2023-09-20",
+    "endDate": "2023-09-20",
+    "results": [
+        {
+            "ciSystem": "PROW",
+            "uuid": "CPT-e3865b03-ce78-454a-becb-b79aeb806a6b",
+            "releaseStream": "4.14.0-0.nightly",
+            "platform": "AWS",
+            "clusterType": "self-managed",
+            "benchmark": "cluster-density-v2",
+            "masterNodesCount": 3,
+            "workerNodesCount": 252,
+            "infraNodesCount": 3,
+            "masterNodesType": "m6a.8xlarge",
+            "workerNodesType": "m5.2xlarge",
+            "infraNodesType": "r5.4xlarge",
+            "totalNodesCount": 258,
+            "clusterName": "ci-op-4n0msnvp-7904a-s5sv8",
+            "ocpVersion": "4.14.0-0.nightly-2023-09-15-233408",
+            "networkType": "OVNKubernetes",
+            "buildTag": "1704299395064795136",
+            "jobStatus": "success",
+            "buildUrl": "https://example.com/1704299395064795136",
+            "upstreamJob": "periodic-ci-openshift",
+            "upstreamJobBuild": "5fe07ad3-5415-433c-b9af-f60545d0d432",
+            "executionDate": "2023-09-20T02:14:07Z",
+            "jobDuration": "5261",
+            "startDate": "2023-09-20T02:14:07Z",
+            "endDate": "2023-09-20T03:41:48Z",
+            "timestamp": "2023-09-20T02:14:07Z",
+            "shortVersion": "4.14"
+        },
+        {
+            "ciSystem": "PROW",
+            "uuid": "CPT-0d58dddf-721a-4952-985e-046bc17ee3cc",
+            "releaseStream": "4.13.0-0.nightly",
+            "platform": "GCP",
+            "clusterType": "self-managed",
+            "benchmark": "node-density",
+            "masterNodesCount": 3,
+            "workerNodesCount": 24,
+            "infraNodesCount": 3,
+            "masterNodesType": "e2-standard-4",
+            "workerNodesType": "e2-standard-4",
+            "infraNodesType": "n1-standard-16",
+            "totalNodesCount": 30,
+            "clusterName": "ci-op-x2ic4nsf-8360f-kzbcg",
+            "ocpVersion": "4.13.0-0.nightly-2023-09-12-074803",
+            "networkType": "OVNKubernetes",
+            "buildTag": "1704367060252889088",
+            "jobStatus": "success",
+            "buildUrl": "https://example/1704367060252889088",
+            "upstreamJob": "periodic-ci-openshift",
+            "upstreamJobBuild": "3ab02e3b-3000-4fc9-a30c-9cd02fe4a78c",
+            "executionDate": "2023-09-20T07:19:00Z",
+            "jobDuration": "582",
+            "startDate": "2023-09-20T07:19:00Z",
+            "endDate": "2023-09-20T07:28:42Z",
+            "timestamp": "2023-09-20T07:19:00Z",
+            "shortVersion": "4.13"
+        },
+    ]
+}
+
+def ocp_200_response():
+    return response_200(ocp_response_example)
+
+cpt_response_example ={
+  "startDate": "2023-11-18",
+  "endDate": "2023-11-23",
+  "results": [
+        {
+        "ciSystem": "PROW",
+        "uuid": "f6d084d5-b154-4108-b4f7-165094ccc838",
+        "releaseStream": "Nightly",
+        "jobStatus": "success",
+        "buildUrl": "https://ci..org/view/1726571333392797696",
+        "startDate": "2023-11-20T13:16:34Z",
+        "endDate": "2023-11-20T13:28:48Z",
+        "product": "ocp",
+        "version": "4.13",
+        "testName": "cluster-density-v2"
+        },
+        {
+        "ciSystem": "JENKINS",
+        "uuid": "5b729011-3b4d-4ec4-953d-6881ac9da505",
+        "releaseStream": "Stable",
+        "jobStatus": "success",
+        "buildUrl": "https://ci..org/view/1726571333392797696",
+        "startDate": "2023-11-20T13:16:30Z",
+        "endDate": "2023-11-20T13:30:40Z",
+        "product": "ocp",
+        "version": "4.14",
+        "testName": "node-density-heavy"
+        },
+    ]
+}
+
+def cpt_200_response():
+    return response_200(cpt_response_example)
diff --git a/backend/app/api/v2/commons/common.py → backend/app/api/v1/commons/ocp.py b/backend/app/api/v2/commons/common.py → backend/app/api/v1/commons/ocp.py
@@ -20,7 +20,7 @@ async def getData(start_datetime: date, end_datetime: date):
     query['query']['bool']['filter']['range']['timestamp']['lte'] = str(end_datetime)
     query['query']['bool']['filter']['range']['timestamp']['gte'] = str(start_datetime)
 
-    es = ElasticService()
+    es = ElasticService(configpath="ocp.elasticsearch")
     response = await es.post(query)
     await es.close()
     tasks = [item['_source'] for item in response["hits"]["hits"]]

diff --git a/backend/app/api/v1/endpoints/airflow.py b/backend/app/api/v1/endpoints/airflow.py
diff --git a/backend/app/api/v1/endpoints/common.py b/backend/app/api/v1/endpoints/common.py
diff --git a/backend/app/api/v1/endpoints/cpt/cptJobs.py b/backend/app/api/v1/endpoints/cpt/cptJobs.py
@@ -0,0 +1,55 @@
+import json
+from fastapi import Response
+import pandas as pd
+from datetime import datetime, timedelta, date
+from fastapi import APIRouter
+from .maps.ocp import ocpMapper
+from ...commons.example_responses import cpt_200_response, response_422
+from fastapi.param_functions import Query
+
+router = APIRouter()
+
+products = {
+            "ocp": ocpMapper,
+           }
+
+@router.get('/api/cpt/v1/jobs',
+            summary="Returns a job list from all the products.",
+            description="Returns a list of jobs in the specified dates. \
+            If not dates are provided the API will default the values. \
+            `startDate`: will be set to the day of the request minus 5 days.\
+            `endDate`: will be set to the day of the request.",
+            responses={
+                200: cpt_200_response(),
+                422: response_422(),
+            },)
+async def jobs(start_date: date = Query(None, description="Start date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-10"]),
+                end_date: date = Query(None, description="End date for searching jobs, format: 'YYYY-MM-DD'", examples=["2020-11-15"]),
+                pretty: bool = Query(False, description="Output contet in pretty format.")):
+    if start_date is None:
+        start_date = datetime.utcnow().date()
+        start_date = start_date - timedelta(days=5)
+
+    if end_date is None:
+        end_date = datetime.utcnow().date()
+
+    if start_date > end_date:
+        return Response(content=json.dumps({'error': "invalid date format, start_date must be less than end_date"}), status_code=422)
+
+    results = pd.DataFrame()
+    for func in products.values():
+        df = await func(start_date, end_date)
+        results = pd.concat([results, df])
+
+    response = {
+        'startDate': start_date.__str__(),
+        'endDate': end_date.__str__(),
+        'results': results.to_dict('records')
+    }
+
+    if pretty:
+        json_str = json.dumps(response, indent=4)
+        return Response(content=json_str, media_type='application/json')
+
+    jsonstring = json.dumps(response)
+    return jsonstring