Dev 2 Test version 0.2.11

aixplain/enums/asset_status.py

@@ -0,0 +1,43 @@
+__author__ = "thiagocastroferreira"
+
+"""
+Copyright 2024 The aiXplain SDK authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Author: Duraikrishna Selvaraju, Thiago Castro Ferreira, Shreyas Sharma and Lucas Pavanelli
+Date: February 21st 2024
+Description:
+    Asset Enum
+"""
+
+from enum import Enum
+from typing import Text
+
+class AssetStatus(Text, Enum):
+    HIDDEN = 'hidden'
+    SCHEDULED = 'scheduled'
+    ONBOARDING = 'onboarding'
+    ONBOARDED = 'onboarded'
+    PENDING = 'pending'
+    FAILED = 'failed'
+    TRAINING = 'training'
+    REJECTED = 'rejected'
+    ENABLING = 'enabling'
+    DELETING = 'deleting'
+    DISABLED = 'disabled'
+    DELETED = 'deleted'
+    IN_PROGRESS = 'in_progress'
+    COMPLETED = 'completed'
+    CANCELING = 'canceling'
+    CANCELED = 'canceled'

aixplain/factories/benchmark_factory.py

@@ -26,6 +26,7 @@
 import json
 import pandas as pd
 from pathlib import Path
+from aixplain.enums.supplier import Supplier
 from aixplain.modules import Dataset, Metric, Model
 from aixplain.modules.benchmark_job import BenchmarkJob
 from aixplain.modules.benchmark import Benchmark
@@ -237,3 +238,24 @@ def list_normalization_options(cls, metric: Metric, model: Model) -> List[str]:
             error_message = f"Listing Normalization Options: Error in getting Normalization Options: {e}"
             logging.error(error_message, exc_info=True)
             return []
+
+    @classmethod
+    def get_benchmark_job_scores(cls, job_id):
+        def __get_model_name(model_id):
+            model = ModelFactory.get(model_id)
+            supplier = str(model.supplier)
+            try:
+                if isinstance(supplier, Supplier):
+                    name = f"{supplier.name}"
+                else:
+                    name = f"{eval(supplier)['name']}"
+            except Exception as e:
+                logging.error(f"{e}")
+                name = f"{supplier}"
+            if model.version is not None:
+                name = f"{name}({model.version})"
+            return name
+        benchmarkJob = cls.get_job(job_id)
+        scores_df = benchmarkJob.get_scores()
+        scores_df["Model"] = scores_df["Model"].apply(lambda x: __get_model_name(x))
+        return scores_df

aixplain/modules/__init__.py

@@ -29,5 +29,6 @@
 from .model import Model
 from .pipeline import Pipeline
 from .finetune import Finetune, FinetuneCost
+from .finetune.status import FinetuneStatus
 from .benchmark import Benchmark
 from .benchmark_job import BenchmarkJob

aixplain/modules/benchmark_job.py

@@ -92,3 +92,125 @@ def download_results_as_csv(self, save_path: Optional[Text] = None, return_dataf
             error_message = f"Downloading Benchmark Results: Error in Downloading Benchmark Results : {e}"
             logging.error(error_message, exc_info=True)
             raise Exception(error_message)
+
+    def __simplify_scores(self, scores):
+        simplified_score_list  = []
+        for model_id, model_info in scores.items():
+            model_scores = model_info["rawScores"]
+            # model = Mode
+            row = {"Model": model_id}
+            for score_info in model_scores:
+                row[score_info["longName"]] = score_info["average"]
+            simplified_score_list.append(row)
+        return simplified_score_list
+
+
+
+
+    def get_scores(self, return_simplified=True, return_as_dataframe=True):
+        try:
+            resp = self._fetch_current_response(self.id)
+            iterations = resp.get("iterations", [])
+            scores = {}
+            for iteration_info in iterations:
+                model_id = iteration_info["pipeline"]
+                model_info = {
+                    "creditsUsed" : round(iteration_info.get("credits", 0),5),
+                    "timeSpent" : round(iteration_info.get("runtime", 0),2),
+                    "status" : iteration_info["status"],
+                    "rawScores" : iteration_info["scores"],
+                }
+                scores[model_id] = model_info
+
+            if return_simplified:
+                simplified_scores = self.__simplify_scores(scores)
+                if return_as_dataframe:
+                    simplified_scores = pd.DataFrame(simplified_scores)
+                return simplified_scores
+            else:
+                return scores
+        except Exception as e:
+            error_message = f"Benchmark scores: Error in Getting benchmark scores: {e}"
+            logging.error(error_message, exc_info=True)
+            raise Exception(error_message)
+
+
+    def get_failuire_rate(self, return_as_dataframe=True):
+        try:
+            scores = self.get_scores(return_simplified=False)
+            failure_rates = {}
+            for model_id, model_info in scores.items():
+                if len(model_info["rawScores"]) == 0:
+                    failure_rates[model_id] = 0
+                    continue
+                score_info = model_info["rawScores"][0] 
+                num_succesful = score_info["count"]
+                num_failed = score_info["failedSegmentsCount"]
+                failuire_rate =  (num_failed * 100) / (num_succesful+num_failed)
+                failure_rates[model_id] = failuire_rate
+            if return_as_dataframe:
+                df = pd.DataFrame()
+                df["Model"] = list(failure_rates.keys())
+                df["Failuire Rate"] = list(failure_rates.values())
+                return df
+            else:
+                return failure_rates
+        except Exception as e:
+            error_message = f"Benchmark scores: Error in Getting benchmark failuire rate: {e}"
+            logging.error(error_message, exc_info=True)
+            raise Exception(error_message)
+
+    def get_all_explanations(self):
+        try:
+            resp = self._fetch_current_response(self)
+            raw_explanations = resp.get("explanation", {})
+            if "metricInDependent" not in raw_explanations:
+                raw_explanations["metricInDependent"] = []
+            if "metricDependent" not in raw_explanations:
+                raw_explanations["metricDependent"] = []
+            return raw_explanations
+        except Exception as e:
+            error_message = f"Benchmark scores: Error in Getting benchmark explanations: {e}"
+            logging.error(error_message, exc_info=True)
+            raise Exception(error_message)
+
+    def get_localized_explanations(self, metric_dependant: bool, group_by_task: bool = False):
+        try:
+            raw_explanations = self.get_all_explanations()
+            if metric_dependant:
+                localized_explanations = raw_explanations["metricDependent"]
+                if len(localized_explanations) == 0:
+                    localized_explanations = {}
+                else:
+                    grouped_explanations = {}
+                    task_list = []
+                    first_explanation = localized_explanations[0]
+                    for task in first_explanation:
+                        if task not in ["scoreId", "datasetId"]:
+                            task_list.append(task)
+
+                    if group_by_task:
+                        for task in task_list:
+                            task_explanation = {}
+                            for explanation_item in localized_explanations:
+                                item_task_explanation = explanation_item[task]
+                                identifier = explanation_item["scoreId"]
+                                task_explanation[identifier] = item_task_explanation
+                            grouped_explanations[task] = task_explanation
+                    else:
+                        for explanation_item in localized_explanations:
+                            identifier = explanation_item["scoreId"]
+                            grouped_explanations[identifier] = explanation_item
+                    localized_explanations = grouped_explanations
+            else:
+                localized_explanations = raw_explanations["metricInDependent"]
+                if len(localized_explanations) == 0:
+                    localized_explanations =  {}
+                else:
+                    localized_explanations = localized_explanations[0]
+            return localized_explanations
+
+        except Exception as e:
+            error_message = f"Benchmark scores: Error in Getting benchmark explanations: {e}"
+            logging.error(error_message, exc_info=True)
+            raise Exception(error_message)

aixplain/modules/finetune/status.py

@@ -0,0 +1,36 @@
+__author__ = "thiagocastroferreira"
+
+"""
+Copyright 2024 The aiXplain SDK authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Author: Duraikrishna Selvaraju, Thiago Castro Ferreira, Shreyas Sharma and Lucas Pavanelli
+Date: February 21st 2024
+Description:
+    FinetuneCost Class
+"""
+
+from aixplain.enums.asset_status import AssetStatus
+from dataclasses import dataclass
+from dataclasses_json import dataclass_json
+from typing import Optional, Text
+
+@dataclass_json
+@dataclass
+class FinetuneStatus(object):
+    status: "AssetStatus"
+    model_status: "AssetStatus"
+    epoch: Optional[float] = None
+    training_loss: Optional[float] = None
+    validation_loss: Optional[float] = None

aixplain/modules/metric.py

@@ -23,6 +23,7 @@
 
 from typing import Optional, Text, List, Union
 from aixplain.modules.asset import Asset
+
 from aixplain.utils.file_utils import _request_with_retry
 from aixplain.factories.model_factory import ModelFactory
 
@@ -92,6 +93,7 @@ def run(self, hypothesis: Optional[Union[str, List[str]]]=None, source: Optional
             source (Optional[Union[str, List[str]]], optional): Can give a single source or a list of sources for metric calculation. Defaults to None.
             reference (Optional[Union[str, List[str]]], optional): Can give a single reference or a list of references for metric calculation. Defaults to None.
         """
+        from aixplain.factories.model_factory import ModelFactory
         model = ModelFactory.get(self.id)
         payload = {
             "function": self.function,

aixplain/modules/model.py

@@ -20,7 +20,6 @@
 Description:
     Model Class
 """
-
 import time
 import json
 import logging
@@ -251,23 +250,65 @@ def run_async(self, data: Union[Text, Dict], name: Text = "model_process", param
                 response["error"] = msg
         return response
 
-    def check_finetune_status(self):
+    def check_finetune_status(self, after_epoch: Optional[int] = None):
         """Check the status of the FineTune model.
 
+        Args:
+            after_epoch (Optional[int], optional): status after a given epoch. Defaults to None.
+
         Raises:
             Exception: If the 'TEAM_API_KEY' is not provided.
 
         Returns:
-            str: The status of the FineTune model.
+            FinetuneStatus: The status of the FineTune model.
         """
+        from aixplain.enums.asset_status import AssetStatus
+        from aixplain.modules.finetune.status import FinetuneStatus
         headers = {"x-api-key": self.api_key, "Content-Type": "application/json"}
+        resp = None
         try:
-            url = urljoin(self.backend_url, f"sdk/models/{self.id}")
+            url = urljoin(self.backend_url, f"sdk/finetune/{self.id}/ml-logs")
             logging.info(f"Start service for GET Check FineTune status Model  - {url} - {headers}")
             r = _request_with_retry("get", url, headers=headers)
             resp = r.json()
-            status = resp["status"]
-            logging.info(f"Response for GET Check FineTune status Model - Id {self.id} / Status {status}.")
+            finetune_status = AssetStatus(resp["finetuneStatus"])
+            model_status = AssetStatus(resp["modelStatus"])
+            logs = sorted(resp["logs"], key=lambda x: float(x["epoch"]))
+
+            target_epoch = None
+            if after_epoch is not None:
+                logs = [log for log in logs if float(log["epoch"]) > after_epoch]
+                if len(logs) > 0:
+                    target_epoch = float(logs[0]["epoch"])
+            elif len(logs) > 0:
+                target_epoch = float(logs[-1]["epoch"])
+
+            if target_epoch is not None:
+                log = None
+                for log_ in logs:
+                    if int(log_["epoch"]) == target_epoch:
+                        if log is None:
+                            log = log_
+                        else:
+                            if log_["trainLoss"] is not None:
+                                log["trainLoss"] = log_["trainLoss"]
+                            if log_["evalLoss"] is not None:
+                                log["evalLoss"] = log_["evalLoss"]
+
+                status = FinetuneStatus(
+                    status=finetune_status,
+                    model_status=model_status,
+                    epoch=float(log["epoch"]) if "epoch" in log and log["epoch"] is not None else None,
+                    training_loss=float(log["trainLoss"]) if "trainLoss" in log and log["trainLoss"] is not None else None,
+                    validation_loss=float(log["evalLoss"]) if "evalLoss" in log and log["evalLoss"] is not None else None,
+                )
+            else:
+                status = FinetuneStatus(
+                    status=finetune_status,
+                    model_status=model_status,
+                )
+
+            logging.info(f"Response for GET Check FineTune status Model - Id {self.id} / Status {status.status.value}.")
             return status
         except Exception as e:
             message = ""

pyproject.toml

@@ -10,7 +10,7 @@ namespaces = true
 
 [project]
 name = "aiXplain"
-version = "0.2.4"
+version = "0.2.5rc"
 description = "aiXplain SDK adds AI functions to software."
 readme = "README.md"
 requires-python = ">=3.5, <4"