Add Valid Question Operator (#577)

* add question completeness operator

* update operator name

uptrain/operators/__init__.pyi

@@ -59,6 +59,7 @@ __all__ = [
     "ResponseCompletenessWrtContext",
     "ResponseConsistency",
     "ResponseConciseness",
+    "ValidQuestion",
     "LanguageCritique",
     "ToneCritique",
     "GuidelineAdherenceScore",
@@ -160,6 +161,9 @@ from .language.response_quality import (
     ResponseRelevance,
     ResponseMatchingScore,
 )
+from .language.question_quality import (
+    ValidQuestion
+)
 from .language.language_quality import LanguageCritique, ResponseCoherence
 from .language.tone import ToneCritique
 from .language.guideline import GuidelineAdherenceScore

uptrain/operators/language/question_quality.py

@@ -0,0 +1,56 @@
+"""
+Implement operators to check the quality of the question given by the user. 
+"""
+
+from __future__ import annotations
+import typing as t
+
+from loguru import logger
+import polars as pl
+
+
+if t.TYPE_CHECKING:
+    from uptrain.framework import Settings
+from uptrain.operators.base import register_op, ColumnOp, TYPE_TABLE_OUTPUT
+
+
+@register_op
+class ValidQuestion(ColumnOp):
+    """
+    Simply check the number of words in the question and grades as incomplete if below a threshold
+    Attributes:
+        col_question: (str) Column Name for the stored questions
+        col_out: (str) Column
+    Raises:
+        Exception: Raises exception for any failed evaluation attempts
+
+    """
+
+    col_question: str = "question"
+    words_threshold: int = 1
+    col_out: str = "score_valid_question"
+
+    def setup(self, settings: t.Optional[Settings] = None):
+        assert settings is not None
+        self.settings = settings
+        return self
+
+    def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
+        data_send = data.to_dicts()
+        results = []
+        try:
+            for row in data_send:
+                question = row.pop(self.col_question)
+                results.append({"score_valid_question": int(len(question.split(" ")) > self.words_threshold)})
+        except Exception as e:
+            logger.error(f"Failed to run evaluation for `ValidQuestion`: {e}")
+            raise e
+
+        assert results is not None
+        return {
+            "output": data.with_columns(
+                pl.from_dicts(results).rename(
+                    {"score_valid_question": self.col_out}
+                )
+            )
+        }