Skip to content

Commit

Permalink
feat: add filter and transformation support to breakdown / attrition …
Browse files Browse the repository at this point in the history 
…endpoint

Also:
- feat: update RetrieveStatsForCohortIdAndConceptId and remove deprecated method
- feat: refactor RetrieveCohortOverlapStats and fix breakdown stats method
...and fix model tests
- feat: comment on a comment
...see also #5 (comment)
- feat: refactor RetrieveDataBySourceIdAndCohortIdAndVariables
... and deprecate model method in favor of the histogram one
that retrieves the same data but is already refactored.
- feat: remove deprecated breakdown stats method
  • Loading branch information
@pieterlukasse
pieterlukasse committed Feb 11, 2025
1 parent d2cbc71 commit 2c9901b
Show file tree
Hide file tree
Showing 9 changed files with 243 additions and 369 deletions.
96 changes: 46 additions & 50 deletions controllers/cohortdata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,7 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co

// parse cohortPairs separately as well, so we can validate permissions
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids", "error": err.Error()})
c.Abort()
return
}

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
log.Printf("Error: invalid request")
Expand All @@ -69,22 +65,15 @@ func (u CohortDataController) RetrieveHistogramForCohortIdAndConceptId(c *gin.Co
}

func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Context) {
sourceIdStr := c.Param("sourceid")
log.Printf("Querying source: %s", sourceIdStr)
cohortIdStr := c.Param("cohortid")
log.Printf("Querying cohort for cohort definition id: %s", cohortIdStr)
conceptIdStr := c.Param("conceptid")
if sourceIdStr == "" || cohortIdStr == "" || conceptIdStr == "" {
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request"})
sourceId, cohortId, conceptIdsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
c.Abort()
return
}

filterConceptIdsAndValues, cohortPairs, _ := utils.ParseConceptDefsAndDichotomousDefs(c)

sourceId, _ := strconv.Atoi(sourceIdStr)
cohortId, _ := strconv.Atoi(cohortIdStr)
conceptId, _ := strconv.ParseInt(conceptIdStr, 10, 64)
// parse cohortPairs separately as well, so we can validate permissions
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
Expand All @@ -94,7 +83,7 @@ func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Contex
return
}

cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsAndCohortPairs(sourceId, cohortId, conceptId, filterConceptIdsAndValues, cohortPairs)
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, conceptIdsAndCohortPairs)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
c.Abort()
Expand All @@ -105,37 +94,30 @@ func (u CohortDataController) RetrieveStatsForCohortIdAndConceptId(c *gin.Contex
for _, personData := range cohortData {
conceptValues = append(conceptValues, float64(*personData.ConceptValueAsNumber))
}
conceptToStat, errGetLast := utils.CheckAndGetLastCustomConceptVariableDef(conceptIdsAndCohortPairs)
if errGetLast != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error: last variable should be of numeric type", "error": errGetLast.Error()})
c.Abort()
return
}

statsData := utils.GenerateStatsData(cohortId, conceptId, conceptValues)
statsData := utils.GenerateStatsData(cohortId, conceptToStat.ConceptId, conceptValues)

c.JSON(http.StatusOK, gin.H{"statsData": statsData})
}

func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *gin.Context) {
// TODO - add some validation to ensure that only calls from Argo are allowed through since it outputs FULL data?

// parse and validate all parameters:
sourceIdStr := c.Param("sourceid")
log.Printf("Querying source: %s", sourceIdStr)
cohortIdStr := c.Param("cohortid")
log.Printf("Querying cohort for cohort definition id: %s", cohortIdStr)
if sourceIdStr == "" || cohortIdStr == "" {
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request"})
c.Abort()
return
}

conceptIdsAndValues, cohortPairs, err := utils.ParseConceptDefsAndDichotomousDefs(c)
conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues)

// -> this concern is considered to be addressed by https://github.com/uc-cdis/cloud-automation/pull/1884
sourceId, cohortId, conceptDefsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error parsing request body for prefixed concept ids and dichotomous Ids", "error": err.Error()})
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
c.Abort()
return
}

sourceId, _ := strconv.Atoi(sourceIdStr)
cohortId, _ := strconv.Atoi(cohortIdStr)
// parse cohortPairs separately as well, so we can validate permissions
conceptDefs, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptDefsAndCohortPairs)

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
Expand All @@ -145,17 +127,32 @@ func (u CohortDataController) RetrieveDataBySourceIdAndCohortIdAndVariables(c *g
return
}

// call model method:
cohortData, err := u.cohortDataModel.RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId, cohortId, conceptIds)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
c.Abort()
return
// Iterate over conceptDefsAndCohortPairs and collect the concept values for each person:
// {PersonId:1, ConceptId:1, ConceptValue: "A value with, comma!"},
// {PersonId:1, ConceptId:2, ConceptValue: B},
// {PersonId:2, ConceptId:1, ConceptValue: C},
var variablesToQuery []interface{}
var finalConceptDataset []*models.PersonConceptAndValue
for _, item := range conceptDefsAndCohortPairs {
variablesToQuery = append(variablesToQuery, item)
// if item is of type CustomConceptVariableDef, get the data:
if _, ok := item.(utils.CustomConceptVariableDef); ok {
// use variablesToQuery to query an increasingly tight set (simulating the attrition table that generated this query)
cohortData, err := u.cohortDataModel.RetrieveHistogramDataBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, variablesToQuery)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept details", "error": err.Error()})
c.Abort()
return
}
// add to final concept data set:
finalConceptDataset = append(finalConceptDataset, cohortData...)
}
}

partialCSV := GeneratePartialCSV(sourceId, cohortData, conceptIds)
conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptDefs)
partialCSV := GeneratePartialCSV(sourceId, finalConceptDataset, conceptIds) // use conceptdefs to improve column description? nah...no person is reading this table....just needs to be unique

personIdToCSVValues, err := u.RetrievePeopleIdAndCohort(sourceId, cohortId, cohortPairs, cohortData)
personIdToCSVValues, err := u.RetrievePeopleIdAndCohort(sourceId, cohortId, cohortPairs, finalConceptDataset)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving people ID to csv value map", "error": err.Error()})
c.Abort()
Expand Down Expand Up @@ -287,13 +284,12 @@ func populateConceptValue(row []string, cohortItem models.PersonConceptAndValue,
func (u CohortDataController) RetrieveCohortOverlapStats(c *gin.Context) {
errors := make([]error, 4)
var sourceId, caseCohortId, controlCohortId int
var conceptIdsAndValues []utils.CustomConceptVariableDef
var cohortPairs []utils.CustomDichotomousVariableDef
var conceptDefsAndCohortPairs []interface{}
sourceId, errors[0] = utils.ParseNumericArg(c, "sourceid")
caseCohortId, errors[1] = utils.ParseNumericArg(c, "casecohortid")
controlCohortId, errors[2] = utils.ParseNumericArg(c, "controlcohortid")
conceptIdsAndValues, cohortPairs, errors[3] = utils.ParseConceptDefsAndDichotomousDefs(c)
conceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(conceptIdsAndValues)
conceptDefsAndCohortPairs, errors[3] = utils.ParseConceptDefsAndDichotomousDefsAsSingleList(c)
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptDefsAndCohortPairs)

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{caseCohortId, controlCohortId}, cohortPairs)
if !validAccessRequest {
Expand All @@ -309,7 +305,7 @@ func (u CohortDataController) RetrieveCohortOverlapStats(c *gin.Context) {
return
}
overlapStats, err := u.cohortDataModel.RetrieveCohortOverlapStats(sourceId, caseCohortId,
controlCohortId, conceptIds, cohortPairs)
controlCohortId, conceptDefsAndCohortPairs)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving stats", "error": err.Error()})
c.Abort()
Expand Down
31 changes: 17 additions & 14 deletions controllers/concept.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,13 +125,16 @@ func (u ConceptController) RetrieveBreakdownStatsBySourceIdAndCohortId(c *gin.Co
}

func (u ConceptController) RetrieveBreakdownStatsBySourceIdAndCohortIdAndVariables(c *gin.Context) {
sourceId, cohortId, conceptIds, cohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesList(c)
sourceId, cohortId, conceptDefsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
c.Abort()
return
}

// parse cohortPairs separately as well, so we can validate permissions
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptDefsAndCohortPairs)

validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
log.Printf("Error: invalid request")
Expand All @@ -147,7 +150,7 @@ func (u ConceptController) RetrieveBreakdownStatsBySourceIdAndCohortIdAndVariabl
c.Abort()
return
}
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, conceptIds, cohortPairs, breakdownConceptId)
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, conceptDefsAndCohortPairs, breakdownConceptId)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving stats", "error": err.Error()})
Expand Down Expand Up @@ -190,14 +193,14 @@ func generateRowForVariable(variableName string, breakdownConceptValuesToPeopleC
}

func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
sourceId, cohortId, conceptIdsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
sourceId, cohortId, conceptDefsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
c.Abort()
return
}
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptIdsAndCohortPairs)
_, cohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(conceptDefsAndCohortPairs)
validAccessRequest := u.teamProjectAuthz.TeamProjectValidation(c, []int{cohortId}, cohortPairs)
if !validAccessRequest {
log.Printf("Error: invalid request")
Expand Down Expand Up @@ -238,7 +241,7 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
c.Abort()
return
}
otherAttritionRows, err := u.GetAttritionRowForConceptIdsAndCohortPairs(sourceId, cohortId, conceptIdsAndCohortPairs, breakdownConceptId, sortedConceptValues)
otherAttritionRows, err := u.GetAttritionRowForConceptDefsAndCohortPairs(sourceId, cohortId, conceptDefsAndCohortPairs, breakdownConceptId, sortedConceptValues)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept breakdown rows for filter conceptIds and cohortPairs", "error": err.Error()})
Expand All @@ -249,13 +252,13 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
c.String(http.StatusOK, b.String())
}

func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId int, cohortId int, conceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([][]string, error) {
func (u ConceptController) GetAttritionRowForConceptDefsAndCohortPairs(sourceId int, cohortId int, conceptDefsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([][]string, error) {
var otherAttritionRows [][]string
for idx, conceptIdOrCohortPair := range conceptIdsAndCohortPairs {
// attrition filter: run each query with an increasingly longer list of filterConceptIdsAndCohortPairs, until the last query is run with them all:
filterConceptIdsAndCohortPairs := conceptIdsAndCohortPairs[0 : idx+1]
for idx, conceptIdOrCohortPair := range conceptDefsAndCohortPairs {
// attrition filter: run each query with an increasingly longer list of filterConceptDefsAndCohortPairs, until the last query is run with them all:
filterConceptDefsAndCohortPairs := conceptDefsAndCohortPairs[0 : idx+1]

attritionRow, err := u.GetAttritionRowForConceptIdOrCohortPair(sourceId, cohortId, conceptIdOrCohortPair, filterConceptIdsAndCohortPairs, breakdownConceptId, sortedConceptValues)
attritionRow, err := u.GetAttritionRowForConceptDefOrCohortPair(sourceId, cohortId, conceptIdOrCohortPair, filterConceptDefsAndCohortPairs, breakdownConceptId, sortedConceptValues)
if err != nil {
log.Printf("Error: %s", err.Error())
return nil, err
Expand All @@ -265,10 +268,10 @@ func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId i
return otherAttritionRows, nil
}

func (u ConceptController) GetAttritionRowForConceptIdOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) {
filterConceptDefsAndValues, filterCohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
func (u ConceptController) GetAttritionRowForConceptDefOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptDefsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) {
filterConceptDefsAndValues, filterCohortPairs := utils.GetConceptDefsAndValuesAndCohortPairsAsSeparateLists(filterConceptDefsAndCohortPairs)
filterConceptIds := utils.ExtractConceptIdsFromCustomConceptVariablesDef(filterConceptDefsAndValues)
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId)
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptDefsPlusCohortPairs(sourceId, cohortId, filterConceptDefsAndCohortPairs, breakdownConceptId)
if err != nil {
return nil, fmt.Errorf("could not retrieve concept Breakdown for concepts %v dichotomous variables %v due to error: %s", filterConceptIds, filterCohortPairs, err.Error())
}
Expand Down
Loading

0 comments on commit 2c9901b

Please # to comment.