From 43de15a9ae563e697e1a6d2a660537ad738573d2 Mon Sep 17 00:00:00 2001
From: vkhristenko <vdkhristenko1991@gmail.com>
Date: Fri, 20 Oct 2017 13:15:37 +0200
Subject: [PATCH] addign options handling and hard null type remove pass

---
 .../sparkroot/experimental/core/ast.scala     |  1 -
 .../experimental/core/optimizations.scala     | 32 ++++++++++++++++---
 .../sparkroot/experimental/package.scala      |  1 +
 3 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/src/main/scala/org/dianahep/sparkroot/experimental/core/ast.scala b/src/main/scala/org/dianahep/sparkroot/experimental/core/ast.scala
index 2072eea..d144052 100644
--- a/src/main/scala/org/dianahep/sparkroot/experimental/core/ast.scala
+++ b/src/main/scala/org/dianahep/sparkroot/experimental/core/ast.scala
@@ -189,7 +189,6 @@ package object core
 
       // when you have an array of something simple kOffsetL by ROOT convention  
       def iterateArray(dimsToGo: Int): core.SRType = {
-        logger.info(s"dimsToGo = ${dimsToGo} for name = ${streamerElement.getName} type = ${streamerElement.getType} typeName = ${streamerElement.getTypeName}")
         if (dimsToGo==1) core.SRArray(streamerElement.getName, b,
           if (b==null) null
           else b.getLeaves.get(0).asInstanceOf[TLeafElement], 
diff --git a/src/main/scala/org/dianahep/sparkroot/experimental/core/optimizations.scala b/src/main/scala/org/dianahep/sparkroot/experimental/core/optimizations.scala
index 7eceb9e..67173ce 100644
--- a/src/main/scala/org/dianahep/sparkroot/experimental/core/optimizations.scala
+++ b/src/main/scala/org/dianahep/sparkroot/experimental/core/optimizations.scala
@@ -24,12 +24,13 @@ package object optimizations {
   trait OptimizationPass {
     def run(x: SRRoot, roptions: ROptions): SRRoot
 
-    val name: String = this.getClass.getSimpleName.filterNot(_=='$')
+    val name: String = this.getClass.getSimpleName.filterNot(_=='$').toLowerCase
 
     val default = false
 
     def shouldRun(roptions: ROptions): Boolean = roptions.get(name) match {
-      case Some(x) => 
+      case Some(x) => {
+        logger.info(s"Optimization ${name} is ${x}")
         // if the option is provided use that!
         if (x.toLowerCase=="off" || x.toLowerCase=="false") 
           false
@@ -37,18 +38,24 @@ package object optimizations {
           true
         else 
           throw UnknownOptionException(x)
-      case None => default
+      }
+      case None => { logger.info(s"Optimization ${name} is defaulted to ${default}"); default }
     }
   }
 
   case object HardRemoveNullTypePass extends OptimizationPass {
+    override val default = true
     private def iterate(t: SRType): SRType = t match {
       // type t occupies a branch that either splittable or that is not splittable
       // but does not contain null
       case x: SRComposite => 
         if (x.split)
           SRComposite(x.name, x.b, x.members.filterNot({case m =>
-            occupiesNonSplittableBranchWithNull(m)}).map(iterate(_)), x.split, x.isTop, x.isBase, x._shouldDrop)
+            // filter out:
+            // 1. members that are null
+            // 2. members that are colelction(recursive of null)
+            // 3. members that occupy a non-splitted branch with a null as a field
+            occupiesNonSplittableBranchWithNull(m) || isNull(m) || collectionWithNull(m)}).map(iterate(_)), x.split, x.isTop, x.isBase, x._shouldDrop)
         else
           x
       case x: SRVector => 
@@ -59,6 +66,21 @@ package object optimizations {
           x
       case _ => t
     }
+    private def isNull(t: SRType): Boolean = t match {
+      case x: SRNull => true
+      case x: SRUnknown => true
+      case _ => false
+    }
+    private def collectionWithNull(t: SRType): Boolean = t match {
+      case x: SRVector => collectionWithNull(x.t)
+      case x: SRMap => collectionWithNull(x.valueType) || collectionWithNull(x.keyType)
+      case x: SRMultiMap => collectionWithNull(x.valueType) || 
+        collectionWithNull(x.keyType)
+      case x: SRArray => collectionWithNull(x.t)
+      case x: SRNull => true
+      case x: SRUnknown => true
+      case _ => false
+    }
 
     private def containsNull(t: SRType): Boolean = t match {
       // check if valid type t contains null inside
@@ -104,7 +126,7 @@ package object optimizations {
   // to the members down the line. 
   //  (nested as well) e.g. array<array<array<NULL>>>
   case object SoftRemoveNullTypePass extends OptimizationPass {
-    override val default = true
+    override val default = false
 
     private def notNull(t: SRType): Boolean =
       !(t.isInstanceOf[SRNull] || t.isInstanceOf[SRUnknown])
diff --git a/src/main/scala/org/dianahep/sparkroot/experimental/package.scala b/src/main/scala/org/dianahep/sparkroot/experimental/package.scala
index d7adfbf..fa82d02 100644
--- a/src/main/scala/org/dianahep/sparkroot/experimental/package.scala
+++ b/src/main/scala/org/dianahep/sparkroot/experimental/package.scala
@@ -124,6 +124,7 @@ package experimental {
       val treeName = roptions.get("tree")
 
       // some logging
+      logger.info(s"options: ${options}")
       logger.info(s"Building the Abstractly Typed Tree... for treeName=$treeName")
       files.map(_.getPath.toString).foreach({x: String => logger.info(s"pathname = $x")})