Small bug fixes

AndMu · Sep 29, 2017 · 52795b3 · 52795b3
1 parent 26e9539
commit 52795b3
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 4 deletions.
diff --git a/src/Sentiment/Wikiled.Sentiment.Analysis/Processing/TextSplitterExtension.cs b/src/Sentiment/Wikiled.Sentiment.Analysis/Processing/TextSplitterExtension.cs
@@ -3,6 +3,7 @@
 using System.IO;
 using System.Reactive.Disposables;
 using System.Reactive.Linq;
+using NLog;
 using Wikiled.Core.Utility.Serialization;
 using Wikiled.Sentiment.Text.Data.Review;
 using Wikiled.Sentiment.Text.Parser;
@@ -12,8 +13,16 @@ namespace Wikiled.Sentiment.Analysis.Processing
 {
     public static class TextSplitterExtension
     {
+        private static readonly Logger log = LogManager.GetCurrentClassLogger();
+
         public static IObservable<IParsedDocumentHolder> GetParsedReviewHolders(this ITextSplitter splitter, string path, bool? positive)
         {
+            if (string.IsNullOrEmpty(path))
+            {
+                log.Warn("One of paths is empty");
+                return Observable.Empty<IParsedDocumentHolder>();
+            }
+
             return Observable.Create<IParsedDocumentHolder>(
                 observer =>
                 {
@@ -68,7 +77,6 @@ public static IObservable<IParsedDocumentHolder> GetParsedReviewHolders(this ITe
 
         private static IEnumerable<Document> GetReview(string path)
         {
-            var dirName = Path.GetDirectoryName(path);
             if (File.Exists(path))
             {
                 foreach (var line in File.ReadLines(path))

diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Machine/BaseBoostrapCommand.cs b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Machine/BaseBoostrapCommand.cs
@@ -228,7 +228,7 @@ private IEnumerable<EvalData> ReadFiles()
 
         private IEnumerable<EvalData> ReadFile(string file)
         {
-            foreach (var semEvalData in GetDataPacket(file))
+            foreach (var semEvalData in GetDataPacket(file).Where(item => item != null))
             {
                 monitor.ManualyCount();
                 yield return semEvalData;

diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Machine/BaseRawCommand.cs b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Machine/BaseRawCommand.cs
@@ -117,7 +117,7 @@ private IObservable<IParsedDocumentHolder> GetNegativeReviews()
         private IObservable<IParsedDocumentHolder> GetOtherReviews()
         {
             log.Info("Other {0}", Positive);
-            return splitter.Splitter.GetParsedReviewHolders(Positive, null);
+            return splitter.Splitter.GetParsedReviewHolders(Input, null);
         }
     }
 }
diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Machine/ImdbBoostrapCommand.cs b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Machine/ImdbBoostrapCommand.cs
@@ -22,6 +22,11 @@ protected override IEnumerable<EvalData> GetDataPacket(string path)
             PositivityType? positivity = null;
             var id = System.IO.Path.GetFileNameWithoutExtension(path);
             var text = File.ReadAllText(path);
+            if (string.IsNullOrEmpty(text))
+            {
+                yield break;
+            }
+
             if (path.Contains(@"\pos"))
             {
                 positivity = PositivityType.Positive;
@@ -31,6 +36,7 @@ protected override IEnumerable<EvalData> GetDataPacket(string path)
                 positivity = PositivityType.Negative;
             }
 
+
             yield return new EvalData(id + $"_{positivity}", positivity, text);
         }
 

diff --git a/src/Utilities/Wikiled.Sentiment.ConsoleApp/Machine/SingleBoostrapCommand.cs b/src/Utilities/Wikiled.Sentiment.ConsoleApp/Machine/SingleBoostrapCommand.cs
@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using Wikiled.Sentiment.ConsoleApp.Machine.Data;
 
 namespace Wikiled.Sentiment.ConsoleApp.Machine
@@ -14,7 +15,7 @@ public class SingleBoostrapCommand : ImdbBoostrapCommand
         protected override IEnumerable<EvalData> GetDataPacket(string path)
         {
             path = path.ToLower();
-            foreach (var line in File.ReadLines(path))
+            foreach (var line in File.ReadLines(path).Where(item => !string.IsNullOrWhiteSpace(item)))
             {
                 id++;
                 yield return new EvalData(id.ToString(), null, line);