Fix formatting (etc) issue with multi-byte characters

A character that is multiple UTF-8 code units that got split across multiple 'data' events
was converted into replacement characters due to Buffer.toString being called on the chunks
individually. Instead collect all chunks and call toString after joining them.

purescript-node/purescript-node-streams#37

Author: nwolverson

src/IdePurescript/Build.purs

@@ -1,6 +1,7 @@
 module IdePurescript.Build where
 
 import Prelude
+
 import Control.Monad.Error.Class (throwError)
 import Data.Array (intercalate, uncons, (:))
 import Data.Array as Array
@@ -20,6 +21,8 @@ import Foreign.Object as Object
 import IdePurescript.Exec (findBins, getPathVar)
 import IdePurescript.PscErrors (PscResult(..), parsePscOutput)
 import IdePurescript.PscIdeServer (ErrorLevel(..), Notify)
+import Node.Buffer (Buffer)
+import Node.Buffer as Buffer
 import Node.ChildProcess (ChildProcess)
 import Node.ChildProcess as CP
 import Node.Encoding as Encoding
@@ -95,19 +98,19 @@ build logCb buildOptions@{ command: Command cmd args } = do
           Just cp -> do
             logCb Info $ "Running build command: " <> intercalate " " (cmd : args)
             CP.onError cp (cb <<< Left <<< CP.toStandardError)
-            errOutput <- Ref.new ""
-            outOutput <- Ref.new ""
+            errOutput <- Ref.new []
+            outOutput <- Ref.new []
             let
-              res :: Ref String -> String -> Effect Unit
-              res r s = Ref.modify_ (_ <> s) r
-            catchException err $ S.onDataString (CP.stderr cp) Encoding.UTF8 (res errOutput)
-            catchException err $ S.onDataString (CP.stdout cp) Encoding.UTF8 (res outOutput)
+              res :: Ref (Array Buffer) -> Buffer -> Effect Unit
+              res r s = Ref.modify_ (_ `Array.snoc` s) r
+            catchException err $ S.onData (CP.stderr cp) (res errOutput)
+            catchException err $ S.onData (CP.stdout cp) (res outOutput)
             CP.onClose cp
               ( \exit -> case exit of
                   CP.Normally n
                     | n == 0 || n == 1 -> do
-                      pursError <- Ref.read (errOutput)
-                      pursOutput <- Ref.read (outOutput)
+                      pursError <- Ref.read errOutput >>= Buffer.concat >>= Buffer.toString Encoding.UTF8
+                      pursOutput <- Ref.read outOutput >>= Buffer.concat >>= Buffer.toString Encoding.UTF8
                       let
                         lines = split (Pattern "\n") $ pursError <> pursOutput
                         { yes: json, no: toLog } = Array.partition (\s -> indexOf (Pattern "{\"") s == Just 0) lines

src/LanguageServer/IdePurescript/Formatting.purs

@@ -5,6 +5,7 @@ module LanguageServer.IdePurescript.Formatting
 
 import Prelude
 
+import Data.Array as Array
 import Data.Either (Either(..))
 import Data.Foldable (length)
 import Data.Maybe (Maybe(..))
@@ -25,6 +26,8 @@ import LanguageServer.Protocol.DocumentStore (getDocument)
 import LanguageServer.Protocol.Handlers (DocumentFormattingParams)
 import LanguageServer.Protocol.TextDocument (getText)
 import LanguageServer.Protocol.Types (DocumentStore, Position(..), Range(..), Settings, TextDocumentIdentifier(..), TextEdit(..))
+import Node.Buffer (Buffer)
+import Node.Buffer as Buffer
 import Node.ChildProcess as CP
 import Node.Encoding (Encoding(..))
 import Node.Encoding as Encoding
@@ -51,14 +54,14 @@ poseCommand = Command "prettier" [ "--parser", "purescript" ]
 
 -- TODO for NoFormatter don't provide formatting provider 
 format :: Notify -> Settings -> ServerState -> Formatter -> String -> Aff String
-format logCb settings state formatter text = do
+format _logCb settings state formatter text = do
   let
     command = case formatter of
       Purty -> purtyCommand
       PursTidy -> pursTidyCommand
       Pose -> poseCommand
       NoFormatter -> Command "echo" [] -- Not possible
-    Command cmd _ = command
+    Command _cmd _ = command
   case state of
     ServerState { root: Just directory } -> do
       makeAff
@@ -68,16 +71,17 @@ format logCb settings state formatter text = do
               err = cb <<< Left
             cp <- spawn { command, directory, useNpmDir: Config.addNpmPath settings }
             CP.onError cp (err <<< CP.toStandardError)
-            result <- Ref.new ""
+            result <- Ref.new []
+            
             let
-              res :: String -> Effect Unit
-              res s = Ref.modify_ (_ <> s) result
+              res :: Buffer -> Effect Unit
+              res s = Ref.modify_ (_ `Array.snoc` s) result
             catchException err $ S.onDataString (CP.stderr cp) Encoding.UTF8 $ err <<< error
-            catchException err $ S.onDataString (CP.stdout cp) Encoding.UTF8 res
+            catchException err $ S.onData (CP.stdout cp) res
             CP.onClose cp \exit -> case exit of
               CP.Normally n
-                | n == 0 || n == 1 ->
-                  Ref.read result >>= succ
+                | n == 0 || n == 1 -> 
+                  Ref.read result >>= Buffer.concat >>= Buffer.toString Encoding.UTF8 >>= succ
               _ -> do
                 let Command cmd _ = command
                 err $ error $ cmd <> " process exited abnormally"