new version with check for illegal characters and a selection window …

…for tmx language pair, if no matching language is found
Helsinki-NLP · Jul 6, 2021 · d8258b9 · d8258b9
1 parent fb0e2db
commit d8258b9
Show file tree

Hide file tree

Showing 17 changed files with 302 additions and 19 deletions.
diff --git a/OpusCatEngineAndPlugins.sln b/OpusCatEngineAndPlugins.sln
@@ -31,6 +31,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SettingsUiTester", "Setting
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpusCatTradosPluginTests", "OpusCatTradosPluginTests\OpusCatTradosPluginTests.csproj", "{7835C453-B4A0-45C4-BDB8-7888D26561FC}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpusCatMTEngineTests", "OpusCatMTEngineTests\OpusCatMTEngineTests.csproj", "{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -173,6 +175,18 @@ Global
 		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x64.Build.0 = Release|Any CPU
 		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x86.ActiveCfg = Release|Any CPU
 		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x86.Build.0 = Release|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|x64.Build.0 = Debug|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|x86.Build.0 = Debug|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|x64.ActiveCfg = Release|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|x64.Build.0 = Release|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|x86.ActiveCfg = Release|Any CPU
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -189,6 +203,7 @@ Global
 		{72114486-2563-4AF6-B40C-515A21B04754} = {7A01F5CB-2378-473E-AF22-69ED318A6235}
 		{23EB17A2-9A46-4D3E-A471-1952430CDA26} = {7A01F5CB-2378-473E-AF22-69ED318A6235}
 		{7835C453-B4A0-45C4-BDB8-7888D26561FC} = {7A01F5CB-2378-473E-AF22-69ED318A6235}
+		{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364} = {05D6D17C-DFCC-4E76-AE07-0F22FD38F3B7}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {8F143A32-2D24-488F-AD4C-CD6D074641C6}

diff --git a/OpusCatMTEngine/HelperFunctions.cs b/OpusCatMTEngine/HelperFunctions.cs
@@ -10,7 +10,7 @@
 
 namespace OpusCatMTEngine
 {
-    class HelperFunctions
+    public class HelperFunctions
     {
         public static string EnumToString(object value)
         {

diff --git a/OpusCatMTEngine/MTModel.cs b/OpusCatMTEngine/MTModel.cs
@@ -172,8 +172,15 @@ internal void Shutdown()
             this.marianProcesses = null;
         }
 
-        internal Task<TranslationPair> Translate(string input, IsoLanguage sourceLang, IsoLanguage targetLang)
+        public Task<TranslationPair> Translate(string input, IsoLanguage sourceLang, IsoLanguage targetLang)
         {
+            if (String.IsNullOrWhiteSpace(input))
+            {
+                var translationPair = new TranslationPair(input, input, input, "0-0", this.ModelSegmentationMethod, targetLang.OriginalCode);
+                return Task.FromResult<TranslationPair>(translationPair);
+
+            }
+
             //Need to get the original codes, since those are the ones the marian model uses
             var modelSourceLang =
                 this.SourceLanguages.SingleOrDefault(x => x.ShortestIsoCode == sourceLang.ShortestIsoCode);

diff --git a/OpusCatMTEngine/MTService.cs b/OpusCatMTEngine/MTService.cs
@@ -229,6 +229,23 @@ public int[] BatchStoreTranslation(string tokenCode, List<string> sources, List<
         }
 
         public string Customize(
+            string tokenCode,
+            List<Tuple<string, string>> input,
+            List<Tuple<string, string>> validation,
+            List<string> uniqueNewSegments,
+            string srcLangCode,
+            string trgLangCode,
+            string modelTag,
+            bool includePlaceholderTags,
+            bool includeTagPairs)
+        {
+            List<ParallelSentence> input2 = input.Select(x => new ParallelSentence(x.Item1, x.Item2)).ToList();
+            List<ParallelSentence> validation2 = validation.Select(x => new ParallelSentence(x.Item1, x.Item2)).ToList();
+
+            return this.Customize(tokenCode, input2, validation2, uniqueNewSegments, srcLangCode, trgLangCode, modelTag, includePlaceholderTags, includeTagPairs);
+        }
+
+            public string Customize(
             string tokenCode,
             List<ParallelSentence> input,
             List<ParallelSentence> validation,

diff --git a/OpusCatMTEngine/ModelManager.cs b/OpusCatMTEngine/ModelManager.cs
@@ -872,6 +872,7 @@ public string ConvertIsoCode(string name)
 
         public Task<TranslationPair> Translate(string input, IsoLanguage srcLang, IsoLanguage trgLang, string modelTag)
         {
+
             var mtModel = this.SelectModel(srcLang, trgLang, modelTag);
 
             if (mtModel == null)

diff --git a/OpusCatMTEngine/OWIN/MachineTranslationController.cs b/OpusCatMTEngine/OWIN/MachineTranslationController.cs
@@ -107,13 +107,19 @@ public Translation TranslatePost(string tokenCode = "", string input = "", strin
         [HttpGet]
         public HttpResponseMessage TranslateJson(string tokenCode="", string input="", string srcLangCode="", string trgLangCode="", string modelTag="")
         {
+            if (input == null)
+            {
+                input = "";
+            }
             //HttpContext.Current.Response.Headers.Add("Access-Control-Allow-Origin", "*");
             var sourceLang = new IsoLanguage(srcLangCode);
             var targetLang = new IsoLanguage(trgLangCode);
 
             var translation = this.mtProvider.Translate(input, sourceLang, targetLang, modelTag);
 
-            var response = Request.CreateResponse<TranslationPair>(HttpStatusCode.OK, translation.Result);
+
+            var translationResult = translation.Result;
+            var response = Request.CreateResponse<TranslationPair>(HttpStatusCode.OK, translationResult);
             response.Headers.Add("Access-Control-Allow-Origin", "*");
 
 

diff --git a/OpusCatMTEngine/Properties/AssemblyInfo.cs b/OpusCatMTEngine/Properties/AssemblyInfo.cs
@@ -51,5 +51,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.1.0.5")]
-[assembly: AssemblyFileVersion("1.1.0.5")]
+[assembly: AssemblyVersion("1.1.0.6")]
+[assembly: AssemblyFileVersion("1.1.0.6")]
diff --git a/OpusCatMTEngine/Properties/Resources.Designer.cs b/OpusCatMTEngine/Properties/Resources.Designer.cs
diff --git a/OpusCatMTEngine/Properties/Resources.resx b/OpusCatMTEngine/Properties/Resources.resx
@@ -371,4 +371,10 @@
   <data name="Main_NonAsciiPathMessage" xml:space="preserve">
     <value>The path where OPUS-CAT MT Engine stores MT models ({0}) contains the following illegal characters: {1}. Install OPUS-CAT MT Engine to a path with no illegal characters in it (for instance, in a folder in the root of the C: drive, such as C:\opuscat).</value>
   </data>
+  <data name="SelectTmxLangPair_Instructions" xml:space="preserve">
+    <value>The TMX file does not contain languages compatible with the language codes of the MT model, or there are not enough segments for the compatible language pair. Language pairs included in the TMX file and the amount of segments per language pair are listed below. You can either select one of the language pairs to extract as fine-tuning material, or cancel fine-tuning.</value>
+  </data>
+  <data name="SelectTmxLangPair_Title" xml:space="preserve">
+    <value>Select language pair</value>
+  </data>
 </root>
diff --git a/OpusCatMTEngine/UI/ModelCustomizerView.xaml.cs b/OpusCatMTEngine/UI/ModelCustomizerView.xaml.cs
@@ -243,12 +243,17 @@ private void customize_Click(object sender, RoutedEventArgs e)
                                     this.IncludeTagPairBox.IsChecked.Value
                                 );
                             }
+                            else
+                            {
+                                return;
+                            }
                         }
                         else
                         {
                             MessageBox.Show(
                                 String.Format(
                                     OpusCatMTEngine.Properties.Resources.Finetune_NotEnoughSegmentsInTmx));
+                            return;
                         }
                     }
                     break;

diff --git a/OpusCatMTEngine/UI/SelectTmxLangPairWindow.xaml b/OpusCatMTEngine/UI/SelectTmxLangPairWindow.xaml
@@ -3,13 +3,14 @@
         xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
         xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
         xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+        xmlns:resx="clr-namespace:OpusCatMTEngine.Properties"
         xmlns:local="clr-namespace:OpusCatMTEngine"
         mc:Ignorable="d"
-        Title="SelectTmxLangPair" Height="400" Width="400">
+        Title="{x:Static resx:Resources.SelectTmxLangPair_Title}" Height="400" Width="400">
     <Border BorderBrush="Black" BorderThickness="1" Margin="3">
         <Grid Margin="5">
         <Grid.ColumnDefinitions>
-            <ColumnDefinition Width="1*"/>
+            <ColumnDefinition Width="1.5*"/>
             <ColumnDefinition Width="1*"/>
             <ColumnDefinition Width="1*"/>
         </Grid.ColumnDefinitions>
@@ -18,15 +19,17 @@
             <RowDefinition Height="10*"/>
             <RowDefinition Height="1*"/>
         </Grid.RowDefinitions>
-        <TextBlock TextWrapping="Wrap" Grid.Column="0" Grid.ColumnSpan="3" Grid.Row="0" Grid.RowSpan="1">
-            The TMX file does not contain languages compatible with the language codes of the
-            MT model, or there is not enough segments for the compatible language pair. Language pairs
-            included in the TMX file and the amount of segments per language pair is listed below.
-            You can either select one language pair to extract as fine-tuning material, or cancel
-            fine-tuning.
-        </TextBlock>
+            <TextBlock 
+                Text="{x:Static resx:Resources.SelectTmxLangPair_Instructions}" 
+                TextWrapping="Wrap" 
+                Grid.Column="0" 
+                Grid.ColumnSpan="3" 
+                Grid.Row="0" 
+                Grid.RowSpan="1"/>
+
+
 
-        <ListView 
+            <ListView 
                 Height="320" 
                 Grid.Column="0" 
                 Grid.ColumnSpan="3"

diff --git a/OpusCatMTEngineTests/App.config b/OpusCatMTEngineTests/App.config
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <configSections>
+    <!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
+    <section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
+  </configSections>
+  <entityFramework>
+    <providers>
+      <provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
+      <provider invariantName="System.Data.SQLite.EF6" type="System.Data.SQLite.EF6.SQLiteProviderServices, System.Data.SQLite.EF6" />
+    </providers>
+  </entityFramework>
+  <system.data>
+    <DbProviderFactories>
+      <remove invariant="System.Data.SQLite.EF6" />
+      <add name="SQLite Data Provider (Entity Framework 6)" invariant="System.Data.SQLite.EF6" description=".NET Framework Data Provider for SQLite (Entity Framework 6)" type="System.Data.SQLite.EF6.SQLiteProviderFactory, System.Data.SQLite.EF6" />
+    <remove invariant="System.Data.SQLite" /><add name="SQLite Data Provider" invariant="System.Data.SQLite" description=".NET Framework Data Provider for SQLite" type="System.Data.SQLite.SQLiteFactory, System.Data.SQLite" /></DbProviderFactories>
+  </system.data>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.Owin" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.1.1.0" newVersion="4.1.1.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Autofac" publicKeyToken="17863af14b0044da" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-6.1.0.0" newVersion="6.1.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Web.Http" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-5.2.7.0" newVersion="5.2.7.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Net.Http.Formatting" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-5.2.7.0" newVersion="5.2.7.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Autofac.Integration.WebApi" publicKeyToken="17863af14b0044da" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-6.0.1.0" newVersion="6.0.1.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Autofac.Integration.Owin" publicKeyToken="17863af14b0044da" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-6.0.1.0" newVersion="6.0.1.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Memory" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.0.1.1" newVersion="4.0.1.1" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
+</configuration>
diff --git a/OpusCatMTEngineTests/FinetuneTest.cs b/OpusCatMTEngineTests/FinetuneTest.cs
@@ -0,0 +1,26 @@
+using System;
+using System.Linq;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace OpusCatMTEngineTests
+{
+    [TestClass]
+    public class FinetuneTest
+    {
+        /// <summary>
+        /// This is not functional, there should be tests for most common
+        /// functionalities, but no time to implement right now. Use this as
+        /// starting point if time is available.
+        /// </summary>
+        [TestMethod]
+        public void TestFinetune()
+        {
+            var modelInstallPath = OpusCatMTEngine.HelperFunctions.GetLocalAppDataPath("models\\eng-fin\\opus+bt-2021-03-09");
+            var model = new OpusCatMTEngine.MTModel("eng-fin\\opus+bt-2021-03-09", modelInstallPath);
+            var testTranslation = model.Translate("this is a test", model.SourceLanguages.First(), model.TargetLanguages.First());
+            var result = testTranslation.Result;
+            var finetuneWindow = new OpusCatMTEngine.ModelCustomizerView(model);
+
+        }
+    }
+}
-Original file line number
+Diff line change
@@ Expand Up / @@ -872,6 +872,7 @@ public string ConvertIsoCode(string name) @@
             public Task<TranslationPair> Translate(string input, IsoLanguage srcLang, IsoLanguage trgLang, string modelTag)
             {
                 var mtModel = this.SelectModel(srcLang, trgLang, modelTag);
                 if (mtModel == null)
@@ Expand Down @@