Added target language selection for multilingual override models. Sta…

…rted migrating Trados plugin to use HTTP API.
Helsinki-NLP · Jun 30, 2021 · eadb974 · eadb974
1 parent 7d584e9
commit eadb974
Show file tree

Hide file tree

Showing 16 changed files with 366 additions and 24 deletions.
diff --git a/OpusCatEngineAndPlugins.sln b/OpusCatEngineAndPlugins.sln
@@ -29,6 +29,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpusCatTranslationProvider2
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SettingsUiTester", "SettingsUiTester\SettingsUiTester.csproj", "{23EB17A2-9A46-4D3E-A471-1952430CDA26}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpusCatTradosPluginTests", "OpusCatTradosPluginTests\OpusCatTradosPluginTests.csproj", "{7835C453-B4A0-45C4-BDB8-7888D26561FC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -159,6 +161,18 @@ Global
 		{23EB17A2-9A46-4D3E-A471-1952430CDA26}.Release|x64.Build.0 = Release|Any CPU
 		{23EB17A2-9A46-4D3E-A471-1952430CDA26}.Release|x86.ActiveCfg = Release|Any CPU
 		{23EB17A2-9A46-4D3E-A471-1952430CDA26}.Release|x86.Build.0 = Release|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Debug|x64.Build.0 = Debug|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Debug|x86.Build.0 = Debug|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x64.ActiveCfg = Release|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x64.Build.0 = Release|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x86.ActiveCfg = Release|Any CPU
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -174,6 +188,7 @@ Global
 		{B92960AA-D96E-464D-BF62-76BA6D440995} = {7A01F5CB-2378-473E-AF22-69ED318A6235}
 		{72114486-2563-4AF6-B40C-515A21B04754} = {7A01F5CB-2378-473E-AF22-69ED318A6235}
 		{23EB17A2-9A46-4D3E-A471-1952430CDA26} = {7A01F5CB-2378-473E-AF22-69ED318A6235}
+		{7835C453-B4A0-45C4-BDB8-7888D26561FC} = {7A01F5CB-2378-473E-AF22-69ED318A6235}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {8F143A32-2D24-488F-AD4C-CD6D074641C6}

diff --git a/OpusCatMTEngine/ModelManager.cs b/OpusCatMTEngine/ModelManager.cs
@@ -169,7 +169,7 @@ public bool ShowTatoebaModels
                 this.FilterOnlineModels();
             }
         }
-        
+
         public string SourceFilter
         {
             get => _sourceFilter;
@@ -202,7 +202,7 @@ public string NameFilter
                 this.FilterOnlineModels();
             }
         }
-        
+
         public bool OnlineModelListFetched
         {
             get => onlineModelListFetched;
@@ -213,7 +213,18 @@ public bool OnlineModelListFetched
             }
         }
 
+        public IsoLanguage OverrideModelTargetLanguage
+        {
+            get => _overrideModelTargetLanguage;
+            set
+            {
+                _overrideModelTargetLanguage = value;
+                NotifyPropertyChanged();
+            }
+        }
+
         private bool onlineModelListFetched;
+        private IsoLanguage _overrideModelTargetLanguage;
 
         public string CheckModelStatus(IsoLanguage sourceLang, IsoLanguage targetLang, string modelTag)
         {
@@ -320,7 +331,7 @@ internal void FilterOnlineModels()
 
             if (this.ShowNewestOnly)
             {
-                var onlyNew = new Dictionary<string,MTModel>();
+                var onlyNew = new Dictionary<string, MTModel>();
                 foreach (var model in filteredModels)
                 {
                     if (onlyNew.ContainsKey(model.ModelBaseName))
@@ -389,7 +400,7 @@ public ModelManager()
             this.ShowOpusModels = true;
             this.ShowTatoebaModels = true;
             this.ShowNewestOnly = true;
-            
+
             var modelDirPath = HelperFunctions.GetOpusCatDataPath(OpusCatMTEngineSettings.Default.ModelDir);
             this.opusModelDir = new DirectoryInfo(modelDirPath);
             if (!this.OpusModelDir.Exists)
@@ -600,15 +611,15 @@ private void AddOnlineStorageModels(List<string> filePaths, Uri storageUri)
         }
 
         private void modelYamlDownloadComplete(Uri modelUri, string model, object sender, DownloadStringCompletedEventArgs e)
-        {           
+        {
             var yamlString = e.Result;
             yamlString = Regex.Replace(yamlString, "- (>>[^<]+<<)", "- \"$1\"");
             yamlString = Regex.Replace(yamlString, "(?<!- )'(>>[^<]+<<)'", "- \"$1\"");
             if (Regex.Match(yamlString, @"(?<!- )devset = top").Success)
             {
                 Log.Information($"Corrupt yaml line in model {model} yaml file, applying fix");
                 yamlString = Regex.Replace(yamlString, @"(?<!- )devset = top", "devset: top");
-                
+
             }
             if (yamlString.Contains("unused dev/test data is added to training data"))
             {
@@ -867,6 +878,14 @@ public Task<TranslationPair> Translate(string input, IsoLanguage srcLang, IsoLan
                 throw new FaultException($"No MT model available for {srcLang}-{trgLang}");
             }
 
+            //If override model has been selected, switch target language to the
+            //target language chosen in the UI (this makes it possible to use multilingual
+            //models as override models).
+            if (this.OverrideModel != null)
+            {
+                trgLang = this.OverrideModelTargetLanguage;
+            }
+
             var translationTask = mtModel.Translate(input, srcLang, trgLang);
 
             if (App.Overlay != null)

diff --git a/OpusCatMTEngine/OWIN/MachineTranslationController.cs b/OpusCatMTEngine/OWIN/MachineTranslationController.cs
@@ -185,15 +185,15 @@ public List<string> BatchTranslate(string tokenCode, List<string> input, string
         /// <param name="trgLangCode"></param>
         /// 
         [HttpPost]
-        public string PreOrderBatch(string tokenCode, List<string> input, string srcLangCode, string trgLangCode, string modelTag)
+        public string PreOrderBatch([FromBody] List<string> input, string tokenCode="", string srcLangCode="", string trgLangCode="", string modelTag="")
         {
-
+            
             if (!TokenCodeGenerator.Instance.TokenCodeIsValid(tokenCode))
                 return "";
 
             var sourceLang = new IsoLanguage(srcLangCode);
             var targetLang = new IsoLanguage(trgLangCode);
-
+            
             if (input.Count == 0)
             {
                 return "input was empty";
@@ -203,7 +203,7 @@ public string PreOrderBatch(string tokenCode, List<string> input, string srcLang
             {
                 this.mtProvider.Translate(inputString, sourceLang, targetLang, modelTag);
             }
-
+            
             /* Batch preordering was done earlier with batch translation, but it doesn't seem
              * to be much quicker than normal translation, and it has the problem of providing all
              * the translations at once in the end. Using normal translation means the MT is ready

diff --git a/OpusCatMTEngine/UI/LocalModelListView.xaml b/OpusCatMTEngine/UI/LocalModelListView.xaml
@@ -24,6 +24,21 @@
                             <Setter Property="Margin" Value="0,0,0,5" />
                         </Style>
                     </StackPanel.Resources>
+                    <Border
+                        BorderBrush="Black" BorderThickness="1"
+                        Visibility="{Binding OverrideModelSet, Converter={StaticResource VisibleIfTrueConverter}, FallbackValue=Collapsed}">
+                        <StackPanel Margin="5">
+                            <Label>Override model target language</Label>
+                        <ComboBox ItemsSource="{Binding Path=OverrideModel.TargetLanguages, Mode=TwoWay}"
+                            DisplayMemberPath="IsoRefName"
+                            SelectedValue="{Binding OverrideModelTargetLanguage, Mode=TwoWay}"
+                            Width="200"/>
+                            <Button 
+                        Name="cancelOverride"
+                        Click="btnCancelOverride_Click"
+                        Content="{x:Static resx:Resources.Main_CancelOverrideButton}"/>
+                        </StackPanel>
+                    </Border>
                     <Button Name="addOnlineModel" Click="btnAddOnlineModel_Click" Content="{x:Static resx:Resources.Main_InstallFromWebButton}" />
                     <Button Name="addZipModel" Click="btnAddZipModel_Click" Content="{x:Static resx:Resources.Main_InstallFromZipButton}" />
                     <Button Name="customizeModel" Click="btnCustomizeModel_Click" Content="{x:Static resx:Resources.Main_FinetuneButton}">
@@ -113,11 +128,6 @@
                         Name="setAsOverride"
                         Click="btnSetOverride_Click"
                         Content="{x:Static resx:Resources.Main_SetAsOverrideModelButton}"/>
-                    <Button 
-                        Visibility="{Binding OverrideModelSet, Converter={StaticResource VisibleIfTrueConverter}, FallbackValue=Collapsed}"
-                        Name="cancelOverride"
-                        Click="btnCancelOverride_Click"
-                        Content="{x:Static resx:Resources.Main_CancelOverrideButton}"/>
                 </StackPanel>
                 <ListView 
                     Name="LocalModelList" 

diff --git a/OpusCatMTEngine/UI/LocalModelListView.xaml.cs b/OpusCatMTEngine/UI/LocalModelListView.xaml.cs
@@ -85,6 +85,7 @@ private void btnSetOverride_Click(object sender, RoutedEventArgs e)
             }
 
             ((ModelManager)this.DataContext).OverrideModel = selectedModel;
+            ((ModelManager)this.DataContext).OverrideModelTargetLanguage = selectedModel.TargetLanguages.First();
             ((ModelManager)this.DataContext).MoveOverrideToTop();
         }
 

diff --git a/OpusCatMTEngine/UI/ModelCustomizerView.xaml b/OpusCatMTEngine/UI/ModelCustomizerView.xaml
@@ -7,7 +7,7 @@
         xmlns:resx="clr-namespace:OpusCatMTEngine.Properties"
         mc:Ignorable="d"
         x:Name="_this"
-        Height="500" Width="700">
+        Height="400" Width="700">
     <UserControl.Resources>
         <BooleanToVisibilityConverter x:Key="BoolToVis"/>
         <local:ValidationErrorConverter x:Key="ValidationErrorConverter" />
@@ -46,7 +46,8 @@
             </Style.Triggers>
         </Style>
     </UserControl.Resources>
-    <DockPanel Margin="5">
+    <ScrollViewer>
+    <StackPanel Margin="5">
         <StackPanel DockPanel.Dock="Top">
             <Border Style="{StaticResource BorderStyle}" BorderBrush="Black" BorderThickness="1">
                 <StackPanel>
@@ -171,7 +172,7 @@
                 <TextBox IsEnabled="{Binding ElementName=_this,Path=CustomizationNotStarted, UpdateSourceTrigger=PropertyChanged}" Style="{StaticResource ValidatedTextBoxStyle}" Name="LabelBox" Text="{Binding ElementName=_this,Path=ModelTag, UpdateSourceTrigger=PropertyChanged, NotifyOnValidationError=true, ValidatesOnDataErrors=True, Mode=TwoWay}" Width="200"></TextBox>
             </StackPanel>
         </StackPanel>
-        <DockPanel DockPanel.Dock="Bottom" LastChildFill="False">
+        <DockPanel DockPanel.Dock="Right" LastChildFill="False" Height="50">
             <!--<Button DockPanel.Dock="Right" Margin="5" Height="30" Width="100" Command="ApplicationCommands.Close">Cancel</Button>-->
             <Button 
                 Name="CustomizeButton" 
@@ -196,5 +197,6 @@
                 </Button.IsEnabled>
             </Button>
         </DockPanel>
-    </DockPanel>
+        </StackPanel>
+    </ScrollViewer>
 </UserControl>
diff --git a/OpusCatTradosPluginTests/OpusCatConnectionTest.cs b/OpusCatTradosPluginTests/OpusCatConnectionTest.cs
@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using OpusCatTranslationProvider;
+
+namespace OpusCatTradosPluginTests
+{
+    [TestClass]
+    public class OpusCatConnectionTest
+    {
+        [TestMethod]
+        public void PreOrderTest()
+        {
+            var connection = new OpusCatMtServiceConnection();
+
+            var preOrderStatus = connection.PreOrderBatch(
+                "localhost", "8500",
+                new List<string>() { "This is a test" }, "en", "fi", "");
+
+            Assert.AreEqual("OK", preOrderStatus);
+        }
+    }
+}
diff --git a/OpusCatTradosPluginTests/OpusCatTradosPluginTests.csproj b/OpusCatTradosPluginTests/OpusCatTradosPluginTests.csproj
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="15.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\packages\MSTest.TestAdapter.1.3.2\build\net45\MSTest.TestAdapter.props" Condition="Exists('..\packages\MSTest.TestAdapter.1.3.2\build\net45\MSTest.TestAdapter.props')" />
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{7835C453-B4A0-45C4-BDB8-7888D26561FC}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>OpusCatTradosPluginTests</RootNamespace>
+    <AssemblyName>OpusCatTradosPluginTests</AssemblyName>
+    <TargetFrameworkVersion>v4.7.2</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <ProjectTypeGuids>{3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">15.0</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+    <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages</ReferencePath>
+    <IsCodedUITest>False</IsCodedUITest>
+    <TestProjectType>UnitTest</TestProjectType>
+    <NuGetPackageImportStamp>
+    </NuGetPackageImportStamp>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="Microsoft.VisualStudio.TestPlatform.TestFramework, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\MSTest.TestFramework.1.3.2\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\MSTest.TestFramework.1.3.2\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="OpusCatConnectionTest.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="packages.config" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Trados2019Plugin\OpusCatTranslationProvider.csproj">
+      <Project>{2a35ccb8-0405-4bdf-ba7b-7b9e87bc4d60}</Project>
+      <Name>OpusCatTranslationProvider</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\packages\MSTest.TestAdapter.1.3.2\build\net45\MSTest.TestAdapter.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\MSTest.TestAdapter.1.3.2\build\net45\MSTest.TestAdapter.props'))" />
+    <Error Condition="!Exists('..\packages\MSTest.TestAdapter.1.3.2\build\net45\MSTest.TestAdapter.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\MSTest.TestAdapter.1.3.2\build\net45\MSTest.TestAdapter.targets'))" />
+  </Target>
+  <Import Project="..\packages\MSTest.TestAdapter.1.3.2\build\net45\MSTest.TestAdapter.targets" Condition="Exists('..\packages\MSTest.TestAdapter.1.3.2\build\net45\MSTest.TestAdapter.targets')" />
+</Project>
diff --git a/OpusCatTradosPluginTests/Properties/AssemblyInfo.cs b/OpusCatTradosPluginTests/Properties/AssemblyInfo.cs
@@ -0,0 +1,20 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+[assembly: AssemblyTitle("OpusCatTradosPluginTests")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("OpusCatTradosPluginTests")]
+[assembly: AssemblyCopyright("Copyright ©  2021")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+[assembly: ComVisible(false)]
+
+[assembly: Guid("7835c453-b4a0-45c4-bdb8-7888d26561fc")]
+
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/OpusCatTradosPluginTests/packages.config b/OpusCatTradosPluginTests/packages.config
@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="MSTest.TestAdapter" version="1.3.2" targetFramework="net472" />
+  <package id="MSTest.TestFramework" version="1.3.2" targetFramework="net472" />
+</packages>
diff --git a/OpusMTInterface/Translation.cs b/OpusMTInterface/Translation.cs
@@ -11,7 +11,10 @@ namespace OpusMTInterface
     public class Translation
     {
         [DataMember]
-        string translation;
+        public string translation;
+
+        [DataMember]
+        public string alignment;
 
         public Translation(string translation)
         {

diff --git a/Trados2019Plugin/ConnectionControl.xaml.cs b/Trados2019Plugin/ConnectionControl.xaml.cs
@@ -53,7 +53,7 @@ private void FetchServiceData(string host, string port, string modeltag)
             StringBuilder connectionResult = new StringBuilder();
             try
             {
-                ConnectionControl.MtServiceLanguagePairs = OpusCatMTServiceHelper.ListSupportedLanguages(host,port);
+                ConnectionControl.MtServiceLanguagePairs = OpusCatProvider.OpusCatMtEngineConnection.ListSupportedLanguages(host,port);
                 IEnumerable<string> modelTagLanguagePairs;
                 if (this.LanguagePairs != null)
                 {
@@ -93,7 +93,7 @@ private void FetchServiceData(string host, string port, string modeltag)
                 foreach (var languagePair in modelTagLanguagePairs)
                 {
                     var pairSplit = languagePair.Split('-');
-                    modelTags.AddRange(OpusCatMTServiceHelper.GetLanguagePairModelTags(host, port, pairSplit[0],pairSplit[1]));
+                    modelTags.AddRange(OpusCatProvider.OpusCatMtEngineConnection.GetLanguagePairModelTags(host, port, pairSplit[0],pairSplit[1]));
                 }
 
                 this.NoConnection = false;
-Original file line number
+Diff line change
@@ Expand Up @@
                 }
                 ((ModelManager)this.DataContext).OverrideModel = selectedModel;
+                ((ModelManager)this.DataContext).OverrideModelTargetLanguage = selectedModel.TargetLanguages.First();
                 ((ModelManager)this.DataContext).MoveOverrideToTop();
             }
@@ Expand Down @@