diff --git a/OpusCatEngineAndPlugins.sln b/OpusCatEngineAndPlugins.sln index cb07fcc..5966490 100644 --- a/OpusCatEngineAndPlugins.sln +++ b/OpusCatEngineAndPlugins.sln @@ -31,6 +31,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SettingsUiTester", "Setting EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpusCatTradosPluginTests", "OpusCatTradosPluginTests\OpusCatTradosPluginTests.csproj", "{7835C453-B4A0-45C4-BDB8-7888D26561FC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpusCatMTEngineTests", "OpusCatMTEngineTests\OpusCatMTEngineTests.csproj", "{E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -173,6 +175,18 @@ Global {7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x64.Build.0 = Release|Any CPU {7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x86.ActiveCfg = Release|Any CPU {7835C453-B4A0-45C4-BDB8-7888D26561FC}.Release|x86.Build.0 = Release|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|x64.ActiveCfg = Debug|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|x64.Build.0 = Debug|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|x86.ActiveCfg = Debug|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Debug|x86.Build.0 = Debug|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|Any CPU.Build.0 = Release|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|x64.ActiveCfg = Release|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|x64.Build.0 = Release|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|x86.ActiveCfg = Release|Any CPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -189,6 +203,7 @@ Global {72114486-2563-4AF6-B40C-515A21B04754} = {7A01F5CB-2378-473E-AF22-69ED318A6235} {23EB17A2-9A46-4D3E-A471-1952430CDA26} = {7A01F5CB-2378-473E-AF22-69ED318A6235} {7835C453-B4A0-45C4-BDB8-7888D26561FC} = {7A01F5CB-2378-473E-AF22-69ED318A6235} + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364} = {05D6D17C-DFCC-4E76-AE07-0F22FD38F3B7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8F143A32-2D24-488F-AD4C-CD6D074641C6} diff --git a/OpusCatMTEngine/HelperFunctions.cs b/OpusCatMTEngine/HelperFunctions.cs index 651ca0e..1f65977 100644 --- a/OpusCatMTEngine/HelperFunctions.cs +++ b/OpusCatMTEngine/HelperFunctions.cs @@ -10,7 +10,7 @@ namespace OpusCatMTEngine { - class HelperFunctions + public class HelperFunctions { public static string EnumToString(object value) { diff --git a/OpusCatMTEngine/MTModel.cs b/OpusCatMTEngine/MTModel.cs index 48f1065..d3cfc08 100644 --- a/OpusCatMTEngine/MTModel.cs +++ b/OpusCatMTEngine/MTModel.cs @@ -172,8 +172,15 @@ internal void Shutdown() this.marianProcesses = null; } - internal Task Translate(string input, IsoLanguage sourceLang, IsoLanguage targetLang) + public Task Translate(string input, IsoLanguage sourceLang, IsoLanguage targetLang) { + if (String.IsNullOrWhiteSpace(input)) + { + var translationPair = new TranslationPair(input, input, input, "0-0", this.ModelSegmentationMethod, targetLang.OriginalCode); + return Task.FromResult(translationPair); + + } + //Need to get the original codes, since those are the ones the marian model uses var modelSourceLang = this.SourceLanguages.SingleOrDefault(x => x.ShortestIsoCode == sourceLang.ShortestIsoCode); diff --git a/OpusCatMTEngine/MTService.cs b/OpusCatMTEngine/MTService.cs index f80b8b4..69ea95a 100644 --- a/OpusCatMTEngine/MTService.cs +++ b/OpusCatMTEngine/MTService.cs @@ -229,6 +229,23 @@ public int[] BatchStoreTranslation(string tokenCode, List sources, List< } public string Customize( + string tokenCode, + List> input, + List> validation, + List uniqueNewSegments, + string srcLangCode, + string trgLangCode, + string modelTag, + bool includePlaceholderTags, + bool includeTagPairs) + { + List input2 = input.Select(x => new ParallelSentence(x.Item1, x.Item2)).ToList(); + List validation2 = validation.Select(x => new ParallelSentence(x.Item1, x.Item2)).ToList(); + + return this.Customize(tokenCode, input2, validation2, uniqueNewSegments, srcLangCode, trgLangCode, modelTag, includePlaceholderTags, includeTagPairs); + } + + public string Customize( string tokenCode, List input, List validation, diff --git a/OpusCatMTEngine/ModelManager.cs b/OpusCatMTEngine/ModelManager.cs index 256a858..5227cd2 100644 --- a/OpusCatMTEngine/ModelManager.cs +++ b/OpusCatMTEngine/ModelManager.cs @@ -872,6 +872,7 @@ public string ConvertIsoCode(string name) public Task Translate(string input, IsoLanguage srcLang, IsoLanguage trgLang, string modelTag) { + var mtModel = this.SelectModel(srcLang, trgLang, modelTag); if (mtModel == null) diff --git a/OpusCatMTEngine/OWIN/MachineTranslationController.cs b/OpusCatMTEngine/OWIN/MachineTranslationController.cs index 97f45bf..c9c595c 100644 --- a/OpusCatMTEngine/OWIN/MachineTranslationController.cs +++ b/OpusCatMTEngine/OWIN/MachineTranslationController.cs @@ -107,13 +107,19 @@ public Translation TranslatePost(string tokenCode = "", string input = "", strin [HttpGet] public HttpResponseMessage TranslateJson(string tokenCode="", string input="", string srcLangCode="", string trgLangCode="", string modelTag="") { + if (input == null) + { + input = ""; + } //HttpContext.Current.Response.Headers.Add("Access-Control-Allow-Origin", "*"); var sourceLang = new IsoLanguage(srcLangCode); var targetLang = new IsoLanguage(trgLangCode); var translation = this.mtProvider.Translate(input, sourceLang, targetLang, modelTag); - var response = Request.CreateResponse(HttpStatusCode.OK, translation.Result); + + var translationResult = translation.Result; + var response = Request.CreateResponse(HttpStatusCode.OK, translationResult); response.Headers.Add("Access-Control-Allow-Origin", "*"); diff --git a/OpusCatMTEngine/Properties/AssemblyInfo.cs b/OpusCatMTEngine/Properties/AssemblyInfo.cs index 6d2401f..43b018f 100644 --- a/OpusCatMTEngine/Properties/AssemblyInfo.cs +++ b/OpusCatMTEngine/Properties/AssemblyInfo.cs @@ -51,5 +51,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.1.0.5")] -[assembly: AssemblyFileVersion("1.1.0.5")] +[assembly: AssemblyVersion("1.1.0.6")] +[assembly: AssemblyFileVersion("1.1.0.6")] diff --git a/OpusCatMTEngine/Properties/Resources.Designer.cs b/OpusCatMTEngine/Properties/Resources.Designer.cs index 7e7a00c..9de7db0 100644 --- a/OpusCatMTEngine/Properties/Resources.Designer.cs +++ b/OpusCatMTEngine/Properties/Resources.Designer.cs @@ -664,6 +664,24 @@ public static string Progress_ValidationStepAxis { } } + /// + /// Looks up a localized string similar to The TMX file does not contain languages compatible with the language codes of the MT model, or there are not enough segments for the compatible language pair. Language pairs included in the TMX file and the amount of segments per language pair are listed below. You can either select one of the language pairs to extract as fine-tuning material, or cancel fine-tuning.. + /// + public static string SelectTmxLangPair_Instructions { + get { + return ResourceManager.GetString("SelectTmxLangPair_Instructions", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Select language pair. + /// + public static string SelectTmxLangPair_Title { + get { + return ResourceManager.GetString("SelectTmxLangPair_Title", resourceCulture); + } + } + /// /// Looks up a localized string similar to The port that OPUS-CAT MT Engine uses ({0}) is not available. Another instance of OPUS-CAT MT Engine may be active and using the port.. /// diff --git a/OpusCatMTEngine/Properties/Resources.resx b/OpusCatMTEngine/Properties/Resources.resx index 2a6b133..60000b2 100644 --- a/OpusCatMTEngine/Properties/Resources.resx +++ b/OpusCatMTEngine/Properties/Resources.resx @@ -371,4 +371,10 @@ The path where OPUS-CAT MT Engine stores MT models ({0}) contains the following illegal characters: {1}. Install OPUS-CAT MT Engine to a path with no illegal characters in it (for instance, in a folder in the root of the C: drive, such as C:\opuscat). + + The TMX file does not contain languages compatible with the language codes of the MT model, or there are not enough segments for the compatible language pair. Language pairs included in the TMX file and the amount of segments per language pair are listed below. You can either select one of the language pairs to extract as fine-tuning material, or cancel fine-tuning. + + + Select language pair + \ No newline at end of file diff --git a/OpusCatMTEngine/UI/ModelCustomizerView.xaml.cs b/OpusCatMTEngine/UI/ModelCustomizerView.xaml.cs index 991ffbc..8fbd697 100644 --- a/OpusCatMTEngine/UI/ModelCustomizerView.xaml.cs +++ b/OpusCatMTEngine/UI/ModelCustomizerView.xaml.cs @@ -243,12 +243,17 @@ private void customize_Click(object sender, RoutedEventArgs e) this.IncludeTagPairBox.IsChecked.Value ); } + else + { + return; + } } else { MessageBox.Show( String.Format( OpusCatMTEngine.Properties.Resources.Finetune_NotEnoughSegmentsInTmx)); + return; } } break; diff --git a/OpusCatMTEngine/UI/SelectTmxLangPairWindow.xaml b/OpusCatMTEngine/UI/SelectTmxLangPairWindow.xaml index 6f7b109..9220b84 100644 --- a/OpusCatMTEngine/UI/SelectTmxLangPairWindow.xaml +++ b/OpusCatMTEngine/UI/SelectTmxLangPairWindow.xaml @@ -3,13 +3,14 @@ xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" + xmlns:resx="clr-namespace:OpusCatMTEngine.Properties" xmlns:local="clr-namespace:OpusCatMTEngine" mc:Ignorable="d" - Title="SelectTmxLangPair" Height="400" Width="400"> + Title="{x:Static resx:Resources.SelectTmxLangPair_Title}" Height="400" Width="400"> - + @@ -18,15 +19,17 @@ - - The TMX file does not contain languages compatible with the language codes of the - MT model, or there is not enough segments for the compatible language pair. Language pairs - included in the TMX file and the amount of segments per language pair is listed below. - You can either select one language pair to extract as fine-tuning material, or cancel - fine-tuning. - + + + - + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/OpusCatMTEngineTests/FinetuneTest.cs b/OpusCatMTEngineTests/FinetuneTest.cs new file mode 100644 index 0000000..b06300c --- /dev/null +++ b/OpusCatMTEngineTests/FinetuneTest.cs @@ -0,0 +1,26 @@ +using System; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace OpusCatMTEngineTests +{ + [TestClass] + public class FinetuneTest + { + /// + /// This is not functional, there should be tests for most common + /// functionalities, but no time to implement right now. Use this as + /// starting point if time is available. + /// + [TestMethod] + public void TestFinetune() + { + var modelInstallPath = OpusCatMTEngine.HelperFunctions.GetLocalAppDataPath("models\\eng-fin\\opus+bt-2021-03-09"); + var model = new OpusCatMTEngine.MTModel("eng-fin\\opus+bt-2021-03-09", modelInstallPath); + var testTranslation = model.Translate("this is a test", model.SourceLanguages.First(), model.TargetLanguages.First()); + var result = testTranslation.Result; + var finetuneWindow = new OpusCatMTEngine.ModelCustomizerView(model); + + } + } +} diff --git a/OpusCatMTEngineTests/OpusCatMTEngineTests.csproj b/OpusCatMTEngineTests/OpusCatMTEngineTests.csproj new file mode 100644 index 0000000..bfc5c14 --- /dev/null +++ b/OpusCatMTEngineTests/OpusCatMTEngineTests.csproj @@ -0,0 +1,97 @@ + + + + + + + Debug + AnyCPU + {E28EC7F7-29A2-4BAD-8C86-95DFE0FBE364} + Library + Properties + OpusCatMTEngineTests + OpusCatMTEngineTests + v4.7.2 + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + 15.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages + False + UnitTest + + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\EntityFramework.6.4.4\lib\net45\EntityFramework.dll + + + ..\packages\EntityFramework.6.4.4\lib\net45\EntityFramework.SqlServer.dll + + + ..\packages\MSTest.TestFramework.1.3.2\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.dll + + + ..\packages\MSTest.TestFramework.1.3.2\lib\net45\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll + + + + + + ..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.114.0\lib\net46\System.Data.SQLite.dll + + + ..\packages\System.Data.SQLite.EF6.1.0.114.0\lib\net46\System.Data.SQLite.EF6.dll + + + ..\packages\System.Data.SQLite.Linq.1.0.114.0\lib\net46\System.Data.SQLite.Linq.dll + + + + + + + + + + + + + {03bf6215-d7c1-4401-a427-3590990c70a8} + OpusCatMTEngine + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + + + \ No newline at end of file diff --git a/OpusCatMTEngineTests/Properties/AssemblyInfo.cs b/OpusCatMTEngineTests/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..b49df02 --- /dev/null +++ b/OpusCatMTEngineTests/Properties/AssemblyInfo.cs @@ -0,0 +1,20 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("OpusCatMTEngineTests")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("OpusCatMTEngineTests")] +[assembly: AssemblyCopyright("Copyright © 2021")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +[assembly: ComVisible(false)] + +[assembly: Guid("e28ec7f7-29a2-4bad-8c86-95dfe0fbe364")] + +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/OpusCatMTEngineTests/packages.config b/OpusCatMTEngineTests/packages.config new file mode 100644 index 0000000..96f01e9 --- /dev/null +++ b/OpusCatMTEngineTests/packages.config @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/OpusMTInterface/IMTService.cs b/OpusMTInterface/IMTService.cs index 092d96e..4b8716c 100644 --- a/OpusMTInterface/IMTService.cs +++ b/OpusMTInterface/IMTService.cs @@ -42,18 +42,18 @@ public interface IMTService [WebGet] string CheckModelStatus(string tokenCode, string srcLangCode, string trgLangCode, string modelTag); - /*[OperationContract] + [OperationContract] [WebInvoke(Method = "POST",BodyStyle = WebMessageBodyStyle.Wrapped)] string Customize( string tokenCode, - List input, - List validation, + List> input, + List> validation, List uniqueNewSegments, string srcLangCode, string trgLangCode, string modelTag, bool includePlaceholderTags, - bool includeTagPairs);*/ + bool includeTagPairs); [OperationContract] [WebInvoke(Method = "POST", BodyStyle = WebMessageBodyStyle.Wrapped)]