From 59cc2f61f0b8324a69ed74db1698996272853086 Mon Sep 17 00:00:00 2001 From: Kateryna Oblakevych Date: Mon, 19 Feb 2024 07:08:43 +0200 Subject: [PATCH 1/4] feat: multilingual option --- .../com/crowdin/cli/properties/FileBean.java | 43 ++++++------------- .../cli/properties/ParamsWithFiles.java | 3 ++ .../cli/properties/PropertiesBuilder.java | 2 + .../PropertiesWithFilesBuilder.java | 6 ++- src/main/resources/crowdin.yml | 6 +++ .../resources/messages/messages.properties | 1 + website/docs/advanced.md | 12 ++---- 7 files changed, 34 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/crowdin/cli/properties/FileBean.java b/src/main/java/com/crowdin/cli/properties/FileBean.java index d02886bc0..eec195e6b 100755 --- a/src/main/java/com/crowdin/cli/properties/FileBean.java +++ b/src/main/java/com/crowdin/cli/properties/FileBean.java @@ -11,34 +11,7 @@ import java.util.Map; import static com.crowdin.cli.BaseCli.RESOURCE_BUNDLE; -import static com.crowdin.cli.properties.PropertiesBuilder.CONTENT_SEGMENTATION; -import static com.crowdin.cli.properties.PropertiesBuilder.CUSTOM_SEGMENTATION; -import static com.crowdin.cli.properties.PropertiesBuilder.DEST; -import static com.crowdin.cli.properties.PropertiesBuilder.ESCAPE_QUOTES; -import static com.crowdin.cli.properties.PropertiesBuilder.ESCAPE_SPECIAL_CHARACTERS; -import static com.crowdin.cli.properties.PropertiesBuilder.EXPORT_QUOTES; -import static com.crowdin.cli.properties.PropertiesBuilder.EXCLUDED_TARGET_LANGUAGES; -import static com.crowdin.cli.properties.PropertiesBuilder.EXPORT_APPROVED_ONLY; -import static com.crowdin.cli.properties.PropertiesBuilder.FIRST_LINE_CONTAINS_HEADER; -import static com.crowdin.cli.properties.PropertiesBuilder.IGNORE; -import static com.crowdin.cli.properties.PropertiesBuilder.LABELS; -import static com.crowdin.cli.properties.PropertiesBuilder.LANGUAGES_MAPPING; -import static com.crowdin.cli.properties.PropertiesBuilder.MULTILINGUAL_SPREADSHEET; -import static com.crowdin.cli.properties.PropertiesBuilder.SCHEME; -import static com.crowdin.cli.properties.PropertiesBuilder.SKIP_UNTRANSLATED_FILES; -import static com.crowdin.cli.properties.PropertiesBuilder.SKIP_UNTRANSLATED_STRINGS; -import static com.crowdin.cli.properties.PropertiesBuilder.EXPORT_STRINGS_THAT_PASSED_WORKFLOW; -import static com.crowdin.cli.properties.PropertiesBuilder.SOURCE; -import static com.crowdin.cli.properties.PropertiesBuilder.TRANSLATABLE_ELEMENTS; -import static com.crowdin.cli.properties.PropertiesBuilder.TRANSLATE_ATTRIBUTES; -import static com.crowdin.cli.properties.PropertiesBuilder.TRANSLATE_CONTENT; -import static com.crowdin.cli.properties.PropertiesBuilder.TRANSLATION; -import static com.crowdin.cli.properties.PropertiesBuilder.TRANSLATION_REPLACE; -import static com.crowdin.cli.properties.PropertiesBuilder.TYPE; -import static com.crowdin.cli.properties.PropertiesBuilder.UPDATE_OPTION; -import static com.crowdin.cli.properties.PropertiesBuilder.IMPORT_TRANSLATIONS; -import static com.crowdin.cli.properties.PropertiesBuilder.checkForDoubleAsterisks; -import static com.crowdin.cli.properties.PropertiesBuilder.hasRelativePaths; +import static com.crowdin.cli.properties.PropertiesBuilder.*; @Data public class FileBean { @@ -48,6 +21,7 @@ public class FileBean { private String source; private String translation; private List ignore; + private Boolean multilingual; private String dest; private String type; private String updateOption; @@ -98,6 +72,7 @@ public FileBean buildFromMap(Map map) { PropertiesBuilder.setBooleanPropertyIfExists(fileBean::setTranslateAttributes, map, TRANSLATE_ATTRIBUTES); PropertiesBuilder.setBooleanPropertyIfExists(fileBean::setTranslateContent, map, TRANSLATE_CONTENT); PropertiesBuilder.setBooleanPropertyIfExists(fileBean::setContentSegmentation, map, CONTENT_SEGMENTATION); + PropertiesBuilder.setBooleanPropertyIfExists(fileBean::setMultilingual, map, MULTILINGUAL); PropertiesBuilder.setBooleanPropertyIfExists(fileBean::setMultilingualSpreadsheet, map, MULTILINGUAL_SPREADSHEET); PropertiesBuilder.setBooleanPropertyIfExists(fileBean::setSkipTranslatedOnly, map, SKIP_UNTRANSLATED_STRINGS); PropertiesBuilder.setBooleanPropertyIfExists(fileBean::setSkipUntranslatedFiles, map, SKIP_UNTRANSLATED_FILES); @@ -120,7 +95,9 @@ public void populateWithDefaultValues(FileBean bean) { //Translation if (bean.getTranslation() != null) { bean.setTranslation(Utils.normalizePath(bean.getTranslation())); - if (!PlaceholderUtil.containsLangPlaceholders(bean.getTranslation()) && bean.getScheme() != null) { + if (!PlaceholderUtil.containsLangPlaceholders(bean.getTranslation()) + && (bean.getScheme() != null + || (bean.getMultilingual() != null && bean.getMultilingual()))) { bean.setTranslation(Utils.noSepAtStart(bean.getTranslation())); } else { bean.setTranslation(Utils.sepAtStart(bean.getTranslation())); @@ -135,6 +112,10 @@ public void populateWithDefaultValues(FileBean bean) { } bean.setIgnore(ignores); } + //Multilingual + if (bean.getMultilingual() == null) { + bean.setMultilingual(Boolean.FALSE); + } //dest if (StringUtils.isNotEmpty(bean.getDest())) { bean.setDest(bean.getDest().replaceAll("[/\\\\]+", Utils.PATH_SEPARATOR_REGEX)); @@ -179,7 +160,9 @@ public List checkProperties(FileBean bean) { errors.add(RESOURCE_BUNDLE.getString("error.config.double_asterisk")); } - if (!PlaceholderUtil.containsLangPlaceholders(bean.getTranslation()) && bean.getScheme() == null) { + if (!PlaceholderUtil.containsLangPlaceholders(bean.getTranslation()) + && bean.getScheme() == null + && (bean.getMultilingual() == null || !bean.getMultilingual())) { errors.add(RESOURCE_BUNDLE.getString("error.config.translation_has_no_language_placeholders")); } diff --git a/src/main/java/com/crowdin/cli/properties/ParamsWithFiles.java b/src/main/java/com/crowdin/cli/properties/ParamsWithFiles.java index 4a1cf147a..f7a3fecc6 100644 --- a/src/main/java/com/crowdin/cli/properties/ParamsWithFiles.java +++ b/src/main/java/com/crowdin/cli/properties/ParamsWithFiles.java @@ -22,6 +22,9 @@ public class ParamsWithFiles extends ProjectParams { @CommandLine.Option(names = {"--preserve-hierarchy"}, negatable = true, paramLabel = "...", descriptionKey = "params.preserve-hierarchy") private Boolean preserveHierarchy; + @CommandLine.Option(names = {"--multilingual"}, negatable = true, paramLabel = "...", descriptionKey = "params.multilingual") + private Boolean multilingual; + private Boolean skipTranslatedOnly; private Boolean skipUntranslatedFiles; private Boolean exportApprovedOnly; diff --git a/src/main/java/com/crowdin/cli/properties/PropertiesBuilder.java b/src/main/java/com/crowdin/cli/properties/PropertiesBuilder.java index fd94ccce6..514df2fe0 100644 --- a/src/main/java/com/crowdin/cli/properties/PropertiesBuilder.java +++ b/src/main/java/com/crowdin/cli/properties/PropertiesBuilder.java @@ -52,6 +52,8 @@ public abstract class PropertiesBuilder public static final String IGNORE = "ignore"; + public static final String MULTILINGUAL = "multilingual"; + public static final String DEST = "dest"; public static final String TYPE = "type"; diff --git a/src/main/java/com/crowdin/cli/properties/PropertiesWithFilesBuilder.java b/src/main/java/com/crowdin/cli/properties/PropertiesWithFilesBuilder.java index 794d69c4e..ddfd91fa6 100644 --- a/src/main/java/com/crowdin/cli/properties/PropertiesWithFilesBuilder.java +++ b/src/main/java/com/crowdin/cli/properties/PropertiesWithFilesBuilder.java @@ -64,7 +64,8 @@ protected Messages checkArgParams(ParamsWithFiles params) { if (!checkForDoubleAsterisks(params.getSourceParam(), params.getTranslationParam())) { messages.addError(RESOURCE_BUNDLE.getString("error.config.double_asterisk")); } - if (!PlaceholderUtil.containsLangPlaceholders(params.getTranslationParam())) { + if (!PlaceholderUtil.containsLangPlaceholders(params.getTranslationParam()) + && (params.getMultilingual() == null || !params.getMultilingual())) { messages.addError(RESOURCE_BUNDLE.getString("error.config.translation_has_no_language_placeholders")); } } else { @@ -121,6 +122,9 @@ protected void populateWithArgParams(@NonNull PropertiesWithFiles props, @NonNul props.setPreserveHierarchy(true); fb.setDest(params.getDestParam()); } + if (params.getMultilingual() != null) { + fb.setMultilingual(params.getMultilingual()); + } props.setFiles(Arrays.asList(fb)); } for (FileBean fb : props.getFiles()) { diff --git a/src/main/resources/crowdin.yml b/src/main/resources/crowdin.yml index c3fd80e7c..d6183b6a9 100644 --- a/src/main/resources/crowdin.yml +++ b/src/main/resources/crowdin.yml @@ -75,6 +75,12 @@ files: [ # # "translatable_elements": [], + # + # Is file format multilingual? + # e.g. true or false + # + # "multilingual": true, + # # Defines whether to split long texts into smaller text segments # e.g. 0 or 1 (Default is 1) diff --git a/src/main/resources/messages/messages.properties b/src/main/resources/messages/messages.properties index 06c9636ef..0f5b56cee 100755 --- a/src/main/resources/messages/messages.properties +++ b/src/main/resources/messages/messages.properties @@ -24,6 +24,7 @@ params.source=Path to the source files params.translation=Path to the translation files params.dest=Specify file name in Crowdin params.preserve-hierarchy=Choose whether to save the directory hierarchy in the Crowdin project +params.multilingual=Specify whether file format is multilingual params.skipUntranslatedStrings=Skip untranslated strings in exported files (does not work with .docx, .html, .md and other document files) params.skipUntranslatedFiles=Omit downloading not fully translated files params.keepArchive=Do not remove the downloaded archive with translations after it's extracting diff --git a/website/docs/advanced.md b/website/docs/advanced.md index 8871dd8c1..183d6b311 100644 --- a/website/docs/advanced.md +++ b/website/docs/advanced.md @@ -123,7 +123,7 @@ Visit the [KB article](https://developer.crowdin.com/pseudolocalization/) to rea **Escape Quotes** -The `escape_qutes` option defines whether a single quote should be escaped by another single quote or backslash in exported translations. You can add the `escape_quotes` per-file option. Acceptable values are `0`, `1`, `2`, `3`. Default is `3`. +The `escape_quotes` option defines whether a single quote should be escaped by another single quote or backslash in exported translations. You can add the `escape_quotes` per-file option. Acceptable values are `0`, `1`, `2`, `3`. Default is `3`. - `0` - do not escape - `1` - escape single quote with another single quote @@ -168,24 +168,20 @@ Example of the configuration: ] ``` -#### Apple Strings Catalog +### Multilingual Support -Apple Strings Catalog is a natively supported file format by Crowdin, you can use it without any additional installation. Since this file format is **multilingual** (contains multiple languages in one file), you can omit the language placeholders in the `translation` pattern: +For multilingual file formats (contains multiple languages in one file), you can use the `multilingual` option in the configuration. With this option you can omit the language placeholders in the `translation` pattern: ```yml title="crowdin.yml" "files": [ { "source": "Localizable.xcstrings", "translation": "Localizable.xcstrings", - "scheme": "" + "multilingual": true } ] ``` -:::caution -Note the `scheme` option in the configuration above. It's needed to indicate that this file is multilingual for correct file processing. -::: - ### Configure export options for each file group There is a way to specify export options for each file-group in the `crowdin.yml` configuration file: From a07f3b9f131fc10c9ae031133860d271eace9e30 Mon Sep 17 00:00:00 2001 From: Kateryna Oblakevych Date: Mon, 19 Feb 2024 08:19:14 +0200 Subject: [PATCH 2/4] tests --- prepare-docs.sh | 1 + .../functionality/PropertiesBuilderTest.java | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/prepare-docs.sh b/prepare-docs.sh index 7f42eb035..4439dde19 100755 --- a/prepare-docs.sh +++ b/prepare-docs.sh @@ -82,6 +82,7 @@ sed -i.bak -e 's/\*\*--\[no-\]preserve-hierarchy\*\*/`--[no-]preserve-hierarchy` sed -i.bak -e 's/\*\*--\[no-\]auto-tag\*\*/`--[no-]auto-tag`/g' -- *.md sed -i.bak -e 's/\*\*--\[no-\]cleanup-mode\*\*/`--[no-]cleanup-mode`/g' -- *.md sed -i.bak -e 's/\*\*--\[no-\]update-strings\*\*/`--[no-]update-strings`/g' -- *.md +sed -i.bak -e 's/\*\*--\[no-\]multilingual\*\*/`--[no-]multilingual`/g' -- *.md rm -- *.md.bak diff --git a/src/test/java/com/crowdin/cli/commands/functionality/PropertiesBuilderTest.java b/src/test/java/com/crowdin/cli/commands/functionality/PropertiesBuilderTest.java index 4f418ba68..4a508e279 100644 --- a/src/test/java/com/crowdin/cli/commands/functionality/PropertiesBuilderTest.java +++ b/src/test/java/com/crowdin/cli/commands/functionality/PropertiesBuilderTest.java @@ -13,6 +13,7 @@ import static com.crowdin.cli.BaseCli.RESOURCE_BUNDLE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -99,6 +100,22 @@ public void testOkBasePath_Params_WithConfigFile() { assertEquals(tempProject.getBasePath() + "folder2" + Utils.PATH_SEPARATOR, pb.getBasePath()); } + @Test + public void testOk_Params_WithoutConfigFile_TranslationNoSepAtStart() { + ParamsWithFiles okParams = new ParamsWithFiles() {{ + setIdParam("666"); + setTokenParam("123abc456"); + setSourceParam(Utils.regexPath(Utils.normalizePath("/Localizable.xcstrings"))); + setTranslationParam("/Localizable.xcstrings"); + setMultilingual(true); + }}; + + PropertiesWithFiles pb = propertiesBuilders.buildPropertiesWithFiles(out, null, null, okParams); + + assertEquals(pb.getFiles().get(0).getSource(), Utils.PATH_SEPARATOR + "Localizable.xcstrings"); + assertEquals(pb.getFiles().get(0).getTranslation(), "Localizable.xcstrings"); + } + @Test public void testBuildNoProperties() { PropertiesBuilders pb = mock(PropertiesBuilders.class); @@ -181,6 +198,21 @@ public void testBuildNoConfigFileAndNoToken() { } + @Test + public void testNoTranslationLangPlaceholder() { + ParamsWithFiles params = new ParamsWithFiles() {{ + setIdParam("666"); + setTokenParam("123abc456"); + setSourceParam(Utils.regexPath(Utils.normalizePath("/Localizable.xcstrings"))); + setTranslationParam("Localizable.xcstrings"); + }}; + + Exception actualException = assertThrows(RuntimeException.class, () -> + propertiesBuilders.buildPropertiesWithFiles(out, null, null, params)); + + assertTrue(actualException.getMessage().contains(RESOURCE_BUNDLE.getString("error.config.translation_has_no_language_placeholders"))); + } + @Test public void testPropertiesWithTarget() { File configFile = new File("folder/crowdinTest.yml"); From f56366e565d865726201b99bde742620ac411dff Mon Sep 17 00:00:00 2001 From: Kateryna Oblakevych Date: Wed, 21 Feb 2024 07:11:42 +0200 Subject: [PATCH 3/4] remove multilingual command line option --- prepare-docs.sh | 1 - .../crowdin/cli/properties/ParamsWithFiles.java | 3 --- .../properties/PropertiesWithFilesBuilder.java | 6 +----- src/main/resources/crowdin.yml | 6 ------ .../functionality/PropertiesBuilderTest.java | 16 ---------------- 5 files changed, 1 insertion(+), 31 deletions(-) diff --git a/prepare-docs.sh b/prepare-docs.sh index 4439dde19..7f42eb035 100755 --- a/prepare-docs.sh +++ b/prepare-docs.sh @@ -82,7 +82,6 @@ sed -i.bak -e 's/\*\*--\[no-\]preserve-hierarchy\*\*/`--[no-]preserve-hierarchy` sed -i.bak -e 's/\*\*--\[no-\]auto-tag\*\*/`--[no-]auto-tag`/g' -- *.md sed -i.bak -e 's/\*\*--\[no-\]cleanup-mode\*\*/`--[no-]cleanup-mode`/g' -- *.md sed -i.bak -e 's/\*\*--\[no-\]update-strings\*\*/`--[no-]update-strings`/g' -- *.md -sed -i.bak -e 's/\*\*--\[no-\]multilingual\*\*/`--[no-]multilingual`/g' -- *.md rm -- *.md.bak diff --git a/src/main/java/com/crowdin/cli/properties/ParamsWithFiles.java b/src/main/java/com/crowdin/cli/properties/ParamsWithFiles.java index f7a3fecc6..4a1cf147a 100644 --- a/src/main/java/com/crowdin/cli/properties/ParamsWithFiles.java +++ b/src/main/java/com/crowdin/cli/properties/ParamsWithFiles.java @@ -22,9 +22,6 @@ public class ParamsWithFiles extends ProjectParams { @CommandLine.Option(names = {"--preserve-hierarchy"}, negatable = true, paramLabel = "...", descriptionKey = "params.preserve-hierarchy") private Boolean preserveHierarchy; - @CommandLine.Option(names = {"--multilingual"}, negatable = true, paramLabel = "...", descriptionKey = "params.multilingual") - private Boolean multilingual; - private Boolean skipTranslatedOnly; private Boolean skipUntranslatedFiles; private Boolean exportApprovedOnly; diff --git a/src/main/java/com/crowdin/cli/properties/PropertiesWithFilesBuilder.java b/src/main/java/com/crowdin/cli/properties/PropertiesWithFilesBuilder.java index ddfd91fa6..794d69c4e 100644 --- a/src/main/java/com/crowdin/cli/properties/PropertiesWithFilesBuilder.java +++ b/src/main/java/com/crowdin/cli/properties/PropertiesWithFilesBuilder.java @@ -64,8 +64,7 @@ protected Messages checkArgParams(ParamsWithFiles params) { if (!checkForDoubleAsterisks(params.getSourceParam(), params.getTranslationParam())) { messages.addError(RESOURCE_BUNDLE.getString("error.config.double_asterisk")); } - if (!PlaceholderUtil.containsLangPlaceholders(params.getTranslationParam()) - && (params.getMultilingual() == null || !params.getMultilingual())) { + if (!PlaceholderUtil.containsLangPlaceholders(params.getTranslationParam())) { messages.addError(RESOURCE_BUNDLE.getString("error.config.translation_has_no_language_placeholders")); } } else { @@ -122,9 +121,6 @@ protected void populateWithArgParams(@NonNull PropertiesWithFiles props, @NonNul props.setPreserveHierarchy(true); fb.setDest(params.getDestParam()); } - if (params.getMultilingual() != null) { - fb.setMultilingual(params.getMultilingual()); - } props.setFiles(Arrays.asList(fb)); } for (FileBean fb : props.getFiles()) { diff --git a/src/main/resources/crowdin.yml b/src/main/resources/crowdin.yml index d6183b6a9..c3fd80e7c 100644 --- a/src/main/resources/crowdin.yml +++ b/src/main/resources/crowdin.yml @@ -75,12 +75,6 @@ files: [ # # "translatable_elements": [], - # - # Is file format multilingual? - # e.g. true or false - # - # "multilingual": true, - # # Defines whether to split long texts into smaller text segments # e.g. 0 or 1 (Default is 1) diff --git a/src/test/java/com/crowdin/cli/commands/functionality/PropertiesBuilderTest.java b/src/test/java/com/crowdin/cli/commands/functionality/PropertiesBuilderTest.java index 4a508e279..2b111020e 100644 --- a/src/test/java/com/crowdin/cli/commands/functionality/PropertiesBuilderTest.java +++ b/src/test/java/com/crowdin/cli/commands/functionality/PropertiesBuilderTest.java @@ -100,22 +100,6 @@ public void testOkBasePath_Params_WithConfigFile() { assertEquals(tempProject.getBasePath() + "folder2" + Utils.PATH_SEPARATOR, pb.getBasePath()); } - @Test - public void testOk_Params_WithoutConfigFile_TranslationNoSepAtStart() { - ParamsWithFiles okParams = new ParamsWithFiles() {{ - setIdParam("666"); - setTokenParam("123abc456"); - setSourceParam(Utils.regexPath(Utils.normalizePath("/Localizable.xcstrings"))); - setTranslationParam("/Localizable.xcstrings"); - setMultilingual(true); - }}; - - PropertiesWithFiles pb = propertiesBuilders.buildPropertiesWithFiles(out, null, null, okParams); - - assertEquals(pb.getFiles().get(0).getSource(), Utils.PATH_SEPARATOR + "Localizable.xcstrings"); - assertEquals(pb.getFiles().get(0).getTranslation(), "Localizable.xcstrings"); - } - @Test public void testBuildNoProperties() { PropertiesBuilders pb = mock(PropertiesBuilders.class); From 7661d83030338410288ee2c80e3dbc933aa366fe Mon Sep 17 00:00:00 2001 From: Kateryna Oblakevych Date: Wed, 21 Feb 2024 13:01:20 +0200 Subject: [PATCH 4/4] docs --- src/main/resources/messages/messages.properties | 1 - website/docs/advanced.md | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/resources/messages/messages.properties b/src/main/resources/messages/messages.properties index 0f5b56cee..06c9636ef 100755 --- a/src/main/resources/messages/messages.properties +++ b/src/main/resources/messages/messages.properties @@ -24,7 +24,6 @@ params.source=Path to the source files params.translation=Path to the translation files params.dest=Specify file name in Crowdin params.preserve-hierarchy=Choose whether to save the directory hierarchy in the Crowdin project -params.multilingual=Specify whether file format is multilingual params.skipUntranslatedStrings=Skip untranslated strings in exported files (does not work with .docx, .html, .md and other document files) params.skipUntranslatedFiles=Omit downloading not fully translated files params.keepArchive=Do not remove the downloaded archive with translations after it's extracting diff --git a/website/docs/advanced.md b/website/docs/advanced.md index 183d6b311..4355592ed 100644 --- a/website/docs/advanced.md +++ b/website/docs/advanced.md @@ -168,9 +168,9 @@ Example of the configuration: ] ``` -### Multilingual Support +### Multilingual Files -For multilingual file formats (contains multiple languages in one file), you can use the `multilingual` option in the configuration. With this option you can omit the language placeholders in the `translation` pattern: +For multilingual file formats (containing multiple languages in one file) you can use the `multilingual` option in the configuration. This option allows you to omit the language placeholders in the `translation` pattern: ```yml title="crowdin.yml" "files": [