Merge pull request #90 from marhali/feat/json5

Feat/json5

CHANGELOG.md

@@ -3,6 +3,8 @@
 # easy-i18n Changelog
 
 ## [Unreleased]
+### Added
+- Support for Json5 files
 
 ## [3.0.1]
 ### Changed

README.md

@@ -41,7 +41,7 @@ This plugin can be used for any project based on one of the formats and structur
 
 ## Builtin Support
 ### File Types
-**<kbd>JSON</kbd>** - **<kbd>YAML</kbd>** - **<kbd>Properties</kbd>**
+**<kbd>JSON</kbd>** - **<kbd>JSON5</kbd>** - **<kbd>YAML</kbd>** - **<kbd>Properties</kbd>**
 
 ### Folder Structure
 - Single Directory: All translation files are within one directory 
@@ -90,7 +90,7 @@ _For more examples, please refer to the [Examples Directory](https://github.com/
 <!-- ROADMAP -->
 ## Roadmap
 
-- [ ] JSON5 Support
+- [X] JSON5 Support
 - [ ] XML Support
 - [ ] Mark duplicate translation values
 

build.gradle.kts

@@ -24,6 +24,10 @@ repositories {
     mavenCentral()
 }
 
+dependencies {
+    implementation("de.marhali:json5-java:2.0.0")
+}
+
 // Configure Gradle IntelliJ Plugin - read more: https://github.com/JetBrains/gradle-intellij-plugin
 intellij {
     pluginName.set(properties("pluginName"))

gradle.properties

@@ -4,7 +4,7 @@
 pluginGroup = de.marhali.easyi18n
 pluginName = easy-i18n
 # SemVer format -> https://semver.org
-pluginVersion = 3.0.1
+pluginVersion = 3.1.0
 
 # See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
 # for insight into build numbers and IntelliJ Platform versions.

src/main/java/de/marhali/easyi18n/io/parser/ParserStrategyType.java

@@ -1,6 +1,7 @@
 package de.marhali.easyi18n.io.parser;
 
 import de.marhali.easyi18n.io.parser.json.JsonParserStrategy;
+import de.marhali.easyi18n.io.parser.json5.Json5ParserStrategy;
 import de.marhali.easyi18n.io.parser.properties.PropertiesParserStrategy;
 import de.marhali.easyi18n.io.parser.yaml.YamlParserStrategy;
 
@@ -10,6 +11,7 @@ import de.marhali.easyi18n.io.parser.yaml.YamlParserStrategy;
  */
 public enum ParserStrategyType {
     JSON(JsonParserStrategy.class),
+    JSON5(Json5ParserStrategy.class),
     YAML(YamlParserStrategy.class),
     YML(YamlParserStrategy.class),
     PROPERTIES(PropertiesParserStrategy.class),

src/main/java/de/marhali/easyi18n/io/parser/json5/Json5ArrayMapper.java

@@ -0,0 +1,53 @@
+package de.marhali.easyi18n.io.parser.json5;
+
+import de.marhali.easyi18n.io.parser.ArrayMapper;
+import de.marhali.easyi18n.util.StringUtil;
+import de.marhali.json5.Json5;
+import de.marhali.json5.Json5Array;
+import de.marhali.json5.Json5Primitive;
+
+import org.apache.commons.lang.math.NumberUtils;
+
+import java.io.IOException;
+
+/**
+ * Map json5 array values.
+ * @author marhali
+ */
+public class Json5ArrayMapper extends ArrayMapper {
+
+    private static final Json5 JSON5 = Json5.builder(builder ->
+            builder.allowInvalidSurrogate().quoteSingle().indentFactor(0).build());
+
+    public static String read(Json5Array array) {
+        return read(array.iterator(), (jsonElement -> {
+            try {
+                return jsonElement.isJson5Array() || jsonElement.isJson5Object()
+                        ? "\\" + JSON5.serialize(jsonElement)
+                        : jsonElement.getAsString();
+            } catch (IOException e) {
+                throw new AssertionError(e.getMessage(), e.getCause());
+            }
+        }));
+    }
+
+    public static Json5Array write(String concat) {
+        Json5Array array = new Json5Array();
+
+        write(concat, (element) -> {
+            if(element.startsWith("\\")) {
+                array.add(JSON5.parse(element.replace("\\", "")));
+            } else {
+                if(StringUtil.isHexString(element)) {
+                    array.add(Json5Primitive.of(element, true));
+                } else if(NumberUtils.isNumber(element)) {
+                    array.add(Json5Primitive.of(NumberUtils.createNumber(element)));
+                } else {
+                    array.add(Json5Primitive.of(element));
+                }
+            }
+        });
+
+        return array;
+    }
+}

src/main/java/de/marhali/easyi18n/io/parser/json5/Json5Mapper.java

@@ -0,0 +1,73 @@
+package de.marhali.easyi18n.io.parser.json5;
+
+import de.marhali.easyi18n.model.Translation;
+import de.marhali.easyi18n.model.TranslationNode;
+import de.marhali.easyi18n.util.StringUtil;
+
+import de.marhali.json5.Json5Element;
+import de.marhali.json5.Json5Object;
+import de.marhali.json5.Json5Primitive;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.math.NumberUtils;
+
+import java.util.Map;
+
+/**
+ * Mapper for mapping json5 objects into translation nodes and backwards.
+ * @author marhali
+ */
+public class Json5Mapper {
+    public static void read(String locale, Json5Object json, TranslationNode node) {
+        for(Map.Entry<String, Json5Element> entry : json.entrySet()) {
+            String key = entry.getKey();
+            Json5Element value = entry.getValue();
+
+            TranslationNode childNode = node.getOrCreateChildren(key);
+
+            if(value.isJson5Object()) {
+                // Nested element - run recursively
+                read(locale, value.getAsJson5Object(), childNode);
+            } else {
+                Translation translation = childNode.getValue();
+
+                String content = value.isJson5Array()
+                        ? Json5ArrayMapper.read(value.getAsJson5Array())
+                        : StringUtil.escapeControls(value.getAsString(), true);
+
+                translation.put(locale, content);
+                childNode.setValue(translation);
+            }
+        }
+    }
+
+    public static void write(String locale, Json5Object json, TranslationNode node) {
+        for(Map.Entry<String, TranslationNode> entry : node.getChildren().entrySet()) {
+            String key = entry.getKey();
+            TranslationNode childNode = entry.getValue();
+
+            if(!childNode.isLeaf()) {
+                // Nested node - run recursively
+                Json5Object childJson = new Json5Object();
+                write(locale, childJson, childNode);
+                if(childJson.size() > 0) {
+                    json.add(key, childJson);
+                }
+
+            } else {
+                Translation translation = childNode.getValue();
+                String content = translation.get(locale);
+                if(content != null) {
+                    if(Json5ArrayMapper.isArray(content)) {
+                        json.add(key, Json5ArrayMapper.write(content));
+                    } else if(StringUtil.isHexString(content)) {
+                        json.add(key, Json5Primitive.of(content, true));
+                    } else if(NumberUtils.isNumber(content)) {
+                        json.add(key, Json5Primitive.of(NumberUtils.createNumber(content)));
+                    } else {
+                        json.add(key, Json5Primitive.of(StringEscapeUtils.unescapeJava(content)));
+                    }
+                }
+            }
+        }
+    }
+}

src/main/java/de/marhali/easyi18n/io/parser/json5/Json5ParserStrategy.java

@@ -0,0 +1,58 @@
+package de.marhali.easyi18n.io.parser.json5;
+
+import com.intellij.openapi.vfs.VirtualFile;
+
+import de.marhali.easyi18n.io.parser.ParserStrategy;
+import de.marhali.easyi18n.model.SettingsState;
+import de.marhali.easyi18n.model.TranslationData;
+import de.marhali.easyi18n.model.TranslationFile;
+import de.marhali.easyi18n.model.TranslationNode;
+import de.marhali.json5.Json5;
+import de.marhali.json5.Json5Element;
+import de.marhali.json5.Json5Object;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Objects;
+
+/**
+ * Json5 file format parser strategy
+ * @author marhali
+ */
+public class Json5ParserStrategy extends ParserStrategy {
+
+    private static final Json5 JSON5 = Json5.builder(builder ->
+            builder.allowInvalidSurrogate().trailingComma().indentFactor(4).build());
+
+    public Json5ParserStrategy(@NotNull SettingsState settings) {
+        super(settings);
+    }
+
+    @Override
+    public void read(@NotNull TranslationFile file, @NotNull TranslationData data) throws Exception {
+        data.addLocale(file.getLocale());
+
+        VirtualFile vf = file.getVirtualFile();
+        TranslationNode targetNode = super.getOrCreateTargetNode(file, data);
+
+        try (Reader reader = new InputStreamReader(vf.getInputStream(), vf.getCharset())) {
+            Json5Element input = JSON5.parse(reader);
+            if(input != null && input.isJson5Object()) {
+                Json5Mapper.read(file.getLocale(), input.getAsJson5Object(), targetNode);
+            }
+        }
+    }
+
+    @Override
+    public void write(@NotNull TranslationData data, @NotNull TranslationFile file) throws Exception {
+        TranslationNode targetNode = super.getTargetNode(data, file);
+
+        Json5Object output = new Json5Object();
+        Json5Mapper.write(file.getLocale(), output, Objects.requireNonNull(targetNode));
+
+        VirtualFile vf = file.getVirtualFile();
+        vf.setBinaryContent(JSON5.serialize(output).getBytes(vf.getCharset()));
+    }
+}

src/main/java/de/marhali/easyi18n/util/StringUtil.java

@@ -3,6 +3,7 @@ package de.marhali.easyi18n.util;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.StringWriter;
+import java.util.regex.Pattern;
 
 /**
  * String utilities
@@ -10,6 +11,17 @@ import java.io.StringWriter;
  */
 public class StringUtil {
 
+    /**
+     * Checks if the provided String represents a hexadecimal number.
+     * For example: {@code 0x100...}, {@code -0x100...} and {@code +0x100...}.
+     * @param string String to evaluate
+     * @return true if hexadecimal string otherwise false
+     */
+    public static boolean isHexString(@NotNull String string) {
+        final Pattern hexNumberPattern = Pattern.compile("[+-]?0[xX][0-9a-fA-F]+");
+        return hexNumberPattern.matcher(string).matches();
+    }
+
     /**
      * Escapes control characters for the given input string.
      * Inspired by Apache Commons (see {@link org.apache.commons.lang.StringEscapeUtils}

src/main/resources/messages.properties

@@ -17,7 +17,7 @@ settings.path.text=Locales directory
 settings.strategy.title=Translation file structure
 settings.strategy.folder=Single Directory;Modularized: Locale / Namespace;Modularized: Namespace / Locale
 settings.strategy.folder.tooltip=What is the folder structure of your translation files?
-settings.strategy.parser=JSON;YAML;YML;Properties;ARB
+settings.strategy.parser=JSON;JSON5;YAML;YML;Properties;ARB
 settings.strategy.parser.tooltip=Which file parser should be used to process your translation files?
 settings.strategy.file-pattern.tooltip=Defines a wildcard matcher to filter relevant translation files. For example *.json, *.???.
 settings.preview=Preview locale

src/test/java/de/marhali/easyi18n/mapper/Json5MapperTest.java

@@ -0,0 +1,160 @@
+package de.marhali.easyi18n.mapper;
+
+import de.marhali.easyi18n.io.parser.json.JsonArrayMapper;
+import de.marhali.easyi18n.io.parser.json5.Json5ArrayMapper;
+import de.marhali.easyi18n.io.parser.json5.Json5Mapper;
+import de.marhali.easyi18n.model.KeyPath;
+import de.marhali.easyi18n.model.TranslationData;
+import de.marhali.json5.Json5Object;
+import de.marhali.json5.Json5Primitive;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.junit.Assert;
+
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * Unit tests for {@link Json5Mapper}.
+ * @author marhali
+ */
+public class Json5MapperTest extends AbstractMapperTest {
+
+    @Override
+    public void testNonSorting() {
+        Json5Object input = new Json5Object();
+        input.add("zulu", Json5Primitive.of("test"));
+        input.add("alpha", Json5Primitive.of("test"));
+        input.add("bravo", Json5Primitive.of("test"));
+
+        TranslationData data = new TranslationData(false);
+        Json5Mapper.read("en", input, data.getRootNode());
+
+        Json5Object output = new Json5Object();
+        Json5Mapper.write("en", output, data.getRootNode());
+
+        Set<String> expect = new LinkedHashSet<>(Arrays.asList("zulu", "alpha", "bravo"));
+        Assert.assertEquals(expect, output.keySet());
+    }
+
+    @Override
+    public void testSorting() {
+        Json5Object input = new Json5Object();
+        input.add("zulu", Json5Primitive.of("test"));
+        input.add("alpha", Json5Primitive.of("test"));
+        input.add("bravo", Json5Primitive.of("test"));
+
+        TranslationData data = new TranslationData(false);
+        Json5Mapper.read("en", input, data.getRootNode());
+
+        Json5Object output = new Json5Object();
+        Json5Mapper.write("en", output, data.getRootNode());
+
+        Set<String> expect = new LinkedHashSet<>(Arrays.asList("alpha", "bravo", "zulu"));
+        Assert.assertEquals(expect, output.keySet());
+    }
+
+    @Override
+    public void testArrays() {
+        TranslationData data = new TranslationData(true);
+        data.setTranslation(KeyPath.of("simple"), create(arraySimple));
+        data.setTranslation(KeyPath.of("escaped"), create(arrayEscaped));
+
+        Json5Object output = new Json5Object();
+        Json5Mapper.write("en", output, data.getRootNode());
+
+        Assert.assertTrue(output.get("simple").isJson5Array());
+        Assert.assertEquals(arraySimple, Json5ArrayMapper.read(output.get("simple").getAsJson5Array()));
+        Assert.assertTrue(output.get("escaped").isJson5Array());
+        Assert.assertEquals(arrayEscaped, StringEscapeUtils.unescapeJava(Json5ArrayMapper.read(output.get("escaped").getAsJson5Array())));
+
+        TranslationData input = new TranslationData(true);
+        Json5Mapper.read("en", output, input.getRootNode());
+
+        Assert.assertTrue(JsonArrayMapper.isArray(input.getTranslation(KeyPath.of("simple")).get("en")));
+        Assert.assertTrue(JsonArrayMapper.isArray(input.getTranslation(KeyPath.of("escaped")).get("en")));
+    }
+
+    @Override
+    public void testSpecialCharacters() {
+        TranslationData data = new TranslationData(true);
+        data.setTranslation(KeyPath.of("chars"), create(specialCharacters));
+
+        Json5Object output = new Json5Object();
+        Json5Mapper.write("en", output, data.getRootNode());
+
+        Assert.assertEquals(specialCharacters, output.get("chars").getAsString());
+
+        TranslationData input = new TranslationData(true);
+        Json5Mapper.read("en", output, input.getRootNode());
+
+        Assert.assertEquals(specialCharacters,
+                StringEscapeUtils.unescapeJava(input.getTranslation(KeyPath.of("chars")).get("en")));
+    }
+
+    @Override
+    public void testNestedKeys() {
+        TranslationData data = new TranslationData(true);
+        data.setTranslation(KeyPath.of("nested", "key", "section"), create("test"));
+
+        Json5Object output = new Json5Object();
+        Json5Mapper.write("en", output, data.getRootNode());
+
+        Assert.assertEquals("test", output.getAsJson5Object("nested").getAsJson5Object("key").get("section").getAsString());
+
+        TranslationData input = new TranslationData(true);
+        Json5Mapper.read("en", output, input.getRootNode());
+
+        Assert.assertEquals("test", input.getTranslation(KeyPath.of("nested", "key", "section")).get("en"));
+    }
+
+    @Override
+    public void testNonNestedKeys() {
+        TranslationData data = new TranslationData(true);
+        data.setTranslation(KeyPath.of("long.key.with.many.sections"), create("test"));
+
+        Json5Object output = new Json5Object();
+        Json5Mapper.write("en", output, data.getRootNode());
+
+        Assert.assertTrue(output.has("long.key.with.many.sections"));
+
+        TranslationData input = new TranslationData(true);
+        Json5Mapper.read("en", output, input.getRootNode());
+
+        Assert.assertEquals("test", input.getTranslation(KeyPath.of("long.key.with.many.sections")).get("en"));
+    }
+
+    @Override
+    public void testLeadingSpace() {
+        TranslationData data = new TranslationData(true);
+        data.setTranslation(KeyPath.of("space"), create(leadingSpace));
+
+        Json5Object output = new Json5Object();
+        Json5Mapper.write("en", output, data.getRootNode());
+
+        Assert.assertEquals(leadingSpace, output.get("space").getAsString());
+
+        TranslationData input = new TranslationData(true);
+        Json5Mapper.read("en", output, input.getRootNode());
+
+        Assert.assertEquals(leadingSpace, input.getTranslation(KeyPath.of("space")).get("en"));
+    }
+
+    @Override
+    public void testNumbers() {
+        TranslationData data = new TranslationData(true);
+        data.setTranslation(KeyPath.of("numbered"), create("15000"));
+
+        Json5Object output = new Json5Object();
+        Json5Mapper.write("en", output, data.getRootNode());
+
+        Assert.assertEquals(15000, output.get("numbered").getAsNumber());
+
+        Json5Object input = new Json5Object();
+        input.addProperty("numbered", 143.23);
+        Json5Mapper.read("en", input, data.getRootNode());
+
+        Assert.assertEquals("143.23", data.getTranslation(KeyPath.of("numbered")).get("en"));
+    }
+}