001    /**
002     * Copyright (c) 2000-present Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portal.tools;
016    
017    import com.liferay.portal.kernel.io.OutputStreamWriter;
018    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedReader;
019    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedWriter;
020    import com.liferay.portal.kernel.io.unsync.UnsyncStringReader;
021    import com.liferay.portal.kernel.microsofttranslator.MicrosoftTranslatorException;
022    import com.liferay.portal.kernel.util.FileUtil;
023    import com.liferay.portal.kernel.util.GetterUtil;
024    import com.liferay.portal.kernel.util.NaturalOrderStringComparator;
025    import com.liferay.portal.kernel.util.PropertiesUtil;
026    import com.liferay.portal.kernel.util.StringBundler;
027    import com.liferay.portal.kernel.util.StringPool;
028    import com.liferay.portal.kernel.util.StringUtil;
029    import com.liferay.portal.kernel.util.Validator;
030    import com.liferay.portal.kernel.webcache.WebCacheItem;
031    import com.liferay.portlet.translator.model.Translation;
032    import com.liferay.portlet.translator.util.TranslationWebCacheItem;
033    
034    import java.io.File;
035    import java.io.FileInputStream;
036    import java.io.FileOutputStream;
037    import java.io.FileWriter;
038    import java.io.IOException;
039    import java.io.InputStream;
040    
041    import java.util.Map;
042    import java.util.Properties;
043    import java.util.TreeMap;
044    
045    /**
046     * @author Brian Wing Shun Chan
047     */
048    public class LangBuilder {
049    
050            public static final String AUTOMATIC_COPY = " (Automatic Copy)";
051    
052            public static final String AUTOMATIC_TRANSLATION =
053                    " (Automatic Translation)";
054    
055            public static void main(String[] args) {
056                    ToolDependencies.wireBasic();
057    
058                    Map<String, String> arguments = ArgumentsUtil.parseArguments(args);
059    
060                    System.setProperty("line.separator", StringPool.NEW_LINE);
061    
062                    String langDir = arguments.get("lang.dir");
063                    String langFile = arguments.get("lang.file");
064                    boolean langPlugin = GetterUtil.getBoolean(
065                            arguments.get("lang.plugin"));
066                    boolean langTranslate = GetterUtil.getBoolean(
067                            arguments.get("lang.translate"), true);
068    
069                    try {
070                            new LangBuilder(langDir, langFile, langPlugin, langTranslate);
071                    }
072                    catch (Exception e) {
073                            e.printStackTrace();
074                    }
075            }
076    
077            public LangBuilder(
078                            String langDir, String langFile, boolean langPlugin,
079                            boolean langTranslate)
080                    throws Exception {
081    
082                    _langDir = langDir;
083                    _langFile = langFile;
084                    _langTranslate = langTranslate;
085    
086                    if (langPlugin) {
087                            _portalLanguageProperties = new Properties();
088    
089                            Class<?> clazz = getClass();
090    
091                            ClassLoader classLoader = clazz.getClassLoader();
092    
093                            InputStream inputStream = classLoader.getResourceAsStream(
094                                    "content/Language.properties");
095    
096                            _portalLanguageProperties.load(inputStream);
097                    }
098    
099                    File renameKeysFile = new File(_langDir + "/rename.properties");
100    
101                    if (renameKeysFile.exists()) {
102                            _renameKeys = PropertiesUtil.load(FileUtil.read(renameKeysFile));
103                    }
104    
105                    String content = _orderProperties(
106                            new File(_langDir + "/" + _langFile + ".properties"));
107    
108                    // Locales that are not invoked by _createProperties should still be
109                    // rewritten to use the right line separator
110    
111                    _orderProperties(
112                            new File(_langDir + "/" + _langFile + "_en_AU.properties"));
113                    _orderProperties(
114                            new File(_langDir + "/" + _langFile + "_en_GB.properties"));
115                    _orderProperties(
116                            new File(_langDir + "/" + _langFile + "_fr_CA.properties"));
117    
118                    _createProperties(content, "ar"); // Arabic
119                    _createProperties(content, "eu"); // Basque
120                    _createProperties(content, "bg"); // Bulgarian
121                    _createProperties(content, "ca"); // Catalan
122                    _createProperties(content, "zh_CN"); // Chinese (China)
123                    _createProperties(content, "zh_TW"); // Chinese (Taiwan)
124                    _createProperties(content, "hr"); // Croatian
125                    _createProperties(content, "cs"); // Czech
126                    _createProperties(content, "da"); // Danish
127                    _createProperties(content, "nl"); // Dutch (Netherlands)
128                    _createProperties(content, "nl_BE", "nl"); // Dutch (Belgium)
129                    _createProperties(content, "et"); // Estonian
130                    _createProperties(content, "fi"); // Finnish
131                    _createProperties(content, "fr"); // French
132                    _createProperties(content, "gl"); // Galician
133                    _createProperties(content, "de"); // German
134                    _createProperties(content, "el"); // Greek
135                    _createProperties(content, "iw"); // Hebrew
136                    _createProperties(content, "hi_IN"); // Hindi (India)
137                    _createProperties(content, "hu"); // Hungarian
138                    _createProperties(content, "in"); // Indonesian
139                    _createProperties(content, "it"); // Italian
140                    _createProperties(content, "ja"); // Japanese
141                    _createProperties(content, "ko"); // Korean
142                    _createProperties(content, "lo"); // Lao
143                    _createProperties(content, "lt"); // Lithuanian
144                    _createProperties(content, "nb"); // Norwegian Bokm??l
145                    _createProperties(content, "fa"); // Persian
146                    _createProperties(content, "pl"); // Polish
147                    _createProperties(content, "pt_BR"); // Portuguese (Brazil)
148                    _createProperties(content, "pt_PT", "pt_BR"); // Portuguese (Portugal)
149                    _createProperties(content, "ro"); // Romanian
150                    _createProperties(content, "ru"); // Russian
151                    _createProperties(content, "sr_RS"); // Serbian (Cyrillic)
152                    _createProperties(content, "sr_RS_latin"); // Serbian (Latin)
153                    _createProperties(content, "sk"); // Slovak
154                    _createProperties(content, "sl"); // Slovene
155                    _createProperties(content, "es"); // Spanish
156                    _createProperties(content, "sv"); // Swedish
157                    _createProperties(content, "tr"); // Turkish
158                    _createProperties(content, "uk"); // Ukrainian
159                    _createProperties(content, "vi"); // Vietnamese
160            }
161    
162            private void _createProperties(String content, String languageId)
163                    throws IOException {
164    
165                    _createProperties(content, languageId, null);
166            }
167    
168            private void _createProperties(
169                            String content, String languageId, String parentLanguageId)
170                    throws IOException {
171    
172                    File propertiesFile = new File(
173                            _langDir + "/" + _langFile + "_" + languageId + ".properties");
174    
175                    Properties properties = new Properties();
176    
177                    if (propertiesFile.exists()) {
178                            properties = PropertiesUtil.load(
179                                    new FileInputStream(propertiesFile), StringPool.UTF8);
180                    }
181    
182                    Properties parentProperties = null;
183    
184                    if (parentLanguageId != null) {
185                            File parentPropertiesFile = new File(
186                                    _langDir + "/" + _langFile + "_" + parentLanguageId +
187                                            ".properties");
188    
189                            if (parentPropertiesFile.exists()) {
190                                    parentProperties = new Properties();
191    
192                                    parentProperties = PropertiesUtil.load(
193                                            new FileInputStream(parentPropertiesFile), StringPool.UTF8);
194                            }
195                    }
196    
197                    try (UnsyncBufferedReader unsyncBufferedReader =
198                                    new UnsyncBufferedReader(new UnsyncStringReader(content));
199                            UnsyncBufferedWriter unsyncBufferedWriter =
200                                    new UnsyncBufferedWriter(
201                                            new OutputStreamWriter(
202                                                    new FileOutputStream(propertiesFile),
203                                                    StringPool.UTF8))) {
204    
205                            boolean firstLine = true;
206                            int state = 0;
207    
208                            String line = null;
209    
210                            while ((line = unsyncBufferedReader.readLine()) != null) {
211                                    line = line.trim();
212    
213                                    int pos = line.indexOf("=");
214    
215                                    if (pos != -1) {
216                                            String key = line.substring(0, pos);
217                                            String value = line.substring(pos + 1);
218    
219                                            if (((state == 1) && !key.startsWith("lang.")) ||
220                                                    ((state == 2) && !key.startsWith("javax.portlet.")) ||
221                                                    ((state == 3) && !key.startsWith("category.")) ||
222                                                    ((state == 4) && !key.startsWith("model.resource.")) ||
223                                                    ((state == 5) && !key.startsWith("action.")) ||
224                                                    ((state == 7) && !key.startsWith("country.")) ||
225                                                    ((state == 8) && !key.startsWith("currency.")) ||
226                                                    ((state == 9) && !key.startsWith("language.")) ||
227                                                    ((state != 9) && key.startsWith("language."))) {
228    
229                                                    throw new RuntimeException(
230                                                            "File " + languageId + " with state " + state +
231                                                                    " has key " + key);
232                                            }
233    
234                                            String translatedText = properties.getProperty(key);
235    
236                                            if ((translatedText == null) && (parentProperties != null))
237                                            {
238                                                    translatedText = parentProperties.getProperty(key);
239                                            }
240    
241                                            if ((translatedText == null) && (_renameKeys != null)) {
242                                                    String renameKey = _renameKeys.getProperty(key);
243    
244                                                    if (renameKey != null) {
245                                                            translatedText = properties.getProperty(key);
246    
247                                                            if ((translatedText == null) &&
248                                                                    (parentProperties != null)) {
249    
250                                                                    translatedText = parentProperties.getProperty(
251                                                                            key);
252                                                            }
253                                                    }
254                                            }
255    
256                                            if (translatedText != null) {
257                                                    if (translatedText.contains("Babel Fish") ||
258                                                            translatedText.contains("Yahoo! - 999")) {
259    
260                                                            translatedText = "";
261                                                    }
262                                                    else if (translatedText.endsWith(AUTOMATIC_COPY)) {
263                                                            translatedText = value + AUTOMATIC_COPY;
264                                                    }
265                                            }
266    
267                                            if ((translatedText == null) || translatedText.equals("")) {
268                                                    if (line.contains("{") || line.contains("<")) {
269                                                            translatedText = value + AUTOMATIC_COPY;
270                                                    }
271                                                    else if (line.contains("[")) {
272                                                            pos = line.indexOf("[");
273    
274                                                            String baseKey = line.substring(0, pos);
275    
276                                                            String translatedBaseKey = properties.getProperty(
277                                                                    baseKey);
278    
279                                                            if (Validator.isNotNull(translatedBaseKey)) {
280                                                                    translatedText = translatedBaseKey;
281                                                            }
282                                                            else {
283                                                                    translatedText = value + AUTOMATIC_COPY;
284                                                            }
285                                                    }
286                                                    else if (key.equals("lang.dir")) {
287                                                            translatedText = "ltr";
288                                                    }
289                                                    else if (key.equals("lang.line.begin")) {
290                                                            translatedText = "left";
291                                                    }
292                                                    else if (key.equals("lang.line.end")) {
293                                                            translatedText = "right";
294                                                    }
295                                                    else if (languageId.equals("el") &&
296                                                                     (key.equals("enabled") || key.equals("on") ||
297                                                                      key.equals("on-date"))) {
298    
299                                                            translatedText = "";
300                                                    }
301                                                    else if (languageId.equals("es") && key.equals("am")) {
302                                                            translatedText = "";
303                                                    }
304                                                    else if (languageId.equals("fi") &&
305                                                                     (key.equals("on") || key.equals("the"))) {
306    
307                                                            translatedText = "";
308                                                    }
309                                                    else if (languageId.equals("it") && key.equals("am")) {
310                                                            translatedText = "";
311                                                    }
312                                                    else if (languageId.equals("ja") &&
313                                                                     (key.equals("any") || key.equals("anytime") ||
314                                                                      key.equals("down") || key.equals("on") ||
315                                                                      key.equals("on-date") || key.equals("the"))) {
316    
317                                                            translatedText = "";
318                                                    }
319                                                    else if (languageId.equals("ko") && key.equals("the")) {
320                                                            translatedText = "";
321                                                    }
322                                                    else {
323                                                            translatedText = _translate(
324                                                                    "en", languageId, key, value, 0);
325    
326                                                            if (Validator.isNull(translatedText)) {
327                                                                    translatedText = value + AUTOMATIC_COPY;
328                                                            }
329                                                            else if (!key.startsWith("country.") &&
330                                                                             !key.startsWith("language.")) {
331    
332                                                                    translatedText =
333                                                                            translatedText + AUTOMATIC_TRANSLATION;
334                                                            }
335                                                    }
336                                            }
337    
338                                            if (Validator.isNotNull(translatedText)) {
339                                                    if (translatedText.contains("Babel Fish") ||
340                                                            translatedText.contains("Yahoo! - 999")) {
341    
342                                                            throw new IOException(
343                                                                    "IP was blocked because of over usage. " +
344                                                                            "Please use another IP.");
345                                                    }
346    
347                                                    translatedText = _fixTranslation(translatedText);
348    
349                                                    if (firstLine) {
350                                                            firstLine = false;
351                                                    }
352                                                    else {
353                                                            unsyncBufferedWriter.newLine();
354                                                    }
355    
356                                                    unsyncBufferedWriter.write(key + "=" + translatedText);
357    
358                                                    unsyncBufferedWriter.flush();
359                                            }
360                                    }
361                                    else {
362                                            if (line.startsWith("## Language settings")) {
363                                                    if (state == 1) {
364                                                            throw new RuntimeException(languageId);
365                                                    }
366    
367                                                    state = 1;
368                                            }
369                                            else if (line.startsWith(
370                                                                    "## Portlet descriptions and titles")) {
371    
372                                                    if (state == 2) {
373                                                            throw new RuntimeException(languageId);
374                                                    }
375    
376                                                    state = 2;
377                                            }
378                                            else if (line.startsWith("## Category titles")) {
379                                                    if (state == 3) {
380                                                            throw new RuntimeException(languageId);
381                                                    }
382    
383                                                    state = 3;
384                                            }
385                                            else if (line.startsWith("## Model resources")) {
386                                                    if (state == 4) {
387                                                            throw new RuntimeException(languageId);
388                                                    }
389    
390                                                    state = 4;
391                                            }
392                                            else if (line.startsWith("## Action names")) {
393                                                    if (state == 5) {
394                                                            throw new RuntimeException(languageId);
395                                                    }
396    
397                                                    state = 5;
398                                            }
399                                            else if (line.startsWith("## Messages")) {
400                                                    if (state == 6) {
401                                                            throw new RuntimeException(languageId);
402                                                    }
403    
404                                                    state = 6;
405                                            }
406                                            else if (line.startsWith("## Country")) {
407                                                    if (state == 7) {
408                                                            throw new RuntimeException(languageId);
409                                                    }
410    
411                                                    state = 7;
412                                            }
413                                            else if (line.startsWith("## Currency")) {
414                                                    if (state == 8) {
415                                                            throw new RuntimeException(languageId);
416                                                    }
417    
418                                                    state = 8;
419                                            }
420                                            else if (line.startsWith("## Language")) {
421                                                    if (state == 9) {
422                                                            throw new RuntimeException(languageId);
423                                                    }
424    
425                                                    state = 9;
426                                            }
427    
428                                            if (firstLine) {
429                                                    firstLine = false;
430                                            }
431                                            else {
432                                                    unsyncBufferedWriter.newLine();
433                                            }
434    
435                                            unsyncBufferedWriter.write(line);
436    
437                                            unsyncBufferedWriter.flush();
438                                    }
439                            }
440                    }
441            }
442    
443            private String _fixEnglishTranslation(String key, String value) {
444    
445                    // http://en.wikibooks.org/wiki/Basic_Book_Design/Capitalizing_Words_in_Titles
446                    // http://titlecapitalization.com
447                    // http://www.imdb.com
448    
449                    if (value.contains(" this ")) {
450                            if (value.contains(".") || value.contains("?") ||
451                                    value.contains(":") ||
452                                    key.equals("the-url-of-the-page-comparing-this-page-content-with-the-previous-version")) {
453                            }
454                            else {
455                                    value = StringUtil.replace(value, " this ", " This ");
456                            }
457                    }
458                    else {
459                            value = StringUtil.replace(value, " From ", " from ");
460                    }
461    
462                    return value;
463            }
464    
465            private String _fixTranslation(String value) {
466                    value = StringUtil.replace(
467                            value.trim(),
468                            new String[] {
469                                    "  ", "<b>", "</b>", "<i>", "</i>", " url ", "&#39;", "&#39 ;",
470                                    "&quot;", "&quot ;", "ReCaptcha", "Captcha"
471                            },
472                            new String[] {
473                                    " ", "<strong>", "</strong>", "<em>", "</em>", " URL ", "\'",
474                                    "\'", "\"", "\"", "reCAPTCHA", "CAPTCHA"
475                            });
476    
477                    return value;
478            }
479    
480            private String _orderProperties(File propertiesFile) throws IOException {
481                    if (!propertiesFile.exists()) {
482                            return null;
483                    }
484    
485                    String content = FileUtil.read(propertiesFile);
486    
487                    try (UnsyncBufferedReader unsyncBufferedReader =
488                            new UnsyncBufferedReader(new UnsyncStringReader(content));
489                            UnsyncBufferedWriter unsyncBufferedWriter =
490                                    new UnsyncBufferedWriter(new FileWriter(propertiesFile))) {
491    
492                            Map<String, String> messages = new TreeMap<String, String>(
493                                    new NaturalOrderStringComparator(true, true));
494    
495                            boolean begin = false;
496                            boolean firstLine = true;
497    
498                            String line = null;
499    
500                            while ((line = unsyncBufferedReader.readLine()) != null) {
501                                    int pos = line.indexOf("=");
502    
503                                    if (pos != -1) {
504                                            String key = line.substring(0, pos);
505    
506                                            String value = line.substring(pos + 1);
507    
508                                            if (Validator.isNotNull(value)) {
509                                                    value = _fixTranslation(line.substring(pos + 1));
510    
511                                                    value = _fixEnglishTranslation(key, value);
512    
513                                                    if (_portalLanguageProperties != null) {
514                                                            String portalValue = String.valueOf(
515                                                                    _portalLanguageProperties.get(key));
516    
517                                                            if (value.equals(portalValue)) {
518                                                                    System.out.println("Duplicate key " + key);
519                                                            }
520                                                    }
521    
522                                                    messages.put(key, value);
523                                            }
524                                    }
525                                    else {
526                                            if (begin && line.equals(StringPool.BLANK)) {
527                                                    _sortAndWrite(
528                                                            unsyncBufferedWriter, messages, firstLine);
529                                            }
530    
531                                            if (line.equals(StringPool.BLANK)) {
532                                                    begin = !begin;
533                                            }
534    
535                                            if (firstLine) {
536                                                    firstLine = false;
537                                            }
538                                            else {
539                                                    unsyncBufferedWriter.newLine();
540                                            }
541    
542                                            unsyncBufferedWriter.write(line);
543                                    }
544    
545                                    unsyncBufferedWriter.flush();
546                            }
547    
548                            if (!messages.isEmpty()) {
549                                    _sortAndWrite(unsyncBufferedWriter, messages, firstLine);
550                            }
551                    }
552    
553                    return FileUtil.read(propertiesFile);
554            }
555    
556            private void _sortAndWrite(
557                            UnsyncBufferedWriter unsyncBufferedWriter,
558                            Map<String, String> messages, boolean firstLine)
559                    throws IOException {
560    
561                    boolean firstEntry = true;
562    
563                    for (Map.Entry<String, String> entry : messages.entrySet()) {
564                            if (!firstLine || !firstEntry) {
565                                    unsyncBufferedWriter.newLine();
566                            }
567    
568                            firstEntry = false;
569    
570                            unsyncBufferedWriter.write(entry.getKey() + "=" + entry.getValue());
571                    }
572    
573                    messages.clear();
574            }
575    
576            private String _translate(
577                    String fromLanguageId, String toLanguageId, String key, String fromText,
578                    int limit) {
579    
580                    if (toLanguageId.equals("ar") ||
581                            toLanguageId.equals("eu") ||
582                            toLanguageId.equals("bg") ||
583                            toLanguageId.equals("ca") ||
584                            toLanguageId.equals("hr") ||
585                            toLanguageId.equals("cs") ||
586                            toLanguageId.equals("da") ||
587                            toLanguageId.equals("et") ||
588                            toLanguageId.equals("fi") ||
589                            toLanguageId.equals("gl") ||
590    
591                            // LPS-26741
592    
593                            toLanguageId.equals("de") ||
594    
595                            toLanguageId.equals("iw") ||
596                            toLanguageId.equals("hi") ||
597                            toLanguageId.equals("hu") ||
598                            toLanguageId.equals("in") ||
599                            toLanguageId.equals("lo") ||
600                            toLanguageId.equals("lt") ||
601                            toLanguageId.equals("nb") ||
602                            toLanguageId.equals("fa") ||
603                            toLanguageId.equals("pl") ||
604                            toLanguageId.equals("ro") ||
605                            toLanguageId.equals("ru") ||
606                            toLanguageId.equals("sr_RS") ||
607                            toLanguageId.equals("sr_RS_latin") ||
608                            toLanguageId.equals("sk") ||
609                            toLanguageId.equals("sl") ||
610                            toLanguageId.equals("sv") ||
611                            toLanguageId.equals("tr") ||
612                            toLanguageId.equals("uk") ||
613                            toLanguageId.equals("vi")) {
614    
615                            // Automatic translator does not support Arabic, Basque, Bulgarian,
616                            // Catalan, Croatian, Czech, Danish, Estonian, Finnish, Galician,
617                            // German, Hebrew, Hindi, Hungarian, Indonesian, Lao, Norwegian
618                            // Bokm??l, Persian, Polish, Romanian, Russian, Serbian, Slovak,
619                            // Slovene, Swedish, Turkish, Ukrainian, or Vietnamese
620    
621                            return null;
622                    }
623    
624                    if (!_langTranslate) {
625                            return null;
626                    }
627    
628                    // Limit the number of retries to 3
629    
630                    if (limit == 3) {
631                            return null;
632                    }
633    
634                    String toText = null;
635    
636                    try {
637                            StringBundler sb = new StringBundler(8);
638    
639                            sb.append("Translating ");
640                            sb.append(fromLanguageId);
641                            sb.append("_");
642                            sb.append(toLanguageId);
643                            sb.append(" ");
644                            sb.append(key);
645                            sb.append(" ");
646                            sb.append(fromText);
647    
648                            System.out.println(sb.toString());
649    
650                            WebCacheItem wci = new TranslationWebCacheItem(
651                                    fromLanguageId, toLanguageId, fromText);
652    
653                            Translation translation = (Translation)wci.convert("");
654    
655                            toText = translation.getToText();
656                    }
657                    catch (Exception e) {
658                            Throwable cause = e.getCause();
659    
660                            if (cause instanceof MicrosoftTranslatorException) {
661                                    System.out.println(
662                                            cause.getClass().getName() + ": " + cause.getMessage());
663                            }
664                            else {
665                                    e.printStackTrace();
666                            }
667                    }
668    
669                    // Keep trying
670    
671                    if (toText == null) {
672                            return _translate(
673                                    fromLanguageId, toLanguageId, key, fromText, ++limit);
674                    }
675    
676                    return toText;
677            }
678    
679            private String _langDir;
680            private String _langFile;
681            private boolean _langTranslate;
682            private Properties _portalLanguageProperties;
683            private Properties _renameKeys;
684    
685    }