001    /**
002     * Copyright (c) 2000-2013 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portal.tools;
016    
017    import com.liferay.portal.kernel.io.OutputStreamWriter;
018    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedReader;
019    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedWriter;
020    import com.liferay.portal.kernel.io.unsync.UnsyncStringReader;
021    import com.liferay.portal.kernel.microsofttranslator.MicrosoftTranslatorException;
022    import com.liferay.portal.kernel.util.FileUtil;
023    import com.liferay.portal.kernel.util.GetterUtil;
024    import com.liferay.portal.kernel.util.NumericalStringComparator;
025    import com.liferay.portal.kernel.util.PropertiesUtil;
026    import com.liferay.portal.kernel.util.StringBundler;
027    import com.liferay.portal.kernel.util.StringPool;
028    import com.liferay.portal.kernel.util.StringUtil;
029    import com.liferay.portal.kernel.util.Validator;
030    import com.liferay.portal.kernel.webcache.WebCacheItem;
031    import com.liferay.portal.util.InitUtil;
032    import com.liferay.portlet.translator.model.Translation;
033    import com.liferay.portlet.translator.util.TranslationWebCacheItem;
034    
035    import java.io.File;
036    import java.io.FileInputStream;
037    import java.io.FileOutputStream;
038    import java.io.FileWriter;
039    import java.io.IOException;
040    import java.io.InputStream;
041    
042    import java.util.Map;
043    import java.util.Properties;
044    import java.util.TreeMap;
045    
046    /**
047     * @author Brian Wing Shun Chan
048     */
049    public class LangBuilder {
050    
051            public static final String AUTOMATIC_COPY = " (Automatic Copy)";
052    
053            public static final String AUTOMATIC_TRANSLATION =
054                    " (Automatic Translation)";
055    
056            public static void main(String[] args) {
057                    Map<String, String> arguments = ArgumentsUtil.parseArguments(args);
058    
059                    System.setProperty("line.separator", StringPool.NEW_LINE);
060    
061                    InitUtil.initWithSpring();
062    
063                    String langDir = arguments.get("lang.dir");
064                    String langFile = arguments.get("lang.file");
065                    boolean langPlugin = GetterUtil.getBoolean(
066                            arguments.get("lang.plugin"));
067                    boolean langTranslate = GetterUtil.getBoolean(
068                            arguments.get("lang.translate"), true);
069    
070                    try {
071                            new LangBuilder(langDir, langFile, langPlugin, langTranslate);
072                    }
073                    catch (Exception e) {
074                            e.printStackTrace();
075                    }
076            }
077    
078            public LangBuilder(
079                            String langDir, String langFile, boolean langPlugin,
080                            boolean langTranslate)
081                    throws Exception {
082    
083                    _langDir = langDir;
084                    _langFile = langFile;
085                    _langTranslate = langTranslate;
086    
087                    if (langPlugin) {
088                            _portalLanguageProperties = new Properties();
089    
090                            Class<?> clazz = getClass();
091    
092                            ClassLoader classLoader = clazz.getClassLoader();
093    
094                            InputStream inputStream = classLoader.getResourceAsStream(
095                                    "content/Language.properties");
096    
097                            _portalLanguageProperties.load(inputStream);
098                    }
099    
100                    File renameKeysFile = new File(_langDir + "/rename.properties");
101    
102                    if (renameKeysFile.exists()) {
103                            _renameKeys = PropertiesUtil.load(FileUtil.read(renameKeysFile));
104                    }
105    
106                    String content = _orderProperties(
107                            new File(_langDir + "/" + _langFile + ".properties"));
108    
109                    // Locales that are not invoked by _createProperties should still be
110                    // rewritten to use the right line separator
111    
112                    _orderProperties(
113                            new File(_langDir + "/" + _langFile + "_en_AU.properties"));
114                    _orderProperties(
115                            new File(_langDir + "/" + _langFile + "_en_GB.properties"));
116                    _orderProperties(
117                            new File(_langDir + "/" + _langFile + "_fr_CA.properties"));
118    
119                    _createProperties(content, "ar"); // Arabic
120                    _createProperties(content, "eu"); // Basque
121                    _createProperties(content, "bg"); // Bulgarian
122                    _createProperties(content, "ca"); // Catalan
123                    _createProperties(content, "zh_CN"); // Chinese (China)
124                    _createProperties(content, "zh_TW"); // Chinese (Taiwan)
125                    _createProperties(content, "hr"); // Croatian
126                    _createProperties(content, "cs"); // Czech
127                    _createProperties(content, "da"); // Danish
128                    _createProperties(content, "nl"); // Dutch (Netherlands)
129                    _createProperties(content, "nl_BE", "nl"); // Dutch (Belgium)
130                    _createProperties(content, "et"); // Estonian
131                    _createProperties(content, "fi"); // Finnish
132                    _createProperties(content, "fr"); // French
133                    _createProperties(content, "gl"); // Galician
134                    _createProperties(content, "de"); // German
135                    _createProperties(content, "el"); // Greek
136                    _createProperties(content, "iw"); // Hebrew
137                    _createProperties(content, "hi_IN"); // Hindi (India)
138                    _createProperties(content, "hu"); // Hungarian
139                    _createProperties(content, "in"); // Indonesian
140                    _createProperties(content, "it"); // Italian
141                    _createProperties(content, "ja"); // Japanese
142                    _createProperties(content, "ko"); // Korean
143                    _createProperties(content, "lo"); // Lao
144                    _createProperties(content, "lt"); // Lithuanian
145                    _createProperties(content, "nb"); // Norwegian Bokm??l
146                    _createProperties(content, "fa"); // Persian
147                    _createProperties(content, "pl"); // Polish
148                    _createProperties(content, "pt_BR"); // Portuguese (Brazil)
149                    _createProperties(content, "pt_PT", "pt_BR"); // Portuguese (Portugal)
150                    _createProperties(content, "ro"); // Romanian
151                    _createProperties(content, "ru"); // Russian
152                    _createProperties(content, "sr_RS"); // Serbian (Cyrillic)
153                    _createProperties(content, "sr_RS_latin"); // Serbian (Latin)
154                    _createProperties(content, "sk"); // Slovak
155                    _createProperties(content, "sl"); // Slovene
156                    _createProperties(content, "es"); // Spanish
157                    _createProperties(content, "sv"); // Swedish
158                    _createProperties(content, "tr"); // Turkish
159                    _createProperties(content, "uk"); // Ukrainian
160                    _createProperties(content, "vi"); // Vietnamese
161            }
162    
163            private void _createProperties(String content, String languageId)
164                    throws IOException {
165    
166                    _createProperties(content, languageId, null);
167            }
168    
169            private void _createProperties(
170                            String content, String languageId, String parentLanguageId)
171                    throws IOException {
172    
173                    File propertiesFile = new File(
174                            _langDir + "/" + _langFile + "_" + languageId + ".properties");
175    
176                    Properties properties = new Properties();
177    
178                    if (propertiesFile.exists()) {
179                            properties = PropertiesUtil.load(
180                                    new FileInputStream(propertiesFile), StringPool.UTF8);
181                    }
182    
183                    Properties parentProperties = null;
184    
185                    if (parentLanguageId != null) {
186                            File parentPropertiesFile = new File(
187                                    _langDir + "/" + _langFile + "_" + parentLanguageId +
188                                            ".properties");
189    
190                            if (parentPropertiesFile.exists()) {
191                                    parentProperties = new Properties();
192    
193                                    parentProperties = PropertiesUtil.load(
194                                            new FileInputStream(parentPropertiesFile), StringPool.UTF8);
195                            }
196                    }
197    
198                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
199                            new UnsyncStringReader(content));
200                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
201                            new OutputStreamWriter(
202                                    new FileOutputStream(propertiesFile), StringPool.UTF8));
203    
204                    boolean firstLine = true;
205                    int state = 0;
206    
207                    String line = null;
208    
209                    while ((line = unsyncBufferedReader.readLine()) != null) {
210                            line = line.trim();
211    
212                            int pos = line.indexOf("=");
213    
214                            if (pos != -1) {
215                                    String key = line.substring(0, pos);
216                                    String value = line.substring(pos + 1);
217    
218                                    if (((state == 1) && !key.startsWith("lang.")) ||
219                                            ((state == 2) && !key.startsWith("javax.portlet.")) ||
220                                            ((state == 3) && !key.startsWith("category.")) ||
221                                            ((state == 4) && !key.startsWith("model.resource.")) ||
222                                            ((state == 5) && !key.startsWith("action.")) ||
223                                            ((state == 7) && !key.startsWith("country.")) ||
224                                            ((state == 8) && !key.startsWith("currency.")) ||
225                                            ((state == 9) && !key.startsWith("language.")) ||
226                                            ((state != 9) && key.startsWith("language."))) {
227    
228                                            throw new RuntimeException(
229                                                    "File " + languageId + " with state " + state +
230                                                            " has key " + key);
231                                    }
232    
233                                    String translatedText = properties.getProperty(key);
234    
235                                    if ((translatedText == null) && (parentProperties != null)) {
236                                            translatedText = parentProperties.getProperty(key);
237                                    }
238    
239                                    if ((translatedText == null) && (_renameKeys != null)) {
240                                            String renameKey = _renameKeys.getProperty(key);
241    
242                                            if (renameKey != null) {
243                                                    translatedText = properties.getProperty(key);
244    
245                                                    if ((translatedText == null) &&
246                                                            (parentProperties != null)) {
247    
248                                                            translatedText = parentProperties.getProperty(key);
249                                                    }
250                                            }
251                                    }
252    
253                                    if (translatedText != null) {
254                                            if (translatedText.contains("Babel Fish") ||
255                                                    translatedText.contains("Yahoo! - 999")) {
256    
257                                                    translatedText = "";
258                                            }
259                                            else if (translatedText.endsWith(AUTOMATIC_COPY)) {
260                                                    translatedText = value + AUTOMATIC_COPY;
261                                            }
262                                    }
263    
264                                    if ((translatedText == null) || translatedText.equals("")) {
265                                            if (line.contains("{") || line.contains("<")) {
266                                                    translatedText = value + AUTOMATIC_COPY;
267                                            }
268                                            else if (line.contains("[")) {
269                                                    pos = line.indexOf("[");
270    
271                                                    String baseKey = line.substring(0, pos);
272    
273                                                    String translatedBaseKey = properties.getProperty(
274                                                            baseKey);
275    
276                                                    if (Validator.isNotNull(translatedBaseKey)) {
277                                                            translatedText = translatedBaseKey + AUTOMATIC_COPY;
278                                                    }
279                                                    else {
280                                                            translatedText = value + AUTOMATIC_COPY;
281                                                    }
282                                            }
283                                            else if (key.equals("lang.dir")) {
284                                                    translatedText = "ltr";
285                                            }
286                                            else if (key.equals("lang.line.begin")) {
287                                                    translatedText = "left";
288                                            }
289                                            else if (key.equals("lang.line.end")) {
290                                                    translatedText = "right";
291                                            }
292                                            else if (languageId.equals("el") &&
293                                                             (key.equals("enabled") || key.equals("on") ||
294                                                              key.equals("on-date"))) {
295    
296                                                    translatedText = "";
297                                            }
298                                            else if (languageId.equals("es") && key.equals("am")) {
299                                                    translatedText = "";
300                                            }
301                                            else if (languageId.equals("it") && key.equals("am")) {
302                                                    translatedText = "";
303                                            }
304                                            else if (languageId.equals("ja") &&
305                                                             (key.equals("any") || key.equals("anytime") ||
306                                                              key.equals("down") || key.equals("on") ||
307                                                              key.equals("on-date") || key.equals("the"))) {
308    
309                                                    translatedText = "";
310                                            }
311                                            else if (languageId.equals("ko") && key.equals("the")) {
312                                                    translatedText = "";
313                                            }
314                                            else {
315                                                    translatedText = _translate(
316                                                            "en", languageId, key, value, 0);
317    
318                                                    if (Validator.isNull(translatedText)) {
319                                                            translatedText = value + AUTOMATIC_COPY;
320                                                    }
321                                                    else if (!key.startsWith("country.") &&
322                                                                     !key.startsWith("language.")) {
323    
324                                                            translatedText =
325                                                                    translatedText + AUTOMATIC_TRANSLATION;
326                                                    }
327                                            }
328                                    }
329    
330                                    if (Validator.isNotNull(translatedText)) {
331                                            if (translatedText.contains("Babel Fish") ||
332                                                    translatedText.contains("Yahoo! - 999")) {
333    
334                                                    throw new IOException(
335                                                            "IP was blocked because of over usage. Please " +
336                                                                    "use another IP.");
337                                            }
338    
339                                            translatedText = _fixTranslation(translatedText);
340    
341                                            if (firstLine) {
342                                                    firstLine = false;
343                                            }
344                                            else {
345                                                    unsyncBufferedWriter.newLine();
346                                            }
347    
348                                            unsyncBufferedWriter.write(key + "=" + translatedText);
349    
350                                            unsyncBufferedWriter.flush();
351                                    }
352                            }
353                            else {
354                                    if (line.startsWith("## Language settings")) {
355                                            if (state == 1) {
356                                                    throw new RuntimeException(languageId);
357                                            }
358    
359                                            state = 1;
360                                    }
361                                    else if (line.startsWith(
362                                                            "## Portlet descriptions and titles")) {
363    
364                                            if (state == 2) {
365                                                    throw new RuntimeException(languageId);
366                                            }
367    
368                                            state = 2;
369                                    }
370                                    else if (line.startsWith("## Category titles")) {
371                                            if (state == 3) {
372                                                    throw new RuntimeException(languageId);
373                                            }
374    
375                                            state = 3;
376                                    }
377                                    else if (line.startsWith("## Model resources")) {
378                                            if (state == 4) {
379                                                    throw new RuntimeException(languageId);
380                                            }
381    
382                                            state = 4;
383                                    }
384                                    else if (line.startsWith("## Action names")) {
385                                            if (state == 5) {
386                                                    throw new RuntimeException(languageId);
387                                            }
388    
389                                            state = 5;
390                                    }
391                                    else if (line.startsWith("## Messages")) {
392                                            if (state == 6) {
393                                                    throw new RuntimeException(languageId);
394                                            }
395    
396                                            state = 6;
397                                    }
398                                    else if (line.startsWith("## Country")) {
399                                            if (state == 7) {
400                                                    throw new RuntimeException(languageId);
401                                            }
402    
403                                            state = 7;
404                                    }
405                                    else if (line.startsWith("## Currency")) {
406                                            if (state == 8) {
407                                                    throw new RuntimeException(languageId);
408                                            }
409    
410                                            state = 8;
411                                    }
412                                    else if (line.startsWith("## Language")) {
413                                            if (state == 9) {
414                                                    throw new RuntimeException(languageId);
415                                            }
416    
417                                            state = 9;
418                                    }
419    
420                                    if (firstLine) {
421                                            firstLine = false;
422                                    }
423                                    else {
424                                            unsyncBufferedWriter.newLine();
425                                    }
426    
427                                    unsyncBufferedWriter.write(line);
428    
429                                    unsyncBufferedWriter.flush();
430                            }
431                    }
432    
433                    unsyncBufferedReader.close();
434                    unsyncBufferedWriter.close();
435            }
436    
437            private String _fixEnglishTranslation(String key, String value) {
438    
439                    // http://en.wikibooks.org/wiki/Basic_Book_Design/Capitalizing_Words_in_Titles
440                    // http://www.imdb.com
441    
442                    if (value.contains(" this ")) {
443                            if (value.contains(".") || value.contains("?") ||
444                                    value.contains(":") ||
445                                    key.equals("the-url-of-the-page-comparing-this-page-content-with-the-previous-version")) {
446                            }
447                            else {
448                                    value = StringUtil.replace(value, " this ", " This ");
449                            }
450                    }
451                    else {
452                            value = StringUtil.replace(value, " From ", " from ");
453                    }
454    
455                    return value;
456            }
457    
458            private String _fixTranslation(String value) {
459                    value = StringUtil.replace(
460                            value.trim(),
461                            new String[] {
462                                    "  ", "<b>", "</b>", "<i>", "</i>", " url ", "&#39;", "&#39 ;",
463                                    "&quot;", "&quot ;", "ReCaptcha", "Captcha"
464                            },
465                            new String[] {
466                                    " ", "<strong>", "</strong>", "<em>", "</em>", " URL ", "\'",
467                                    "\'", "\"", "\"", "reCAPTCHA", "CAPTCHA"
468                            });
469    
470                    return value;
471            }
472    
473            private String _orderProperties(File propertiesFile) throws IOException {
474                    if (!propertiesFile.exists()) {
475                            return null;
476                    }
477    
478                    String content = FileUtil.read(propertiesFile);
479    
480                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
481                            new UnsyncStringReader(content));
482                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
483                            new FileWriter(propertiesFile));
484    
485                    Map<String, String> messages = new TreeMap<String, String>(
486                            new NumericalStringComparator(true, true));
487    
488                    boolean begin = false;
489                    boolean firstLine = true;
490    
491                    String line = null;
492    
493                    while ((line = unsyncBufferedReader.readLine()) != null) {
494                            int pos = line.indexOf("=");
495    
496                            if (pos != -1) {
497                                    String key = line.substring(0, pos);
498    
499                                    String value = _fixTranslation(line.substring(pos + 1));
500    
501                                    value = _fixEnglishTranslation(key, value);
502    
503                                    if (_portalLanguageProperties != null) {
504                                            String portalValue = String.valueOf(
505                                                    _portalLanguageProperties.get(key));
506    
507                                            if (value.equals(portalValue)) {
508                                                    System.out.println("Duplicate key " + key);
509                                            }
510                                    }
511    
512                                    messages.put(key, value);
513                            }
514                            else {
515                                    if (begin && line.equals(StringPool.BLANK)) {
516                                            _sortAndWrite(unsyncBufferedWriter, messages, firstLine);
517                                    }
518    
519                                    if (line.equals(StringPool.BLANK)) {
520                                            begin = !begin;
521                                    }
522    
523                                    if (firstLine) {
524                                            firstLine = false;
525                                    }
526                                    else {
527                                            unsyncBufferedWriter.newLine();
528                                    }
529    
530                                    unsyncBufferedWriter.write(line);
531                            }
532    
533                            unsyncBufferedWriter.flush();
534                    }
535    
536                    if (!messages.isEmpty()) {
537                            _sortAndWrite(unsyncBufferedWriter, messages, firstLine);
538                    }
539    
540                    unsyncBufferedReader.close();
541                    unsyncBufferedWriter.close();
542    
543                    return FileUtil.read(propertiesFile);
544            }
545    
546            private void _sortAndWrite(
547                            UnsyncBufferedWriter unsyncBufferedWriter,
548                            Map<String, String> messages, boolean firstLine)
549                    throws IOException {
550    
551                    boolean firstEntry = true;
552    
553                    for (Map.Entry<String, String> entry : messages.entrySet()) {
554                            if (!firstLine || !firstEntry) {
555                                    unsyncBufferedWriter.newLine();
556                            }
557    
558                            firstEntry = false;
559    
560                            unsyncBufferedWriter.write(entry.getKey() + "=" + entry.getValue());
561                    }
562    
563                    messages.clear();
564            }
565    
566            private String _translate(
567                    String fromLanguageId, String toLanguageId, String key, String fromText,
568                    int limit) {
569    
570                    if (toLanguageId.equals("ar") ||
571                            toLanguageId.equals("eu") ||
572                            toLanguageId.equals("bg") ||
573                            toLanguageId.equals("ca") ||
574                            toLanguageId.equals("hr") ||
575                            toLanguageId.equals("cs") ||
576                            toLanguageId.equals("da") ||
577                            toLanguageId.equals("et") ||
578                            toLanguageId.equals("fi") ||
579                            toLanguageId.equals("gl") ||
580    
581                            // LPS-26741
582    
583                            toLanguageId.equals("de") ||
584    
585                            toLanguageId.equals("iw") ||
586                            toLanguageId.equals("hi") ||
587                            toLanguageId.equals("hu") ||
588                            toLanguageId.equals("in") ||
589                            toLanguageId.equals("lo") ||
590                            toLanguageId.equals("lt") ||
591                            toLanguageId.equals("nb") ||
592                            toLanguageId.equals("fa") ||
593                            toLanguageId.equals("pl") ||
594                            toLanguageId.equals("ro") ||
595                            toLanguageId.equals("ru") ||
596                            toLanguageId.equals("sr_RS") ||
597                            toLanguageId.equals("sr_RS_latin") ||
598                            toLanguageId.equals("sk") ||
599                            toLanguageId.equals("sl") ||
600                            toLanguageId.equals("sv") ||
601                            toLanguageId.equals("tr") ||
602                            toLanguageId.equals("uk") ||
603                            toLanguageId.equals("vi")) {
604    
605                            // Automatic translator does not support Arabic, Basque, Bulgarian,
606                            // Catalan, Croatian, Czech, Danish, Estonian, Finnish, Galician,
607                            // German, Hebrew, Hindi, Hungarian, Indonesian, Lao, Norwegian
608                            // Bokm??l, Persian, Polish, Romanian, Russian, Serbian, Slovak,
609                            // Slovene, Swedish, Turkish, Ukrainian, or Vietnamese
610    
611                            return null;
612                    }
613    
614                    if (!_langTranslate) {
615                            return null;
616                    }
617    
618                    // Limit the number of retries to 3
619    
620                    if (limit == 3) {
621                            return null;
622                    }
623    
624                    String toText = null;
625    
626                    try {
627                            StringBundler sb = new StringBundler(8);
628    
629                            sb.append("Translating ");
630                            sb.append(fromLanguageId);
631                            sb.append("_");
632                            sb.append(toLanguageId);
633                            sb.append(" ");
634                            sb.append(key);
635                            sb.append(" ");
636                            sb.append(fromText);
637    
638                            System.out.println(sb.toString());
639    
640                            WebCacheItem wci = new TranslationWebCacheItem(
641                                    fromLanguageId, toLanguageId, fromText);
642    
643                            Translation translation = (Translation)wci.convert("");
644    
645                            toText = translation.getToText();
646                    }
647                    catch (Exception e) {
648                            Throwable cause = e.getCause();
649    
650                            if (cause instanceof MicrosoftTranslatorException) {
651                                    System.out.println(
652                                            cause.getClass().getName() + ": " + cause.getMessage());
653                            }
654                            else {
655                                    e.printStackTrace();
656                            }
657                    }
658    
659                    // Keep trying
660    
661                    if (toText == null) {
662                            return _translate(
663                                    fromLanguageId, toLanguageId, key, fromText, ++limit);
664                    }
665    
666                    return toText;
667            }
668    
669            private String _langDir;
670            private String _langFile;
671            private boolean _langTranslate;
672            private Properties _portalLanguageProperties;
673            private Properties _renameKeys;
674    
675    }