001    /**
002     * Copyright (c) 2000-2013 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portal.tools;
016    
017    import com.liferay.portal.kernel.io.OutputStreamWriter;
018    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedReader;
019    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedWriter;
020    import com.liferay.portal.kernel.io.unsync.UnsyncStringReader;
021    import com.liferay.portal.kernel.microsofttranslator.MicrosoftTranslatorException;
022    import com.liferay.portal.kernel.util.FileUtil;
023    import com.liferay.portal.kernel.util.GetterUtil;
024    import com.liferay.portal.kernel.util.NumericalStringComparator;
025    import com.liferay.portal.kernel.util.PropertiesUtil;
026    import com.liferay.portal.kernel.util.StringBundler;
027    import com.liferay.portal.kernel.util.StringPool;
028    import com.liferay.portal.kernel.util.StringUtil;
029    import com.liferay.portal.kernel.util.Validator;
030    import com.liferay.portal.kernel.webcache.WebCacheItem;
031    import com.liferay.portal.util.InitUtil;
032    import com.liferay.portlet.translator.model.Translation;
033    import com.liferay.portlet.translator.util.TranslationWebCacheItem;
034    
035    import java.io.File;
036    import java.io.FileInputStream;
037    import java.io.FileOutputStream;
038    import java.io.FileWriter;
039    import java.io.IOException;
040    import java.io.InputStream;
041    
042    import java.util.Map;
043    import java.util.Properties;
044    import java.util.TreeMap;
045    
046    /**
047     * @author Brian Wing Shun Chan
048     */
049    public class LangBuilder {
050    
051            public static final String AUTOMATIC_COPY = " (Automatic Copy)";
052    
053            public static final String AUTOMATIC_TRANSLATION =
054                    " (Automatic Translation)";
055    
056            public static void main(String[] args) {
057                    Map<String, String> arguments = ArgumentsUtil.parseArguments(args);
058    
059                    System.setProperty("line.separator", StringPool.NEW_LINE);
060    
061                    InitUtil.initWithSpring();
062    
063                    String langDir = arguments.get("lang.dir");
064                    String langFile = arguments.get("lang.file");
065                    boolean langPlugin = GetterUtil.getBoolean(
066                            arguments.get("lang.plugin"));
067                    boolean langTranslate = GetterUtil.getBoolean(
068                            arguments.get("lang.translate"), true);
069    
070                    try {
071                            new LangBuilder(langDir, langFile, langPlugin, langTranslate);
072                    }
073                    catch (Exception e) {
074                            e.printStackTrace();
075                    }
076            }
077    
078            public LangBuilder(
079                            String langDir, String langFile, boolean langPlugin,
080                            boolean langTranslate)
081                    throws Exception {
082    
083                    _langDir = langDir;
084                    _langFile = langFile;
085                    _langTranslate = langTranslate;
086    
087                    if (langPlugin) {
088                            _portalLanguageProperties = new Properties();
089    
090                            Class<?> clazz = getClass();
091    
092                            ClassLoader classLoader = clazz.getClassLoader();
093    
094                            InputStream inputStream = classLoader.getResourceAsStream(
095                                    "content/Language.properties");
096    
097                            _portalLanguageProperties.load(inputStream);
098                    }
099    
100                    File renameKeysFile = new File(_langDir + "/rename.properties");
101    
102                    if (renameKeysFile.exists()) {
103                            _renameKeys = PropertiesUtil.load(FileUtil.read(renameKeysFile));
104                    }
105    
106                    String content = _orderProperties(
107                            new File(_langDir + "/" + _langFile + ".properties"));
108    
109                    // Locales that are not invoked by _createProperties should still be
110                    // rewritten to use the right line separator
111    
112                    _orderProperties(
113                            new File(_langDir + "/" + _langFile + "_en_AU.properties"));
114                    _orderProperties(
115                            new File(_langDir + "/" + _langFile + "_en_GB.properties"));
116                    _orderProperties(
117                            new File(_langDir + "/" + _langFile + "_fr_CA.properties"));
118    
119                    _createProperties(content, "ar"); // Arabic
120                    _createProperties(content, "eu"); // Basque
121                    _createProperties(content, "bg"); // Bulgarian
122                    _createProperties(content, "ca"); // Catalan
123                    _createProperties(content, "zh_CN"); // Chinese (China)
124                    _createProperties(content, "zh_TW"); // Chinese (Taiwan)
125                    _createProperties(content, "hr"); // Croatian
126                    _createProperties(content, "cs"); // Czech
127                    _createProperties(content, "da"); // Danish
128                    _createProperties(content, "nl"); // Dutch (Netherlands)
129                    _createProperties(content, "nl_BE", "nl"); // Dutch (Belgium)
130                    _createProperties(content, "et"); // Estonian
131                    _createProperties(content, "fi"); // Finnish
132                    _createProperties(content, "fr"); // French
133                    _createProperties(content, "gl"); // Galician
134                    _createProperties(content, "de"); // German
135                    _createProperties(content, "el"); // Greek
136                    _createProperties(content, "iw"); // Hebrew
137                    _createProperties(content, "hi_IN"); // Hindi (India)
138                    _createProperties(content, "hu"); // Hungarian
139                    _createProperties(content, "in"); // Indonesian
140                    _createProperties(content, "it"); // Italian
141                    _createProperties(content, "ja"); // Japanese
142                    _createProperties(content, "ko"); // Korean
143                    _createProperties(content, "lo"); // Lao
144                    _createProperties(content, "lt"); // Lithuanian
145                    _createProperties(content, "nb"); // Norwegian Bokm??l
146                    _createProperties(content, "fa"); // Persian
147                    _createProperties(content, "pl"); // Polish
148                    _createProperties(content, "pt_BR"); // Portuguese (Brazil)
149                    _createProperties(content, "pt_PT", "pt_BR"); // Portuguese (Portugal)
150                    _createProperties(content, "ro"); // Romanian
151                    _createProperties(content, "ru"); // Russian
152                    _createProperties(content, "sr_RS"); // Serbian (Cyrillic)
153                    _createProperties(content, "sr_RS_latin"); // Serbian (Latin)
154                    _createProperties(content, "sk"); // Slovak
155                    _createProperties(content, "sl"); // Slovene
156                    _createProperties(content, "es"); // Spanish
157                    _createProperties(content, "sv"); // Swedish
158                    _createProperties(content, "tr"); // Turkish
159                    _createProperties(content, "uk"); // Ukrainian
160                    _createProperties(content, "vi"); // Vietnamese
161            }
162    
163            private void _createProperties(String content, String languageId)
164                    throws IOException {
165    
166                    _createProperties(content, languageId, null);
167            }
168    
169            private void _createProperties(
170                            String content, String languageId, String parentLanguageId)
171                    throws IOException {
172    
173                    File propertiesFile = new File(
174                            _langDir + "/" + _langFile + "_" + languageId + ".properties");
175    
176                    Properties properties = new Properties();
177    
178                    if (propertiesFile.exists()) {
179                            properties = PropertiesUtil.load(
180                                    new FileInputStream(propertiesFile), StringPool.UTF8);
181                    }
182    
183                    Properties parentProperties = null;
184    
185                    if (parentLanguageId != null) {
186                            File parentPropertiesFile = new File(
187                                    _langDir + "/" + _langFile + "_" + parentLanguageId +
188                                            ".properties");
189    
190                            if (parentPropertiesFile.exists()) {
191                                    parentProperties = new Properties();
192    
193                                    parentProperties = PropertiesUtil.load(
194                                            new FileInputStream(parentPropertiesFile), StringPool.UTF8);
195                            }
196                    }
197    
198                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
199                            new UnsyncStringReader(content));
200                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
201                            new OutputStreamWriter(
202                                    new FileOutputStream(propertiesFile), StringPool.UTF8));
203    
204                    boolean firstLine = true;
205                    int state = 0;
206    
207                    String line = null;
208    
209                    while ((line = unsyncBufferedReader.readLine()) != null) {
210                            line = line.trim();
211    
212                            int pos = line.indexOf("=");
213    
214                            if (pos != -1) {
215                                    String key = line.substring(0, pos);
216                                    String value = line.substring(pos + 1);
217    
218                                    if (((state == 1) && !key.startsWith("lang.")) ||
219                                            ((state == 2) && !key.startsWith("javax.portlet.")) ||
220                                            ((state == 3) && !key.startsWith("category.")) ||
221                                            ((state == 4) && !key.startsWith("model.resource.")) ||
222                                            ((state == 5) && !key.startsWith("action.")) ||
223                                            ((state == 7) && !key.startsWith("country.")) ||
224                                            ((state == 8) && !key.startsWith("currency.")) ||
225                                            ((state == 9) && !key.startsWith("language.")) ||
226                                            ((state != 9) && key.startsWith("language."))) {
227    
228                                            throw new RuntimeException(
229                                                    "File " + languageId + " with state " + state +
230                                                            " has key " + key);
231                                    }
232    
233                                    String translatedText = properties.getProperty(key);
234    
235                                    if ((translatedText == null) && (parentProperties != null)) {
236                                            translatedText = parentProperties.getProperty(key);
237                                    }
238    
239                                    if ((translatedText == null) && (_renameKeys != null)) {
240                                            String renameKey = _renameKeys.getProperty(key);
241    
242                                            if (renameKey != null) {
243                                                    translatedText = properties.getProperty(key);
244    
245                                                    if ((translatedText == null) &&
246                                                            (parentProperties != null)) {
247    
248                                                            translatedText = parentProperties.getProperty(key);
249                                                    }
250                                            }
251                                    }
252    
253                                    if (translatedText != null) {
254                                            if (translatedText.contains("Babel Fish") ||
255                                                    translatedText.contains("Yahoo! - 999")) {
256    
257                                                    translatedText = "";
258                                            }
259                                            else if (translatedText.endsWith(AUTOMATIC_COPY)) {
260                                                    translatedText = value + AUTOMATIC_COPY;
261                                            }
262                                    }
263    
264                                    if ((translatedText == null) || translatedText.equals("")) {
265                                            if (line.contains("{") || line.contains("<")) {
266                                                    translatedText = value + AUTOMATIC_COPY;
267                                            }
268                                            else if (line.contains("[")) {
269                                                    pos = line.indexOf("[");
270    
271                                                    String baseKey = line.substring(0, pos);
272    
273                                                    String translatedBaseKey = properties.getProperty(
274                                                            baseKey);
275    
276                                                    if (Validator.isNotNull(translatedBaseKey)) {
277                                                            translatedText = translatedBaseKey + AUTOMATIC_COPY;
278                                                    }
279                                                    else {
280                                                            translatedText = value + AUTOMATIC_COPY;
281                                                    }
282                                            }
283                                            else if (key.equals("lang.dir")) {
284                                                    translatedText = "ltr";
285                                            }
286                                            else if (key.equals("lang.line.begin")) {
287                                                    translatedText = "left";
288                                            }
289                                            else if (key.equals("lang.line.end")) {
290                                                    translatedText = "right";
291                                            }
292                                            else if (languageId.equals("el") &&
293                                                             (key.equals("enabled") || key.equals("on") ||
294                                                              key.equals("on-date"))) {
295    
296                                                    translatedText = "";
297                                            }
298                                            else if (languageId.equals("es") && key.equals("am")) {
299                                                    translatedText = "";
300                                            }
301                                            else if (languageId.equals("fi") &&
302                                                             (key.equals("on") || key.equals("the"))) {
303    
304                                                    translatedText = "";
305                                            }
306                                            else if (languageId.equals("it") && key.equals("am")) {
307                                                    translatedText = "";
308                                            }
309                                            else if (languageId.equals("ja") &&
310                                                             (key.equals("any") || key.equals("anytime") ||
311                                                              key.equals("down") || key.equals("on") ||
312                                                              key.equals("on-date") || key.equals("the"))) {
313    
314                                                    translatedText = "";
315                                            }
316                                            else if (languageId.equals("ko") && key.equals("the")) {
317                                                    translatedText = "";
318                                            }
319                                            else {
320                                                    translatedText = _translate(
321                                                            "en", languageId, key, value, 0);
322    
323                                                    if (Validator.isNull(translatedText)) {
324                                                            translatedText = value + AUTOMATIC_COPY;
325                                                    }
326                                                    else if (!key.startsWith("country.") &&
327                                                                     !key.startsWith("language.")) {
328    
329                                                            translatedText =
330                                                                    translatedText + AUTOMATIC_TRANSLATION;
331                                                    }
332                                            }
333                                    }
334    
335                                    if (Validator.isNotNull(translatedText)) {
336                                            if (translatedText.contains("Babel Fish") ||
337                                                    translatedText.contains("Yahoo! - 999")) {
338    
339                                                    throw new IOException(
340                                                            "IP was blocked because of over usage. Please " +
341                                                                    "use another IP.");
342                                            }
343    
344                                            translatedText = _fixTranslation(translatedText);
345    
346                                            if (firstLine) {
347                                                    firstLine = false;
348                                            }
349                                            else {
350                                                    unsyncBufferedWriter.newLine();
351                                            }
352    
353                                            unsyncBufferedWriter.write(key + "=" + translatedText);
354    
355                                            unsyncBufferedWriter.flush();
356                                    }
357                            }
358                            else {
359                                    if (line.startsWith("## Language settings")) {
360                                            if (state == 1) {
361                                                    throw new RuntimeException(languageId);
362                                            }
363    
364                                            state = 1;
365                                    }
366                                    else if (line.startsWith(
367                                                            "## Portlet descriptions and titles")) {
368    
369                                            if (state == 2) {
370                                                    throw new RuntimeException(languageId);
371                                            }
372    
373                                            state = 2;
374                                    }
375                                    else if (line.startsWith("## Category titles")) {
376                                            if (state == 3) {
377                                                    throw new RuntimeException(languageId);
378                                            }
379    
380                                            state = 3;
381                                    }
382                                    else if (line.startsWith("## Model resources")) {
383                                            if (state == 4) {
384                                                    throw new RuntimeException(languageId);
385                                            }
386    
387                                            state = 4;
388                                    }
389                                    else if (line.startsWith("## Action names")) {
390                                            if (state == 5) {
391                                                    throw new RuntimeException(languageId);
392                                            }
393    
394                                            state = 5;
395                                    }
396                                    else if (line.startsWith("## Messages")) {
397                                            if (state == 6) {
398                                                    throw new RuntimeException(languageId);
399                                            }
400    
401                                            state = 6;
402                                    }
403                                    else if (line.startsWith("## Country")) {
404                                            if (state == 7) {
405                                                    throw new RuntimeException(languageId);
406                                            }
407    
408                                            state = 7;
409                                    }
410                                    else if (line.startsWith("## Currency")) {
411                                            if (state == 8) {
412                                                    throw new RuntimeException(languageId);
413                                            }
414    
415                                            state = 8;
416                                    }
417                                    else if (line.startsWith("## Language")) {
418                                            if (state == 9) {
419                                                    throw new RuntimeException(languageId);
420                                            }
421    
422                                            state = 9;
423                                    }
424    
425                                    if (firstLine) {
426                                            firstLine = false;
427                                    }
428                                    else {
429                                            unsyncBufferedWriter.newLine();
430                                    }
431    
432                                    unsyncBufferedWriter.write(line);
433    
434                                    unsyncBufferedWriter.flush();
435                            }
436                    }
437    
438                    unsyncBufferedReader.close();
439                    unsyncBufferedWriter.close();
440            }
441    
442            private String _fixEnglishTranslation(String key, String value) {
443    
444                    // http://en.wikibooks.org/wiki/Basic_Book_Design/Capitalizing_Words_in_Titles
445                    // http://www.imdb.com
446    
447                    if (value.contains(" this ")) {
448                            if (value.contains(".") || value.contains("?") ||
449                                    value.contains(":") ||
450                                    key.equals("the-url-of-the-page-comparing-this-page-content-with-the-previous-version")) {
451                            }
452                            else {
453                                    value = StringUtil.replace(value, " this ", " This ");
454                            }
455                    }
456                    else {
457                            value = StringUtil.replace(value, " From ", " from ");
458                    }
459    
460                    return value;
461            }
462    
463            private String _fixTranslation(String value) {
464                    value = StringUtil.replace(
465                            value.trim(),
466                            new String[] {
467                                    "  ", "<b>", "</b>", "<i>", "</i>", " url ", "&#39;", "&#39 ;",
468                                    "&quot;", "&quot ;", "ReCaptcha", "Captcha"
469                            },
470                            new String[] {
471                                    " ", "<strong>", "</strong>", "<em>", "</em>", " URL ", "\'",
472                                    "\'", "\"", "\"", "reCAPTCHA", "CAPTCHA"
473                            });
474    
475                    return value;
476            }
477    
478            private String _orderProperties(File propertiesFile) throws IOException {
479                    if (!propertiesFile.exists()) {
480                            return null;
481                    }
482    
483                    String content = FileUtil.read(propertiesFile);
484    
485                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
486                            new UnsyncStringReader(content));
487                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
488                            new FileWriter(propertiesFile));
489    
490                    Map<String, String> messages = new TreeMap<String, String>(
491                            new NumericalStringComparator(true, true));
492    
493                    boolean begin = false;
494                    boolean firstLine = true;
495    
496                    String line = null;
497    
498                    while ((line = unsyncBufferedReader.readLine()) != null) {
499                            int pos = line.indexOf("=");
500    
501                            if (pos != -1) {
502                                    String key = line.substring(0, pos);
503    
504                                    String value = _fixTranslation(line.substring(pos + 1));
505    
506                                    value = _fixEnglishTranslation(key, value);
507    
508                                    if (_portalLanguageProperties != null) {
509                                            String portalValue = String.valueOf(
510                                                    _portalLanguageProperties.get(key));
511    
512                                            if (value.equals(portalValue)) {
513                                                    System.out.println("Duplicate key " + key);
514                                            }
515                                    }
516    
517                                    messages.put(key, value);
518                            }
519                            else {
520                                    if (begin && line.equals(StringPool.BLANK)) {
521                                            _sortAndWrite(unsyncBufferedWriter, messages, firstLine);
522                                    }
523    
524                                    if (line.equals(StringPool.BLANK)) {
525                                            begin = !begin;
526                                    }
527    
528                                    if (firstLine) {
529                                            firstLine = false;
530                                    }
531                                    else {
532                                            unsyncBufferedWriter.newLine();
533                                    }
534    
535                                    unsyncBufferedWriter.write(line);
536                            }
537    
538                            unsyncBufferedWriter.flush();
539                    }
540    
541                    if (!messages.isEmpty()) {
542                            _sortAndWrite(unsyncBufferedWriter, messages, firstLine);
543                    }
544    
545                    unsyncBufferedReader.close();
546                    unsyncBufferedWriter.close();
547    
548                    return FileUtil.read(propertiesFile);
549            }
550    
551            private void _sortAndWrite(
552                            UnsyncBufferedWriter unsyncBufferedWriter,
553                            Map<String, String> messages, boolean firstLine)
554                    throws IOException {
555    
556                    boolean firstEntry = true;
557    
558                    for (Map.Entry<String, String> entry : messages.entrySet()) {
559                            if (!firstLine || !firstEntry) {
560                                    unsyncBufferedWriter.newLine();
561                            }
562    
563                            firstEntry = false;
564    
565                            unsyncBufferedWriter.write(entry.getKey() + "=" + entry.getValue());
566                    }
567    
568                    messages.clear();
569            }
570    
571            private String _translate(
572                    String fromLanguageId, String toLanguageId, String key, String fromText,
573                    int limit) {
574    
575                    if (toLanguageId.equals("ar") ||
576                            toLanguageId.equals("eu") ||
577                            toLanguageId.equals("bg") ||
578                            toLanguageId.equals("ca") ||
579                            toLanguageId.equals("hr") ||
580                            toLanguageId.equals("cs") ||
581                            toLanguageId.equals("da") ||
582                            toLanguageId.equals("et") ||
583                            toLanguageId.equals("fi") ||
584                            toLanguageId.equals("gl") ||
585    
586                            // LPS-26741
587    
588                            toLanguageId.equals("de") ||
589    
590                            toLanguageId.equals("iw") ||
591                            toLanguageId.equals("hi") ||
592                            toLanguageId.equals("hu") ||
593                            toLanguageId.equals("in") ||
594                            toLanguageId.equals("lo") ||
595                            toLanguageId.equals("lt") ||
596                            toLanguageId.equals("nb") ||
597                            toLanguageId.equals("fa") ||
598                            toLanguageId.equals("pl") ||
599                            toLanguageId.equals("ro") ||
600                            toLanguageId.equals("ru") ||
601                            toLanguageId.equals("sr_RS") ||
602                            toLanguageId.equals("sr_RS_latin") ||
603                            toLanguageId.equals("sk") ||
604                            toLanguageId.equals("sl") ||
605                            toLanguageId.equals("sv") ||
606                            toLanguageId.equals("tr") ||
607                            toLanguageId.equals("uk") ||
608                            toLanguageId.equals("vi")) {
609    
610                            // Automatic translator does not support Arabic, Basque, Bulgarian,
611                            // Catalan, Croatian, Czech, Danish, Estonian, Finnish, Galician,
612                            // German, Hebrew, Hindi, Hungarian, Indonesian, Lao, Norwegian
613                            // Bokm??l, Persian, Polish, Romanian, Russian, Serbian, Slovak,
614                            // Slovene, Swedish, Turkish, Ukrainian, or Vietnamese
615    
616                            return null;
617                    }
618    
619                    if (!_langTranslate) {
620                            return null;
621                    }
622    
623                    // Limit the number of retries to 3
624    
625                    if (limit == 3) {
626                            return null;
627                    }
628    
629                    String toText = null;
630    
631                    try {
632                            StringBundler sb = new StringBundler(8);
633    
634                            sb.append("Translating ");
635                            sb.append(fromLanguageId);
636                            sb.append("_");
637                            sb.append(toLanguageId);
638                            sb.append(" ");
639                            sb.append(key);
640                            sb.append(" ");
641                            sb.append(fromText);
642    
643                            System.out.println(sb.toString());
644    
645                            WebCacheItem wci = new TranslationWebCacheItem(
646                                    fromLanguageId, toLanguageId, fromText);
647    
648                            Translation translation = (Translation)wci.convert("");
649    
650                            toText = translation.getToText();
651                    }
652                    catch (Exception e) {
653                            Throwable cause = e.getCause();
654    
655                            if (cause instanceof MicrosoftTranslatorException) {
656                                    System.out.println(
657                                            cause.getClass().getName() + ": " + cause.getMessage());
658                            }
659                            else {
660                                    e.printStackTrace();
661                            }
662                    }
663    
664                    // Keep trying
665    
666                    if (toText == null) {
667                            return _translate(
668                                    fromLanguageId, toLanguageId, key, fromText, ++limit);
669                    }
670    
671                    return toText;
672            }
673    
674            private String _langDir;
675            private String _langFile;
676            private boolean _langTranslate;
677            private Properties _portalLanguageProperties;
678            private Properties _renameKeys;
679    
680    }