001    /**
002     * Copyright (c) 2000-2012 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portal.tools;
016    
017    import com.liferay.portal.kernel.io.OutputStreamWriter;
018    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedReader;
019    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedWriter;
020    import com.liferay.portal.kernel.io.unsync.UnsyncStringReader;
021    import com.liferay.portal.kernel.microsofttranslator.MicrosoftTranslatorException;
022    import com.liferay.portal.kernel.util.FileUtil;
023    import com.liferay.portal.kernel.util.GetterUtil;
024    import com.liferay.portal.kernel.util.NumericalStringComparator;
025    import com.liferay.portal.kernel.util.PropertiesUtil;
026    import com.liferay.portal.kernel.util.StringBundler;
027    import com.liferay.portal.kernel.util.StringPool;
028    import com.liferay.portal.kernel.util.StringUtil;
029    import com.liferay.portal.kernel.util.Validator;
030    import com.liferay.portal.kernel.webcache.WebCacheItem;
031    import com.liferay.portal.util.InitUtil;
032    import com.liferay.portlet.translator.model.Translation;
033    import com.liferay.portlet.translator.util.TranslationWebCacheItem;
034    
035    import java.io.File;
036    import java.io.FileInputStream;
037    import java.io.FileOutputStream;
038    import java.io.FileWriter;
039    import java.io.IOException;
040    import java.io.InputStream;
041    
042    import java.util.Map;
043    import java.util.Properties;
044    import java.util.Set;
045    import java.util.TreeSet;
046    
047    /**
048     * @author Brian Wing Shun Chan
049     */
050    public class LangBuilder {
051    
052            public static final String AUTOMATIC_COPY = " (Automatic Copy)";
053    
054            public static final String AUTOMATIC_TRANSLATION =
055                    " (Automatic Translation)";
056    
057            public static void main(String[] args) {
058                    Map<String, String> arguments = ArgumentsUtil.parseArguments(args);
059    
060                    System.setProperty("line.separator", StringPool.NEW_LINE);
061    
062                    InitUtil.initWithSpring();
063    
064                    String langDir = arguments.get("lang.dir");
065                    String langFile = arguments.get("lang.file");
066                    boolean langPlugin = GetterUtil.getBoolean(
067                            arguments.get("lang.plugin"));
068                    boolean langTranslate = GetterUtil.getBoolean(
069                            arguments.get("lang.translate"), true);
070    
071                    try {
072                            new LangBuilder(langDir, langFile, langPlugin, langTranslate);
073                    }
074                    catch (Exception e) {
075                            e.printStackTrace();
076                    }
077            }
078    
079            public LangBuilder(
080                            String langDir, String langFile, boolean langPlugin,
081                            boolean langTranslate)
082                    throws Exception {
083    
084                    _langDir = langDir;
085                    _langFile = langFile;
086                    _langTranslate = langTranslate;
087    
088                    if (langPlugin) {
089                            _portalLanguageProperties = new Properties();
090    
091                            Class<?> clazz = getClass();
092    
093                            ClassLoader classLoader = clazz.getClassLoader();
094    
095                            InputStream inputStream = classLoader.getResourceAsStream(
096                                    "content/Language.properties");
097    
098                            _portalLanguageProperties.load(inputStream);
099                    }
100    
101                    File renameKeysFile = new File(_langDir + "/rename.properties");
102    
103                    if (renameKeysFile.exists()) {
104                            _renameKeys = PropertiesUtil.load(FileUtil.read(renameKeysFile));
105                    }
106    
107                    String content = _orderProperties(
108                            new File(_langDir + "/" + _langFile + ".properties"));
109    
110                    // Locales that are not invoked by _createProperties should still be
111                    // rewritten to use the rignt line separator
112    
113                    _orderProperties(
114                            new File(_langDir + "/" + _langFile + "_en_AU.properties"));
115                    _orderProperties(
116                            new File(_langDir + "/" + _langFile + "_en_GB.properties"));
117                    _orderProperties(
118                            new File(_langDir + "/" + _langFile + "_fr_CA.properties"));
119    
120                    _createProperties(content, "ar"); // Arabic
121                    _createProperties(content, "eu"); // Basque
122                    _createProperties(content, "bg"); // Bulgarian
123                    _createProperties(content, "ca"); // Catalan
124                    _createProperties(content, "zh_CN"); // Chinese (China)
125                    _createProperties(content, "zh_TW"); // Chinese (Taiwan)
126                    _createProperties(content, "hr"); // Croatian
127                    _createProperties(content, "cs"); // Czech
128                    _createProperties(content, "da"); // Danish
129                    _createProperties(content, "nl"); // Dutch (Netherlands)
130                    _createProperties(content, "nl_BE", "nl"); // Dutch (Belgium)
131                    _createProperties(content, "et"); // Estonian
132                    _createProperties(content, "fi"); // Finnish
133                    _createProperties(content, "fr"); // French
134                    _createProperties(content, "gl"); // Galician
135                    _createProperties(content, "de"); // German
136                    _createProperties(content, "el"); // Greek
137                    _createProperties(content, "iw"); // Hebrew
138                    _createProperties(content, "hi_IN"); // Hindi (India)
139                    _createProperties(content, "hu"); // Hungarian
140                    _createProperties(content, "in"); // Indonesian
141                    _createProperties(content, "it"); // Italian
142                    _createProperties(content, "ja"); // Japanese
143                    _createProperties(content, "ko"); // Korean
144                    _createProperties(content, "lo"); // Lao
145                    _createProperties(content, "nb"); // Norwegian Bokmål
146                    _createProperties(content, "fa"); // Persian
147                    _createProperties(content, "pl"); // Polish
148                    _createProperties(content, "pt_BR"); // Portuguese (Brazil)
149                    _createProperties(content, "pt_PT", "pt_BR"); // Portuguese (Portugal)
150                    _createProperties(content, "ro"); // Romanian
151                    _createProperties(content, "ru"); // Russian
152                    _createProperties(content, "sr_RS"); // Serbian (Cyrillic)
153                    _createProperties(content, "sr_RS_latin"); // Serbian (Latin)
154                    _createProperties(content, "sk"); // Slovak
155                    _createProperties(content, "sl"); // Slovene
156                    _createProperties(content, "es"); // Spanish
157                    _createProperties(content, "sv"); // Swedish
158                    _createProperties(content, "tr"); // Turkish
159                    _createProperties(content, "uk"); // Ukrainian
160                    _createProperties(content, "vi"); // Vietnamese
161            }
162    
163            private void _createProperties(String content, String languageId)
164                    throws IOException {
165    
166                    _createProperties(content, languageId, null);
167            }
168    
169            private void _createProperties(
170                            String content, String languageId, String parentLanguageId)
171                    throws IOException {
172    
173                    File propertiesFile = new File(
174                            _langDir + "/" + _langFile + "_" + languageId + ".properties");
175    
176                    Properties properties = new Properties();
177    
178                    if (propertiesFile.exists()) {
179                            properties = PropertiesUtil.load(
180                                    new FileInputStream(propertiesFile), StringPool.UTF8);
181                    }
182    
183                    Properties parentProperties = null;
184    
185                    if (parentLanguageId != null) {
186                            File parentPropertiesFile = new File(
187                                    _langDir + "/" + _langFile + "_" + parentLanguageId +
188                                            ".properties");
189    
190                            if (parentPropertiesFile.exists()) {
191                                    parentProperties = new Properties();
192    
193                                    parentProperties = PropertiesUtil.load(
194                                            new FileInputStream(parentPropertiesFile), StringPool.UTF8);
195                            }
196                    }
197    
198                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
199                            new UnsyncStringReader(content));
200                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
201                            new OutputStreamWriter(
202                                    new FileOutputStream(propertiesFile), StringPool.UTF8));
203    
204                    boolean firstLine = true;
205                    int state = 0;
206    
207                    String line = null;
208    
209                    while ((line = unsyncBufferedReader.readLine()) != null) {
210                            line = line.trim();
211    
212                            int pos = line.indexOf("=");
213    
214                            if (pos != -1) {
215                                    String key = line.substring(0, pos);
216                                    String value = line.substring(pos + 1);
217    
218                                    if (((state == 1) && !key.startsWith("lang.")) ||
219                                            ((state == 2) && !key.startsWith("javax.portlet.")) ||
220                                            ((state == 3) && !key.startsWith("category.")) ||
221                                            ((state == 4) && !key.startsWith("model.resource.")) ||
222                                            ((state == 5) && !key.startsWith("action.")) ||
223                                            ((state == 7) && !key.startsWith("country.")) ||
224                                            ((state == 8) && !key.startsWith("currency.")) ||
225                                            ((state == 9) && !key.startsWith("language.")) ||
226                                            ((state != 9) && key.startsWith("language."))) {
227    
228                                            throw new RuntimeException(
229                                                    "File " + languageId + " with state " + state +
230                                                            " has key " + key);
231                                    }
232    
233                                    String translatedText = properties.getProperty(key);
234    
235                                    if ((translatedText == null) && (parentProperties != null)) {
236                                            translatedText = parentProperties.getProperty(key);
237                                    }
238    
239                                    if ((translatedText == null) && (_renameKeys != null)) {
240                                            String renameKey = _renameKeys.getProperty(key);
241    
242                                            if (renameKey != null) {
243                                                    translatedText = properties.getProperty(key);
244    
245                                                    if ((translatedText == null) &&
246                                                            (parentProperties != null)) {
247    
248                                                            translatedText = parentProperties.getProperty(key);
249                                                    }
250                                            }
251                                    }
252    
253                                    if (translatedText != null) {
254                                            if (translatedText.contains("Babel Fish") ||
255                                                    translatedText.contains("Yahoo! - 999")) {
256    
257                                                    translatedText = "";
258                                            }
259                                            else if (translatedText.endsWith(AUTOMATIC_COPY)) {
260                                                    translatedText = value + AUTOMATIC_COPY;
261                                            }
262                                    }
263    
264                                    if ((translatedText == null) || translatedText.equals("")) {
265                                            if (line.contains("{") || line.contains("<")) {
266                                                    translatedText = value + AUTOMATIC_COPY;
267                                            }
268                                            else if (line.contains("[")) {
269                                                    pos = line.indexOf("[");
270    
271                                                    String baseKey = line.substring(0, pos);
272    
273                                                    String translatedBaseKey = properties.getProperty(
274                                                            baseKey);
275    
276                                                    if (Validator.isNotNull(translatedBaseKey)) {
277                                                            translatedText = translatedBaseKey + AUTOMATIC_COPY;
278                                                    }
279                                                    else {
280                                                            translatedText = value + AUTOMATIC_COPY;
281                                                    }
282                                            }
283                                            else if (key.equals("lang.dir")) {
284                                                    translatedText = "ltr";
285                                            }
286                                            else if (key.equals("lang.line.begin")) {
287                                                    translatedText = "left";
288                                            }
289                                            else if (key.equals("lang.line.end")) {
290                                                    translatedText = "right";
291                                            }
292                                            else if (languageId.equals("el") &&
293                                                             (key.equals("enabled") || key.equals("on") ||
294                                                              key.equals("on-date"))) {
295    
296                                                    translatedText = "";
297                                            }
298                                            else if (languageId.equals("es") && key.equals("am")) {
299                                                    translatedText = "";
300                                            }
301                                            else if (languageId.equals("it") && key.equals("am")) {
302                                                    translatedText = "";
303                                            }
304                                            else if (languageId.equals("ja") &&
305                                                             (key.equals("any") || key.equals("anytime") ||
306                                                              key.equals("down") || key.equals("on") ||
307                                                              key.equals("on-date") || key.equals("the"))) {
308    
309                                                    translatedText = "";
310                                            }
311                                            else if (languageId.equals("ko") && key.equals("the")) {
312                                                    translatedText = "";
313                                            }
314                                            else {
315                                                    translatedText = _translate(
316                                                            "en", languageId, key, value, 0);
317    
318                                                    if (Validator.isNull(translatedText)) {
319                                                            translatedText = value + AUTOMATIC_COPY;
320                                                    }
321                                                    else if (!key.startsWith("country.") &&
322                                                                     !key.startsWith("language.")) {
323    
324                                                            translatedText =
325                                                                    translatedText + AUTOMATIC_TRANSLATION;
326                                                    }
327                                            }
328                                    }
329    
330                                    if (Validator.isNotNull(translatedText)) {
331                                            if (translatedText.contains("Babel Fish") ||
332                                                    translatedText.contains("Yahoo! - 999")) {
333    
334                                                    throw new IOException(
335                                                            "IP was blocked because of over usage. Please " +
336                                                                    "use another IP.");
337                                            }
338    
339                                            translatedText = _fixTranslation(translatedText);
340    
341                                            if (firstLine) {
342                                                    firstLine = false;
343                                            }
344                                            else {
345                                                    unsyncBufferedWriter.newLine();
346                                            }
347    
348                                            unsyncBufferedWriter.write(key + "=" + translatedText);
349    
350                                            unsyncBufferedWriter.flush();
351                                    }
352                            }
353                            else {
354                                    if (line.startsWith("## Language settings")) {
355                                            if (state == 1) {
356                                                    throw new RuntimeException(languageId);
357                                            }
358    
359                                            state = 1;
360                                    }
361                                    else if (line.startsWith(
362                                                            "## Portlet descriptions and titles")) {
363    
364                                            if (state == 2) {
365                                                    throw new RuntimeException(languageId);
366                                            }
367    
368                                            state = 2;
369                                    }
370                                    else if (line.startsWith("## Category titles")) {
371                                            if (state == 3) {
372                                                    throw new RuntimeException(languageId);
373                                            }
374    
375                                            state = 3;
376                                    }
377                                    else if (line.startsWith("## Model resources")) {
378                                            if (state == 4) {
379                                                    throw new RuntimeException(languageId);
380                                            }
381    
382                                            state = 4;
383                                    }
384                                    else if (line.startsWith("## Action names")) {
385                                            if (state == 5) {
386                                                    throw new RuntimeException(languageId);
387                                            }
388    
389                                            state = 5;
390                                    }
391                                    else if (line.startsWith("## Messages")) {
392                                            if (state == 6) {
393                                                    throw new RuntimeException(languageId);
394                                            }
395    
396                                            state = 6;
397                                    }
398                                    else if (line.startsWith("## Country")) {
399                                            if (state == 7) {
400                                                    throw new RuntimeException(languageId);
401                                            }
402    
403                                            state = 7;
404                                    }
405                                    else if (line.startsWith("## Currency")) {
406                                            if (state == 8) {
407                                                    throw new RuntimeException(languageId);
408                                            }
409    
410                                            state = 8;
411                                    }
412                                    else if (line.startsWith("## Language")) {
413                                            if (state == 9) {
414                                                    throw new RuntimeException(languageId);
415                                            }
416    
417                                            state = 9;
418                                    }
419    
420                                    if (firstLine) {
421                                            firstLine = false;
422                                    }
423                                    else {
424                                            unsyncBufferedWriter.newLine();
425                                    }
426    
427                                    unsyncBufferedWriter.write(line);
428    
429                                    unsyncBufferedWriter.flush();
430                            }
431                    }
432    
433                    unsyncBufferedReader.close();
434                    unsyncBufferedWriter.close();
435            }
436    
437            private String _fixEnglishTranslation(String key, String value) {
438                    if (value.contains(" this ")) {
439                            if (value.contains(".") || value.contains("?") ||
440                                    value.contains(":") ||
441                                    key.equals("the-url-of-the-page-comparing-this-page-content-with-the-previous-version")) {
442                            }
443                            else {
444                                    value = StringUtil.replace(value, " this ", " This ");
445                            }
446                    }
447                    else {
448                            value = StringUtil.replace(value, " From ", " from ");
449                    }
450    
451                    return value;
452            }
453    
454            private String _fixTranslation(String value) {
455                    value = StringUtil.replace(
456                            value.trim(),
457                            new String[] {
458                                    "  ", "<b>", "</b>", "<i>", "</i>", " url ", "&#39;", "&#39 ;",
459                                    "&quot;", "&quot ;", "ReCaptcha", "Captcha"
460                            },
461                            new String[] {
462                                    " ", "<strong>", "</strong>", "<em>", "</em>", " URL ", "\'",
463                                    "\'", "\"", "\"", "reCAPTCHA", "CAPTCHA"
464                            });
465    
466                    return value;
467            }
468    
469            private String _orderProperties(File propertiesFile) throws IOException {
470                    if (!propertiesFile.exists()) {
471                            return null;
472                    }
473    
474                    String content = FileUtil.read(propertiesFile);
475    
476                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
477                            new UnsyncStringReader(content));
478                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
479                            new FileWriter(propertiesFile));
480    
481                    Set<String> messages = new TreeSet<String>(
482                            new NumericalStringComparator(true, true));
483    
484                    boolean begin = false;
485                    boolean firstLine = true;
486    
487                    String line = null;
488    
489                    while ((line = unsyncBufferedReader.readLine()) != null) {
490                            int pos = line.indexOf("=");
491    
492                            if (pos != -1) {
493                                    String key = line.substring(0, pos);
494    
495                                    String value = _fixTranslation(line.substring(pos + 1));
496    
497                                    value = _fixEnglishTranslation(key, value);
498    
499                                    if (_portalLanguageProperties != null) {
500                                            String portalValue = String.valueOf(
501                                                    _portalLanguageProperties.get(key));
502    
503                                            if (value.equals(portalValue)) {
504                                                    System.out.println("Duplicate key " + key);
505                                            }
506                                    }
507    
508                                    messages.add(key + "=" + value);
509                            }
510                            else {
511                                    if (begin && line.equals("")) {
512                                            _sortAndWrite(unsyncBufferedWriter, messages, firstLine);
513                                    }
514    
515                                    if (line.equals("")) {
516                                            begin = !begin;
517                                    }
518    
519                                    if (firstLine) {
520                                            firstLine = false;
521                                    }
522                                    else {
523                                            unsyncBufferedWriter.newLine();
524                                    }
525    
526                                    unsyncBufferedWriter.write(line);
527                            }
528    
529                            unsyncBufferedWriter.flush();
530                    }
531    
532                    if (!messages.isEmpty()) {
533                            _sortAndWrite(unsyncBufferedWriter, messages, firstLine);
534                    }
535    
536                    unsyncBufferedReader.close();
537                    unsyncBufferedWriter.close();
538    
539                    return FileUtil.read(propertiesFile);
540            }
541    
542            private void _sortAndWrite(
543                            UnsyncBufferedWriter unsyncBufferedWriter, Set<String> messages,
544                            boolean firstLine)
545                    throws IOException {
546    
547                    String[] messagesArray = messages.toArray(new String[messages.size()]);
548    
549                    for (int i = 0; i < messagesArray.length; i++) {
550                            if (!firstLine || (i != 0)) {
551                                    unsyncBufferedWriter.newLine();
552                            }
553    
554                            unsyncBufferedWriter.write(messagesArray[i]);
555                    }
556    
557                    messages.clear();
558            }
559    
560            private String _translate(
561                    String fromLanguageId, String toLanguageId, String key, String fromText,
562                    int limit) {
563    
564                    if (toLanguageId.equals("ar") ||
565                            toLanguageId.equals("eu") ||
566                            toLanguageId.equals("bg") ||
567                            toLanguageId.equals("ca") ||
568                            toLanguageId.equals("hr") ||
569                            toLanguageId.equals("cs") ||
570                            toLanguageId.equals("da") ||
571                            toLanguageId.equals("et") ||
572                            toLanguageId.equals("fi") ||
573                            toLanguageId.equals("gl") ||
574    
575                            // LPS-26741
576    
577                            toLanguageId.equals("de") ||
578    
579                            toLanguageId.equals("iw") ||
580                            toLanguageId.equals("hi") ||
581                            toLanguageId.equals("hu") ||
582                            toLanguageId.equals("in") ||
583                            toLanguageId.equals("lo") ||
584                            toLanguageId.equals("nb") ||
585                            toLanguageId.equals("fa") ||
586                            toLanguageId.equals("pl") ||
587                            toLanguageId.equals("ro") ||
588                            toLanguageId.equals("ru") ||
589                            toLanguageId.equals("sr_RS") ||
590                            toLanguageId.equals("sr_RS_latin") ||
591                            toLanguageId.equals("sk") ||
592                            toLanguageId.equals("sl") ||
593                            toLanguageId.equals("sv") ||
594                            toLanguageId.equals("tr") ||
595                            toLanguageId.equals("uk") ||
596                            toLanguageId.equals("vi")) {
597    
598                            // Automatic translator does not support Arabic, Basque, Bulgarian,
599                            // Catalan, Croatian, Czech, Danish, Estonian, Finnish, Galician,
600                            // German, Hebrew, Hindi, Hungarian, Indonesian, Lao,
601                            // Norwegian Bokmål, Persian, Polish, Romanian, Russian, Serbian,
602                            // Slovak, Slovene, Swedish, Turkish, Ukrainian, or Vietnamese
603    
604                            return null;
605                    }
606    
607                    if (!_langTranslate) {
608                            return null;
609                    }
610    
611                    // Limit the number of retries to 3
612    
613                    if (limit == 3) {
614                            return null;
615                    }
616    
617                    String toText = null;
618    
619                    try {
620                            StringBundler sb = new StringBundler(8);
621    
622                            sb.append("Translating ");
623                            sb.append(fromLanguageId);
624                            sb.append("_");
625                            sb.append(toLanguageId);
626                            sb.append(" ");
627                            sb.append(key);
628                            sb.append(" ");
629                            sb.append(fromText);
630    
631                            System.out.println(sb.toString());
632    
633                            WebCacheItem wci = new TranslationWebCacheItem(
634                                    fromLanguageId, toLanguageId, fromText);
635    
636                            Translation translation = (Translation)wci.convert("");
637    
638                            toText = translation.getToText();
639                    }
640                    catch (Exception e) {
641                            Throwable cause = e.getCause();
642    
643                            if (cause instanceof MicrosoftTranslatorException) {
644                                    System.out.println(
645                                            cause.getClass().getName() + ": " + cause.getMessage());
646                            }
647                            else {
648                                    e.printStackTrace();
649                            }
650                    }
651    
652                    // Keep trying
653    
654                    if (toText == null) {
655                            return _translate(
656                                    fromLanguageId, toLanguageId, key, fromText, ++limit);
657                    }
658    
659                    return toText;
660            }
661    
662            private String _langDir;
663            private String _langFile;
664            private boolean _langTranslate;
665            private Properties _portalLanguageProperties;
666            private Properties _renameKeys;
667    
668    }