001    /**
002     * Copyright (c) 2000-2013 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portal.tools;
016    
017    import com.liferay.portal.kernel.io.OutputStreamWriter;
018    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedReader;
019    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedWriter;
020    import com.liferay.portal.kernel.io.unsync.UnsyncStringReader;
021    import com.liferay.portal.kernel.microsofttranslator.MicrosoftTranslatorException;
022    import com.liferay.portal.kernel.util.FileUtil;
023    import com.liferay.portal.kernel.util.GetterUtil;
024    import com.liferay.portal.kernel.util.NumericalStringComparator;
025    import com.liferay.portal.kernel.util.PropertiesUtil;
026    import com.liferay.portal.kernel.util.StringPool;
027    import com.liferay.portal.kernel.util.StringUtil;
028    import com.liferay.portal.kernel.util.Validator;
029    import com.liferay.portal.kernel.webcache.WebCacheItem;
030    import com.liferay.portal.util.InitUtil;
031    import com.liferay.portlet.translator.model.Translation;
032    import com.liferay.portlet.translator.util.TranslationWebCacheItem;
033    
034    import java.io.File;
035    import java.io.FileInputStream;
036    import java.io.FileOutputStream;
037    import java.io.FileWriter;
038    import java.io.IOException;
039    import java.io.InputStream;
040    
041    import java.util.Map;
042    import java.util.Properties;
043    import java.util.Set;
044    import java.util.TreeSet;
045    
046    /**
047     * @author Brian Wing Shun Chan
048     */
049    public class LangBuilder {
050    
051            public static final String AUTOMATIC_COPY = " (Automatic Copy)";
052    
053            public static final String AUTOMATIC_TRANSLATION =
054                    " (Automatic Translation)";
055    
056            public static void main(String[] args) {
057                    Map<String, String> arguments = ArgumentsUtil.parseArguments(args);
058    
059                    System.setProperty("line.separator", StringPool.NEW_LINE);
060    
061                    InitUtil.initWithSpring();
062    
063                    String langDir = arguments.get("lang.dir");
064                    String langFile = arguments.get("lang.file");
065                    boolean langPlugin = GetterUtil.getBoolean(
066                            arguments.get("lang.plugin"));
067                    boolean langTranslate = GetterUtil.getBoolean(
068                            arguments.get("lang.translate"), true);
069    
070                    try {
071                            new LangBuilder(langDir, langFile, langPlugin, langTranslate);
072                    }
073                    catch (Exception e) {
074                            e.printStackTrace();
075                    }
076            }
077    
078            public LangBuilder(
079                            String langDir, String langFile, boolean langPlugin,
080                            boolean langTranslate)
081                    throws Exception {
082    
083                    _langDir = langDir;
084                    _langFile = langFile;
085                    _langTranslate = langTranslate;
086    
087                    if (langPlugin) {
088                            _portalLanguageProperties = new Properties();
089    
090                            Class<?> clazz = getClass();
091    
092                            ClassLoader classLoader = clazz.getClassLoader();
093    
094                            InputStream inputStream = classLoader.getResourceAsStream(
095                                    "content/Language.properties");
096    
097                            _portalLanguageProperties.load(inputStream);
098                    }
099    
100                    File renameKeysFile = new File(_langDir + "/rename.properties");
101    
102                    if (renameKeysFile.exists()) {
103                            _renameKeys = PropertiesUtil.load(FileUtil.read(renameKeysFile));
104                    }
105    
106                    String content = _orderProperties(
107                            new File(_langDir + "/" + _langFile + ".properties"));
108    
109                    // Locales that are not invoked by _createProperties should still be
110                    // rewritten to use the right line separator
111    
112                    _orderProperties(
113                            new File(_langDir + "/" + _langFile + "_en_GB.properties"));
114    
115                    _createProperties(content, "ar"); // Arabic
116                    _createProperties(content, "eu"); // Basque
117                    _createProperties(content, "bg"); // Bulgarian
118                    _createProperties(content, "ca"); // Catalan
119                    _createProperties(content, "zh_CN"); // Chinese (China)
120                    _createProperties(content, "zh_TW"); // Chinese (Taiwan)
121                    _createProperties(content, "hr"); // Croatian
122                    _createProperties(content, "cs"); // Czech
123                    _createProperties(content, "da"); // Danish
124                    _createProperties(content, "nl"); // Dutch (Netherlands)
125                    _createProperties(content, "nl_BE", "nl"); // Dutch (Belgium)
126                    _createProperties(content, "et"); // Estonian
127                    _createProperties(content, "fi"); // Finnish
128                    _createProperties(content, "fr"); // French
129                    _createProperties(content, "gl"); // Galician
130                    _createProperties(content, "de"); // German
131                    _createProperties(content, "el"); // Greek
132                    _createProperties(content, "iw"); // Hebrew
133                    _createProperties(content, "hi_IN"); // Hindi (India)
134                    _createProperties(content, "hu"); // Hungarian
135                    _createProperties(content, "in"); // Indonesian
136                    _createProperties(content, "it"); // Italian
137                    _createProperties(content, "ja"); // Japanese
138                    _createProperties(content, "ko"); // Korean
139                    _createProperties(content, "lo"); // Lao
140                    _createProperties(content, "nb"); // Norwegian Bokm??l
141                    _createProperties(content, "fa"); // Persian
142                    _createProperties(content, "pl"); // Polish
143                    _createProperties(content, "pt_BR"); // Portuguese (Brazil)
144                    _createProperties(content, "pt_PT", "pt_BR"); // Portuguese (Portugal)
145                    _createProperties(content, "ro"); // Romanian
146                    _createProperties(content, "ru"); // Russian
147                    _createProperties(content, "sr_RS"); // Serbian (Cyrillic)
148                    _createProperties(content, "sr_RS_latin"); // Serbian (Latin)
149                    _createProperties(content, "sk"); // Slovak
150                    _createProperties(content, "sl"); // Slovene
151                    _createProperties(content, "es"); // Spanish
152                    _createProperties(content, "sv"); // Swedish
153                    _createProperties(content, "tr"); // Turkish
154                    _createProperties(content, "uk"); // Ukrainian
155                    _createProperties(content, "vi"); // Vietnamese
156            }
157    
158            private void _createProperties(String content, String languageId)
159                    throws IOException {
160    
161                    _createProperties(content, languageId, null);
162            }
163    
164            private void _createProperties(
165                            String content, String languageId, String parentLanguageId)
166                    throws IOException {
167    
168                    File propertiesFile = new File(
169                            _langDir + "/" + _langFile + "_" + languageId + ".properties");
170    
171                    Properties properties = new Properties();
172    
173                    if (propertiesFile.exists()) {
174                            properties = PropertiesUtil.load(
175                                    new FileInputStream(propertiesFile), StringPool.UTF8);
176                    }
177    
178                    Properties parentProperties = null;
179    
180                    if (parentLanguageId != null) {
181                            File parentPropertiesFile = new File(
182                                    _langDir + "/" + _langFile + "_" + parentLanguageId +
183                                            ".properties");
184    
185                            if (parentPropertiesFile.exists()) {
186                                    parentProperties = new Properties();
187    
188                                    parentProperties = PropertiesUtil.load(
189                                            new FileInputStream(parentPropertiesFile), StringPool.UTF8);
190                            }
191                    }
192    
193                    String translationId = "en_" + languageId;
194    
195                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
196                            new UnsyncStringReader(content));
197                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
198                            new OutputStreamWriter(
199                                    new FileOutputStream(propertiesFile), StringPool.UTF8));
200    
201                    boolean firstLine = true;
202                    int state = 0;
203    
204                    String line = null;
205    
206                    while ((line = unsyncBufferedReader.readLine()) != null) {
207                            line = line.trim();
208    
209                            int pos = line.indexOf("=");
210    
211                            if (pos != -1) {
212                                    String key = line.substring(0, pos);
213                                    String value = line.substring(pos + 1);
214    
215                                    if (((state == 1) && !key.startsWith("lang.")) ||
216                                            ((state == 2) && !key.startsWith("javax.portlet.")) ||
217                                            ((state == 3) && !key.startsWith("category.")) ||
218                                            ((state == 4) && !key.startsWith("model.resource.")) ||
219                                            ((state == 5) && !key.startsWith("action.")) ||
220                                            ((state == 7) && !key.startsWith("currency.")) ||
221                                            ((state != 7) && key.startsWith("currency."))) {
222    
223                                            throw new RuntimeException(
224                                                    "File " + languageId + " with state " + state +
225                                                            " has key " + key);
226                                    }
227    
228                                    String translatedText = properties.getProperty(key);
229    
230                                    if ((translatedText == null) && (parentProperties != null)) {
231                                            translatedText = parentProperties.getProperty(key);
232                                    }
233    
234                                    if ((translatedText == null) && (_renameKeys != null)) {
235                                            String renameKey = _renameKeys.getProperty(key);
236    
237                                            if (renameKey != null) {
238                                                    translatedText = properties.getProperty(key);
239    
240                                                    if ((translatedText == null) &&
241                                                            (parentProperties != null)) {
242    
243                                                            translatedText = parentProperties.getProperty(key);
244                                                    }
245                                            }
246                                    }
247    
248                                    if (translatedText != null) {
249                                            if (translatedText.contains("Babel Fish") ||
250                                                    translatedText.contains("Yahoo! - 999")) {
251    
252                                                    translatedText = "";
253                                            }
254                                            else if (translatedText.endsWith(AUTOMATIC_COPY)) {
255                                                    translatedText = value + AUTOMATIC_COPY;
256                                            }
257                                    }
258    
259                                    if ((translatedText == null) || translatedText.equals("")) {
260                                            if (line.contains("{") || line.contains("<")) {
261                                                    translatedText = value + AUTOMATIC_COPY;
262                                            }
263                                            else if (line.contains("[")) {
264                                                    pos = line.indexOf("[");
265    
266                                                    String baseKey = line.substring(0, pos);
267    
268                                                    String translatedBaseKey = properties.getProperty(
269                                                            baseKey);
270    
271                                                    if (Validator.isNotNull(translatedBaseKey)) {
272                                                            translatedText = translatedBaseKey + AUTOMATIC_COPY;
273                                                    }
274                                                    else {
275                                                            translatedText = value + AUTOMATIC_COPY;
276                                                    }
277                                            }
278                                            else if (key.equals("lang.dir")) {
279                                                    translatedText = "ltr";
280                                            }
281                                            else if (key.equals("lang.line.begin")) {
282                                                    translatedText = "left";
283                                            }
284                                            else if (key.equals("lang.line.end")) {
285                                                    translatedText = "right";
286                                            }
287                                            else if (translationId.equals("en_el") &&
288                                                             (key.equals("enabled") || key.equals("on") ||
289                                                              key.equals("on-date"))) {
290    
291                                                    translatedText = "";
292                                            }
293                                            else if (translationId.equals("en_es") &&
294                                                             key.equals("am")) {
295    
296                                                    translatedText = "";
297                                            }
298                                            else if (translationId.equals("en_it") &&
299                                                             key.equals("am")) {
300    
301                                                    translatedText = "";
302                                            }
303                                            else if (translationId.equals("en_ja") &&
304                                                             (key.equals("any") || key.equals("anytime") ||
305                                                              key.equals("down") || key.equals("on") ||
306                                                              key.equals("on-date") || key.equals("the"))) {
307    
308                                                    translatedText = "";
309                                            }
310                                            else if (translationId.equals("en_ko") &&
311                                                             key.equals("the")) {
312    
313                                                    translatedText = "";
314                                            }
315                                            else {
316                                                    translatedText = _translate(
317                                                            translationId, key, value, 0);
318    
319                                                    if (Validator.isNull(translatedText)) {
320                                                            translatedText = value + AUTOMATIC_COPY;
321                                                    }
322                                                    else {
323                                                            translatedText =
324                                                                    translatedText + AUTOMATIC_TRANSLATION;
325                                                    }
326                                            }
327                                    }
328    
329                                    if (Validator.isNotNull(translatedText)) {
330                                            if (translatedText.contains("Babel Fish") ||
331                                                    translatedText.contains("Yahoo! - 999")) {
332    
333                                                    throw new IOException(
334                                                            "IP was blocked because of over usage. Please " +
335                                                                    "use another IP.");
336                                            }
337    
338                                            translatedText = _fixTranslation(translatedText);
339    
340                                            if (firstLine) {
341                                                    firstLine = false;
342                                            }
343                                            else {
344                                                    unsyncBufferedWriter.newLine();
345                                            }
346    
347                                            unsyncBufferedWriter.write(key + "=" + translatedText);
348    
349                                            unsyncBufferedWriter.flush();
350                                    }
351                            }
352                            else {
353                                    if (line.startsWith("## Language settings")) {
354                                            if (state == 1) {
355                                                    throw new RuntimeException(languageId);
356                                            }
357    
358                                            state = 1;
359                                    }
360                                    else if (line.startsWith(
361                                                            "## Portlet descriptions and titles")) {
362    
363                                            if (state == 2) {
364                                                    throw new RuntimeException(languageId);
365                                            }
366    
367                                            state = 2;
368                                    }
369                                    else if (line.startsWith("## Category titles")) {
370                                            if (state == 3) {
371                                                    throw new RuntimeException(languageId);
372                                            }
373    
374                                            state = 3;
375                                    }
376                                    else if (line.startsWith("## Model resources")) {
377                                            if (state == 4) {
378                                                    throw new RuntimeException(languageId);
379                                            }
380    
381                                            state = 4;
382                                    }
383                                    else if (line.startsWith("## Action names")) {
384                                            if (state == 5) {
385                                                    throw new RuntimeException(languageId);
386                                            }
387    
388                                            state = 5;
389                                    }
390                                    else if (line.startsWith("## Messages")) {
391                                            if (state == 6) {
392                                                    throw new RuntimeException(languageId);
393                                            }
394    
395                                            state = 6;
396                                    }
397                                    else if (line.startsWith("## Currency")) {
398                                            if (state == 7) {
399                                                    throw new RuntimeException(languageId);
400                                            }
401    
402                                            state = 7;
403                                    }
404    
405                                    if (firstLine) {
406                                            firstLine = false;
407                                    }
408                                    else {
409                                            unsyncBufferedWriter.newLine();
410                                    }
411    
412                                    unsyncBufferedWriter.write(line);
413    
414                                    unsyncBufferedWriter.flush();
415                            }
416                    }
417    
418                    unsyncBufferedReader.close();
419                    unsyncBufferedWriter.close();
420            }
421    
422            private String _fixEnglishTranslation(String key, String value) {
423    
424                    // http://en.wikibooks.org/wiki/Basic_Book_Design/Capitalizing_Words_in_Titles
425                    // http://www.imdb.com
426    
427                    if (value.contains(" this ")) {
428                            if (value.contains(".") || value.contains("?") ||
429                                    value.contains(":") ||
430                                    key.equals("the-url-of-the-page-comparing-this-page-content-with-the-previous-version")) {
431                            }
432                            else {
433                                    value = StringUtil.replace(value, " this ", " This ");
434                            }
435                    }
436                    else {
437                            value = StringUtil.replace(value, " From ", " from ");
438                    }
439    
440                    return value;
441            }
442    
443            private String _fixTranslation(String value) {
444                    value = StringUtil.replace(
445                            value.trim(),
446                            new String[] {
447                                    "  ", "<b>", "</b>", "<i>", "</i>", " url ", "&#39;", "&#39 ;",
448                                    "&quot;", "&quot ;", "ReCaptcha", "Captcha"
449                            },
450                            new String[] {
451                                    " ", "<strong>", "</strong>", "<em>", "</em>", " URL ", "\'",
452                                    "\'", "\"", "\"", "reCAPTCHA", "CAPTCHA"
453                            });
454    
455                    return value;
456            }
457    
458            private String _orderProperties(File propertiesFile) throws IOException {
459                    if (!propertiesFile.exists()) {
460                            return null;
461                    }
462    
463                    String content = FileUtil.read(propertiesFile);
464    
465                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
466                            new UnsyncStringReader(content));
467                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
468                            new FileWriter(propertiesFile));
469    
470                    Set<String> messages = new TreeSet<String>(
471                            new NumericalStringComparator(true, true));
472    
473                    boolean begin = false;
474                    boolean firstLine = true;
475    
476                    String line = null;
477    
478                    while ((line = unsyncBufferedReader.readLine()) != null) {
479                            int pos = line.indexOf("=");
480    
481                            if (pos != -1) {
482                                    String key = line.substring(0, pos);
483    
484                                    String value = _fixTranslation(line.substring(pos + 1));
485    
486                                    value = _fixEnglishTranslation(key, value);
487    
488                                    if (_portalLanguageProperties != null) {
489                                            String portalValue = String.valueOf(
490                                                    _portalLanguageProperties.get(key));
491    
492                                            if (value.equals(portalValue)) {
493                                                    System.out.println("Duplicate key " + key);
494                                            }
495                                    }
496    
497                                    messages.add(key + "=" + value);
498                            }
499                            else {
500                                    if (begin && line.equals(StringPool.BLANK)) {
501                                            _sortAndWrite(unsyncBufferedWriter, messages, firstLine);
502                                    }
503    
504                                    if (line.equals(StringPool.BLANK)) {
505                                            begin = !begin;
506                                    }
507    
508                                    if (firstLine) {
509                                            firstLine = false;
510                                    }
511                                    else {
512                                            unsyncBufferedWriter.newLine();
513                                    }
514    
515                                    unsyncBufferedWriter.write(line);
516                            }
517    
518                            unsyncBufferedWriter.flush();
519                    }
520    
521                    if (!messages.isEmpty()) {
522                            _sortAndWrite(unsyncBufferedWriter, messages, firstLine);
523                    }
524    
525                    unsyncBufferedReader.close();
526                    unsyncBufferedWriter.close();
527    
528                    return FileUtil.read(propertiesFile);
529            }
530    
531            private void _sortAndWrite(
532                            UnsyncBufferedWriter unsyncBufferedWriter, Set<String> messages,
533                            boolean firstLine)
534                    throws IOException {
535    
536                    String[] messagesArray = messages.toArray(new String[messages.size()]);
537    
538                    for (int i = 0; i < messagesArray.length; i++) {
539                            if (!firstLine || (i != 0)) {
540                                    unsyncBufferedWriter.newLine();
541                            }
542    
543                            unsyncBufferedWriter.write(messagesArray[i]);
544                    }
545    
546                    messages.clear();
547            }
548    
549            private String _translate(
550                    String translationId, String key, String fromText, int limit) {
551    
552                    if (translationId.equals("en_ar") ||
553                            translationId.equals("en_eu") ||
554                            translationId.equals("en_bg") ||
555                            translationId.equals("en_ca") ||
556                            translationId.equals("en_hr") ||
557                            translationId.equals("en_cs") ||
558                            translationId.equals("en_da") ||
559                            translationId.equals("en_fi") ||
560                            translationId.equals("en_gl") ||
561                            translationId.equals("en_iw") ||
562                            translationId.equals("en_hi") ||
563                            translationId.equals("en_hu") ||
564                            translationId.equals("en_in") ||
565                            translationId.equals("en_lo") ||
566                            translationId.equals("en_nb") ||
567                            translationId.equals("en_fa") ||
568                            translationId.equals("en_pl") ||
569                            translationId.equals("en_ro") ||
570                            translationId.equals("en_ru") ||
571                            translationId.equals("en_sr_RS") ||
572                            translationId.equals("en_sr_RS_latin") ||
573                            translationId.equals("en_sk") ||
574                            translationId.equals("en_sl") ||
575                            translationId.equals("en_sv") ||
576                            translationId.equals("en_tr") ||
577                            translationId.equals("en_uk") ||
578                            translationId.equals("en_vi") ||
579                            translationId.equals("en_et")) {
580    
581                            // Automatic translator does not support Arabic, Basque, Bulgarian,
582                            // Catalan, Czech, Croatian, Danish, Finnish, Galician, Hebrew,
583                            // Hindi, Hungarian, Indonesian, Lao, Norwegian Bokm??l, Persian,
584                            // Polish, Romanian, Russian, Serbian, Slovak, Slovene, Swedish,
585                            // Turkish, Ukrainian, or Vietnamese
586    
587                            return null;
588                    }
589    
590                    if (!_langTranslate) {
591                            return null;
592                    }
593    
594                    // Limit the number of retries to 3
595    
596                    if (limit == 3) {
597                            return null;
598                    }
599    
600                    String toText = null;
601    
602                    try {
603                            System.out.println(
604                                    "Translating " + translationId + " " + key + " " + fromText);
605    
606                            WebCacheItem wci = new TranslationWebCacheItem(
607                                    translationId, fromText);
608    
609                            Translation translation = (Translation)wci.convert("");
610    
611                            toText = translation.getToText();
612                    }
613                    catch (Exception e) {
614                            Throwable cause = e.getCause();
615    
616                            if (cause instanceof MicrosoftTranslatorException) {
617                                    System.out.println(
618                                            cause.getClass().getName() + ": " + cause.getMessage());
619                            }
620                            else {
621                                    e.printStackTrace();
622                            }
623                    }
624    
625                    // Keep trying
626    
627                    if (toText == null) {
628                            return _translate(translationId, key, fromText, ++limit);
629                    }
630    
631                    return toText;
632            }
633    
634            private String _langDir;
635            private String _langFile;
636            private boolean _langTranslate;
637            private Properties _portalLanguageProperties;
638            private Properties _renameKeys;
639    
640    }