001    /**
002     * Copyright (c) 2000-2012 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portal.tools;
016    
017    import com.liferay.portal.kernel.io.OutputStreamWriter;
018    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedReader;
019    import com.liferay.portal.kernel.io.unsync.UnsyncBufferedWriter;
020    import com.liferay.portal.kernel.io.unsync.UnsyncStringReader;
021    import com.liferay.portal.kernel.util.FileUtil;
022    import com.liferay.portal.kernel.util.GetterUtil;
023    import com.liferay.portal.kernel.util.PropertiesUtil;
024    import com.liferay.portal.kernel.util.StringPool;
025    import com.liferay.portal.kernel.util.StringUtil;
026    import com.liferay.portal.kernel.util.Validator;
027    import com.liferay.portal.kernel.webcache.WebCacheItem;
028    import com.liferay.portal.util.InitUtil;
029    import com.liferay.portlet.translator.model.Translation;
030    import com.liferay.portlet.translator.util.TranslationWebCacheItem;
031    
032    import java.io.File;
033    import java.io.FileInputStream;
034    import java.io.FileOutputStream;
035    import java.io.FileWriter;
036    import java.io.IOException;
037    import java.io.InputStream;
038    
039    import java.util.Map;
040    import java.util.Properties;
041    import java.util.Set;
042    import java.util.TreeSet;
043    
044    /**
045     * @author Brian Wing Shun Chan
046     */
047    public class LangBuilder {
048    
049            public static final String AUTOMATIC_COPY = " (Automatic Copy)";
050    
051            public static final String AUTOMATIC_TRANSLATION =
052                    " (Automatic Translation)";
053    
054            public static void main(String[] args) {
055                    Map<String, String> arguments = ArgumentsUtil.parseArguments(args);
056    
057                    System.setProperty("line.separator", StringPool.NEW_LINE);
058    
059                    InitUtil.initWithSpring();
060    
061                    String langDir = arguments.get("lang.dir");
062                    String langFile = arguments.get("lang.file");
063                    boolean langPlugin = GetterUtil.getBoolean(
064                            arguments.get("lang.plugin"));
065                    boolean langTranslate = GetterUtil.getBoolean(
066                            arguments.get("lang.translate"), true);
067    
068                    try {
069                            new LangBuilder(langDir, langFile, langPlugin, langTranslate);
070                    }
071                    catch (Exception e) {
072                            e.printStackTrace();
073                    }
074            }
075    
076            public LangBuilder(
077                            String langDir, String langFile, boolean langPlugin,
078                            boolean langTranslate)
079                    throws Exception {
080    
081                    _langDir = langDir;
082                    _langFile = langFile;
083                    _langTranslate = langTranslate;
084    
085                    if (langPlugin) {
086                            _portalLanguageProperties = new Properties();
087    
088                            Class<?> clazz = getClass();
089    
090                            ClassLoader classLoader = clazz.getClassLoader();
091    
092                            InputStream inputStream = classLoader.getResourceAsStream(
093                                    "content/Language.properties");
094    
095                            _portalLanguageProperties.load(inputStream);
096                    }
097    
098                    File renameKeysFile = new File(_langDir + "/rename.properties");
099    
100                    if (renameKeysFile.exists()) {
101                            _renameKeys = PropertiesUtil.load(FileUtil.read(renameKeysFile));
102                    }
103    
104                    String content = _orderProperties(
105                            new File(_langDir + "/" + _langFile + ".properties"));
106    
107                    // Locales that are not invoked by _createProperties should still be
108                    // rewritten to use the rignt line separator
109    
110                    _orderProperties(
111                            new File(_langDir + "/" + _langFile + "_en_GB.properties"));
112    
113                    _createProperties(content, "ar"); // Arabic
114                    _createProperties(content, "eu"); // Basque
115                    _createProperties(content, "bg"); // Bulgarian
116                    _createProperties(content, "ca"); // Catalan
117                    _createProperties(content, "zh_CN"); // Chinese (China)
118                    _createProperties(content, "zh_TW"); // Chinese (Taiwan)
119                    _createProperties(content, "hr"); // Croatian
120                    _createProperties(content, "cs"); // Czech
121                    _createProperties(content, "nl"); // Dutch (Netherlands)
122                    _createProperties(content, "nl_BE", "nl"); // Dutch (Belgium)
123                    _createProperties(content, "et"); // Estonian
124                    _createProperties(content, "fi"); // Finnish
125                    _createProperties(content, "fr"); // French
126                    _createProperties(content, "gl"); // Galician
127                    _createProperties(content, "de"); // German
128                    _createProperties(content, "el"); // Greek
129                    _createProperties(content, "iw"); // Hebrew
130                    _createProperties(content, "hi_IN"); // Hindi (India)
131                    _createProperties(content, "hu"); // Hungarian
132                    _createProperties(content, "in"); // Indonesian
133                    _createProperties(content, "it"); // Italian
134                    _createProperties(content, "ja"); // Japanese
135                    _createProperties(content, "ko"); // Korean
136                    _createProperties(content, "nb"); // Norwegian Bokmål
137                    _createProperties(content, "fa"); // Persian
138                    _createProperties(content, "pl"); // Polish
139                    _createProperties(content, "pt_BR"); // Portuguese (Brazil)
140                    _createProperties(content, "pt_PT", "pt_BR"); // Portuguese (Portugal)
141                    _createProperties(content, "ro"); // Romanian
142                    _createProperties(content, "ru"); // Russian
143                    _createProperties(content, "sr_RS"); // Serbian (Cyrillic)
144                    _createProperties(content, "sr_RS_latin"); // Serbian (Latin)
145                    _createProperties(content, "sk"); // Slovak
146                    _createProperties(content, "sl"); // Slovene
147                    _createProperties(content, "es"); // Spanish
148                    _createProperties(content, "sv"); // Swedish
149                    _createProperties(content, "tr"); // Turkish
150                    _createProperties(content, "uk"); // Ukrainian
151                    _createProperties(content, "vi"); // Vietnamese
152            }
153    
154            private void _createProperties(String content, String languageId)
155                    throws IOException {
156    
157                    _createProperties(content, languageId, null);
158            }
159    
160            private void _createProperties(
161                            String content, String languageId, String parentLanguageId)
162                    throws IOException {
163    
164                    File propertiesFile = new File(
165                            _langDir + "/" + _langFile + "_" + languageId + ".properties");
166    
167                    Properties properties = new Properties();
168    
169                    if (propertiesFile.exists()) {
170                            properties = PropertiesUtil.load(
171                                    new FileInputStream(propertiesFile), StringPool.UTF8);
172                    }
173    
174                    Properties parentProperties = null;
175    
176                    if (parentLanguageId != null) {
177                            File parentPropertiesFile = new File(
178                                    _langDir + "/" + _langFile + "_" + parentLanguageId +
179                                            ".properties");
180    
181                            if (parentPropertiesFile.exists()) {
182                                    parentProperties = new Properties();
183    
184                                    parentProperties = PropertiesUtil.load(
185                                            new FileInputStream(parentPropertiesFile), StringPool.UTF8);
186                            }
187                    }
188    
189                    String translationId = "en_" + languageId;
190    
191                    if (translationId.equals("en_pt_BR")) {
192                            translationId = "en_pt";
193                    }
194                    else if (translationId.equals("en_pt_PT")) {
195                            translationId = "en_pt";
196                    }
197                    else if (translationId.equals("en_zh_CN")) {
198                            translationId = "en_zh";
199                    }
200                    else if (translationId.equals("en_zh_TW")) {
201                            translationId = "en_zt";
202                    }
203                    else if (translationId.equals("en_hi_IN")) {
204                            translationId = "en_hi";
205                    }
206    
207                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
208                            new UnsyncStringReader(content));
209                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
210                            new OutputStreamWriter(
211                                    new FileOutputStream(propertiesFile), StringPool.UTF8));
212    
213                    int state = 0;
214    
215                    String line = null;
216    
217                    while ((line = unsyncBufferedReader.readLine()) != null) {
218                            line = line.trim();
219    
220                            int pos = line.indexOf("=");
221    
222                            if (pos != -1) {
223                                    String key = line.substring(0, pos);
224                                    String value = line.substring(pos + 1, line.length());
225    
226                                    if (((state == 1) && !key.startsWith("lang.")) ||
227                                            ((state == 2) && !key.startsWith("javax.portlet.")) ||
228                                            ((state == 3) && !key.startsWith("category.")) ||
229                                            ((state == 4) && !key.startsWith("model.resource.")) ||
230                                            ((state == 5) && !key.startsWith("action.")) ||
231                                            ((state == 7) && !key.startsWith("currency.")) ||
232                                            ((state != 7) && key.startsWith("currency."))) {
233    
234                                            throw new RuntimeException(
235                                                    "File " + languageId + " with state " + state +
236                                                            " has key " + key);
237                                    }
238    
239                                    String translatedText = properties.getProperty(key);
240    
241                                    if ((translatedText == null) && (parentProperties != null)) {
242                                            translatedText = parentProperties.getProperty(key);
243                                    }
244    
245                                    if ((translatedText == null) && (_renameKeys != null)) {
246                                            String renameKey = _renameKeys.getProperty(key);
247    
248                                            if (renameKey != null) {
249                                                    translatedText = properties.getProperty(key);
250    
251                                                    if ((translatedText == null) &&
252                                                            (parentProperties != null)) {
253    
254                                                            translatedText = parentProperties.getProperty(key);
255                                                    }
256                                            }
257                                    }
258    
259                                    if (translatedText != null) {
260                                            if (translatedText.contains("Babel Fish") ||
261                                                    translatedText.contains("Yahoo! - 999")) {
262    
263                                                    translatedText = "";
264                                            }
265                                            else if (translatedText.endsWith(AUTOMATIC_COPY)) {
266                                                    translatedText = value + AUTOMATIC_COPY;
267                                            }
268                                    }
269    
270                                    if ((translatedText == null) || translatedText.equals("")) {
271                                            if (line.contains("{") || line.contains("<")) {
272                                                    translatedText = value + AUTOMATIC_COPY;
273                                            }
274                                            else if (line.contains("[")) {
275                                                    pos = line.indexOf("[");
276    
277                                                    String baseKey = line.substring(0, pos);
278    
279                                                    translatedText =
280                                                            properties.getProperty(baseKey) + AUTOMATIC_COPY;
281                                            }
282                                            else if (key.equals("lang.dir")) {
283                                                    translatedText = "ltr";
284                                            }
285                                            else if (key.equals("lang.line.begin")) {
286                                                    translatedText = "left";
287                                            }
288                                            else if (key.equals("lang.line.end")) {
289                                                    translatedText = "right";
290                                            }
291                                            else if (translationId.equals("en_el") &&
292                                                             (key.equals("enabled") || key.equals("on") ||
293                                                              key.equals("on-date"))) {
294    
295                                                    translatedText = "";
296                                            }
297                                            else if (translationId.equals("en_es") &&
298                                                             key.equals("am")) {
299    
300                                                    translatedText = "";
301                                            }
302                                            else if (translationId.equals("en_it") &&
303                                                             key.equals("am")) {
304    
305                                                    translatedText = "";
306                                            }
307                                            else if (translationId.equals("en_ja") &&
308                                                             (key.equals("any") || key.equals("anytime") ||
309                                                              key.equals("down") || key.equals("on") ||
310                                                              key.equals("on-date") || key.equals("the"))) {
311    
312                                                    translatedText = "";
313                                            }
314                                            else if (translationId.equals("en_ko") &&
315                                                             key.equals("the")) {
316    
317                                                    translatedText = "";
318                                            }
319                                            else {
320                                                    translatedText = _translate(
321                                                            translationId, key, value, 0);
322    
323                                                    if (Validator.isNull(translatedText)) {
324                                                            translatedText = value + AUTOMATIC_COPY;
325                                                    }
326                                                    else {
327                                                            translatedText =
328                                                                    translatedText + AUTOMATIC_TRANSLATION;
329                                                    }
330                                            }
331                                    }
332    
333                                    if (Validator.isNotNull(translatedText)) {
334                                            if (translatedText.contains("Babel Fish") ||
335                                                    translatedText.contains("Yahoo! - 999")) {
336    
337                                                    throw new IOException(
338                                                            "IP was blocked because of over usage. Please " +
339                                                                    "use another IP.");
340                                            }
341    
342                                            translatedText = _fixTranslation(translatedText);
343    
344                                            unsyncBufferedWriter.write(key + "=" + translatedText);
345    
346                                            unsyncBufferedWriter.newLine();
347                                            unsyncBufferedWriter.flush();
348                                    }
349                            }
350                            else {
351                                    if (line.startsWith("## Language settings")) {
352                                            if (state == 1) {
353                                                    throw new RuntimeException(languageId);
354                                            }
355    
356                                            state = 1;
357                                    }
358                                    else if (line.startsWith(
359                                                            "## Portlet descriptions and titles")) {
360    
361                                            if (state == 2) {
362                                                    throw new RuntimeException(languageId);
363                                            }
364    
365                                            state = 2;
366                                    }
367                                    else if (line.startsWith("## Category titles")) {
368                                            if (state == 3) {
369                                                    throw new RuntimeException(languageId);
370                                            }
371    
372                                            state = 3;
373                                    }
374                                    else if (line.startsWith("## Model resources")) {
375                                            if (state == 4) {
376                                                    throw new RuntimeException(languageId);
377                                            }
378    
379                                            state = 4;
380                                    }
381                                    else if (line.startsWith("## Action names")) {
382                                            if (state == 5) {
383                                                    throw new RuntimeException(languageId);
384                                            }
385    
386                                            state = 5;
387                                    }
388                                    else if (line.startsWith("## Messages")) {
389                                            if (state == 6) {
390                                                    throw new RuntimeException(languageId);
391                                            }
392    
393                                            state = 6;
394                                    }
395                                    else if (line.startsWith("## Currency")) {
396                                            if (state == 7) {
397                                                    throw new RuntimeException(languageId);
398                                            }
399    
400                                            state = 7;
401                                    }
402    
403                                    unsyncBufferedWriter.write(line);
404    
405                                    unsyncBufferedWriter.newLine();
406                                    unsyncBufferedWriter.flush();
407                            }
408                    }
409    
410                    unsyncBufferedReader.close();
411                    unsyncBufferedWriter.close();
412            }
413    
414            private String _fixEnglishTranslation(String key, String value) {
415                    if (value.contains(" this ")) {
416                            if (value.contains(".") || value.contains("?") ||
417                                    value.contains(":") ||
418                                    key.equals("the-url-of-the-page-comparing-this-page-content-with-the-previous-version")) {
419                            }
420                            else {
421                                    value = StringUtil.replace(value, " this ", " This ");
422                            }
423                    }
424    
425                    return value;
426            }
427    
428            private String _fixTranslation(String value) {
429                    value = StringUtil.replace(
430                            value.trim(),
431                            new String[] {
432                                    "  ", "<b>", "</b>", "<i>", "</i>", " url ", "&#39;",
433                                    "&#39 ;", "&quot;", "&quot ;"
434                            },
435                            new String[] {
436                                    " ", "<strong>", "</strong>", "<em>", "</em>", " URL ", "\'",
437                                    "\'", "\"", "\""
438                            });
439    
440                    return value;
441            }
442    
443            private String _orderProperties(File propertiesFile) throws IOException {
444                    if (!propertiesFile.exists()) {
445                            return null;
446                    }
447    
448                    String content = FileUtil.read(propertiesFile);
449    
450                    UnsyncBufferedReader unsyncBufferedReader = new UnsyncBufferedReader(
451                            new UnsyncStringReader(content));
452                    UnsyncBufferedWriter unsyncBufferedWriter = new UnsyncBufferedWriter(
453                            new FileWriter(propertiesFile));
454    
455                    Set<String> messages = new TreeSet<String>();
456    
457                    boolean begin = false;
458    
459                    String line = null;
460    
461                    while ((line = unsyncBufferedReader.readLine()) != null) {
462                            int pos = line.indexOf("=");
463    
464                            if (pos != -1) {
465                                    String key = line.substring(0, pos);
466    
467                                    String value = _fixTranslation(
468                                            line.substring(pos + 1, line.length()));
469    
470                                    value = _fixEnglishTranslation(key, value);
471    
472                                    if (_portalLanguageProperties != null) {
473                                            String portalValue = String.valueOf(
474                                                    _portalLanguageProperties.get(key));
475    
476                                            if (value.equals(portalValue)) {
477                                                    System.out.println("Duplicate key " + key);
478                                            }
479                                    }
480    
481                                    messages.add(key + "=" + value);
482                            }
483                            else {
484                                    if (begin == true && line.equals("")) {
485                                            _sortAndWrite(unsyncBufferedWriter, messages);
486                                    }
487    
488                                    if (line.equals("")) {
489                                            begin = !begin;
490                                    }
491    
492                                    unsyncBufferedWriter.write(line);
493                                    unsyncBufferedWriter.newLine();
494                            }
495    
496                            unsyncBufferedWriter.flush();
497                    }
498    
499                    if (messages.size() > 0) {
500                            _sortAndWrite(unsyncBufferedWriter, messages);
501                    }
502    
503                    unsyncBufferedReader.close();
504                    unsyncBufferedWriter.close();
505    
506                    return FileUtil.read(propertiesFile);
507            }
508    
509            private void _sortAndWrite(
510                            UnsyncBufferedWriter unsyncBufferedWriter, Set<String> messages)
511                    throws IOException {
512    
513                    String[] messagesArray = messages.toArray(new String[messages.size()]);
514    
515                    for (int i = 0; i < messagesArray.length; i++) {
516                            unsyncBufferedWriter.write(messagesArray[i]);
517                            unsyncBufferedWriter.newLine();
518                    }
519    
520                    messages.clear();
521            }
522    
523            private String _translate(
524                    String translationId, String key, String fromText, int limit) {
525    
526                    if (translationId.equals("en_ar") ||
527                            translationId.equals("en_eu") ||
528                            translationId.equals("en_bg") ||
529                            translationId.equals("en_ca") ||
530                            translationId.equals("en_hr") ||
531                            translationId.equals("en_cs") ||
532                            translationId.equals("en_fi") ||
533                            translationId.equals("en_gl") ||
534                            translationId.equals("en_iw") ||
535                            translationId.equals("en_hi") ||
536                            translationId.equals("en_hu") ||
537                            translationId.equals("en_in") ||
538                            translationId.equals("en_nb") ||
539                            translationId.equals("en_fa") ||
540                            translationId.equals("en_pl") ||
541                            translationId.equals("en_ro") ||
542                            translationId.equals("en_ru") ||
543                            translationId.equals("en_sr_RS") ||
544                            translationId.equals("en_sr_RS_latin") ||
545                            translationId.equals("en_sk") ||
546                            translationId.equals("en_sl") ||
547                            translationId.equals("en_sv") ||
548                            translationId.equals("en_tr") ||
549                            translationId.equals("en_uk") ||
550                            translationId.equals("en_vi") ||
551                            translationId.equals("en_et")) {
552    
553                            // Automatic translator does not support Arabic, Basque, Bulgarian,
554                            // Catalan, Czech, Croatian, Finnish, Galician, Hebrew, Hindi,
555                            // Hungarian, Indonesian, Norwegian Bokmål, Persian, Polish,
556                            // Romanian, Russian, Serbian, Slovak, Slovene, Swedish, Turkish,
557                            // Ukrainian, or Vietnamese
558    
559                            return null;
560                    }
561    
562                    if (!_langTranslate) {
563                            return null;
564                    }
565    
566                    // Limit the number of retries to 3
567    
568                    if (limit == 3) {
569                            return null;
570                    }
571    
572                    String toText = null;
573    
574                    try {
575                            System.out.println(
576                                    "Translating " + translationId + " " + key + " " + fromText);
577    
578                            WebCacheItem wci = new TranslationWebCacheItem(
579                                    translationId, fromText);
580    
581                            Translation translation = (Translation)wci.convert("");
582    
583                            toText = translation.getToText();
584    
585                            if ((toText != null) && toText.contains("Babel Fish")) {
586                                    toText = null;
587                            }
588                    }
589                    catch (Exception e) {
590                            e.printStackTrace();
591                    }
592    
593                    // Keep trying
594    
595                    if (toText == null) {
596                            return _translate(translationId, key, fromText, ++limit);
597                    }
598    
599                    return toText;
600            }
601    
602            private String _langDir;
603            private String _langFile;
604            private boolean _langTranslate;
605            private Properties _portalLanguageProperties;
606            private Properties _renameKeys;
607    
608    }