1   /**
2    * Copyright (c) 2000-2008 Liferay, Inc. All rights reserved.
3    *
4    * Permission is hereby granted, free of charge, to any person obtaining a copy
5    * of this software and associated documentation files (the "Software"), to deal
6    * in the Software without restriction, including without limitation the rights
7    * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8    * copies of the Software, and to permit persons to whom the Software is
9    * furnished to do so, subject to the following conditions:
10   *
11   * The above copyright notice and this permission notice shall be included in
12   * all copies or substantial portions of the Software.
13   *
14   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  
23  package com.liferay.portal.search.lucene;
24  
25  import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26  import com.liferay.portal.kernel.search.Field;
27  import com.liferay.portal.kernel.util.FileUtil;
28  import com.liferay.portal.kernel.util.InfrastructureUtil;
29  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
30  import com.liferay.portal.kernel.util.StringPool;
31  import com.liferay.portal.kernel.util.StringUtil;
32  import com.liferay.portal.kernel.util.Validator;
33  import com.liferay.portal.util.PropsKeys;
34  import com.liferay.portal.util.PropsUtil;
35  import com.liferay.portal.util.PropsValues;
36  import com.liferay.util.lucene.KeywordsUtil;
37  
38  import java.io.IOException;
39  
40  import java.sql.Connection;
41  import java.sql.DatabaseMetaData;
42  import java.sql.ResultSet;
43  import java.sql.Statement;
44  
45  import java.util.Date;
46  import java.util.Map;
47  import java.util.concurrent.ConcurrentHashMap;
48  
49  import javax.sql.DataSource;
50  
51  import org.apache.commons.logging.Log;
52  import org.apache.commons.logging.LogFactory;
53  import org.apache.lucene.analysis.Analyzer;
54  import org.apache.lucene.analysis.WhitespaceAnalyzer;
55  import org.apache.lucene.document.Document;
56  import org.apache.lucene.index.IndexReader;
57  import org.apache.lucene.index.IndexWriter;
58  import org.apache.lucene.index.Term;
59  import org.apache.lucene.queryParser.ParseException;
60  import org.apache.lucene.queryParser.QueryParser;
61  import org.apache.lucene.search.BooleanClause;
62  import org.apache.lucene.search.BooleanQuery;
63  import org.apache.lucene.search.IndexSearcher;
64  import org.apache.lucene.search.Query;
65  import org.apache.lucene.search.TermQuery;
66  import org.apache.lucene.search.WildcardQuery;
67  import org.apache.lucene.store.Directory;
68  import org.apache.lucene.store.FSDirectory;
69  import org.apache.lucene.store.RAMDirectory;
70  import org.apache.lucene.store.jdbc.JdbcDirectory;
71  import org.apache.lucene.store.jdbc.JdbcStoreException;
72  import org.apache.lucene.store.jdbc.dialect.Dialect;
73  import org.apache.lucene.store.jdbc.lock.JdbcLock;
74  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
75  
76  /**
77   * <a href="LuceneUtil.java.html"><b><i>View Source</i></b></a>
78   *
79   * @author Brian Wing Shun Chan
80   * @author Harry Mark
81   *
82   */
83  public class LuceneUtil {
84  
85      public static void acquireLock(long companyId) {
86          try {
87              _instance._sharedWriter.acquireLock(companyId, true);
88          }
89          catch (InterruptedException ie) {
90              _log.error(ie);
91          }
92      }
93  
94      public static void addDate(Document doc, String field, Date value) {
95          doc.add(LuceneFields.getDate(field, value));
96      }
97  
98      public static void addExactTerm(
99          BooleanQuery booleanQuery, String field, boolean value) {
100 
101         addExactTerm(booleanQuery, field, String.valueOf(value));
102     }
103 
104     public static void addExactTerm(
105         BooleanQuery booleanQuery, String field, Boolean value) {
106 
107         addExactTerm(booleanQuery, field, String.valueOf(value));
108     }
109 
110     public static void addExactTerm(
111         BooleanQuery booleanQuery, String field, double value) {
112 
113         addExactTerm(booleanQuery, field, String.valueOf(value));
114     }
115 
116     public static void addExactTerm(
117         BooleanQuery booleanQuery, String field, Double value) {
118 
119         addExactTerm(booleanQuery, field, String.valueOf(value));
120     }
121 
122     public static void addExactTerm(
123         BooleanQuery booleanQuery, String field, int value) {
124 
125         addExactTerm(booleanQuery, field, String.valueOf(value));
126     }
127 
128     public static void addExactTerm(
129         BooleanQuery booleanQuery, String field, Integer value) {
130 
131         addExactTerm(booleanQuery, field, String.valueOf(value));
132     }
133 
134     public static void addExactTerm(
135         BooleanQuery booleanQuery, String field, long value) {
136 
137         addExactTerm(booleanQuery, field, String.valueOf(value));
138     }
139 
140     public static void addExactTerm(
141         BooleanQuery booleanQuery, String field, Long value) {
142 
143         addExactTerm(booleanQuery, field, String.valueOf(value));
144     }
145 
146     public static void addExactTerm(
147         BooleanQuery booleanQuery, String field, short value) {
148 
149         addExactTerm(booleanQuery, field, String.valueOf(value));
150     }
151 
152     public static void addExactTerm(
153         BooleanQuery booleanQuery, String field, Short value) {
154 
155         addExactTerm(booleanQuery, field, String.valueOf(value));
156     }
157 
158     public static void addExactTerm(
159         BooleanQuery booleanQuery, String field, String value) {
160 
161         //text = KeywordsUtil.escape(value);
162 
163         Query query = new TermQuery(new Term(field, value));
164 
165         booleanQuery.add(query, BooleanClause.Occur.SHOULD);
166     }
167 
168     public static void addRequiredTerm(
169         BooleanQuery booleanQuery, String field, boolean value) {
170 
171         addRequiredTerm(booleanQuery, field, String.valueOf(value));
172     }
173 
174     public static void addRequiredTerm(
175         BooleanQuery booleanQuery, String field, Boolean value) {
176 
177         addRequiredTerm(booleanQuery, field, String.valueOf(value));
178     }
179 
180     public static void addRequiredTerm(
181         BooleanQuery booleanQuery, String field, double value) {
182 
183         addRequiredTerm(booleanQuery, field, String.valueOf(value));
184     }
185 
186     public static void addRequiredTerm(
187         BooleanQuery booleanQuery, String field, Double value) {
188 
189         addRequiredTerm(booleanQuery, field, String.valueOf(value));
190     }
191 
192     public static void addRequiredTerm(
193         BooleanQuery booleanQuery, String field, int value) {
194 
195         addRequiredTerm(booleanQuery, field, String.valueOf(value));
196     }
197 
198     public static void addRequiredTerm(
199         BooleanQuery booleanQuery, String field, Integer value) {
200 
201         addRequiredTerm(booleanQuery, field, String.valueOf(value));
202     }
203 
204     public static void addRequiredTerm(
205         BooleanQuery booleanQuery, String field, long value) {
206 
207         addRequiredTerm(booleanQuery, field, String.valueOf(value));
208     }
209 
210     public static void addRequiredTerm(
211         BooleanQuery booleanQuery, String field, Long value) {
212 
213         addRequiredTerm(booleanQuery, field, String.valueOf(value));
214     }
215 
216     public static void addRequiredTerm(
217         BooleanQuery booleanQuery, String field, short value) {
218 
219         addRequiredTerm(booleanQuery, field, String.valueOf(value));
220     }
221 
222     public static void addRequiredTerm(
223         BooleanQuery booleanQuery, String field, Short value) {
224 
225         addRequiredTerm(booleanQuery, field, String.valueOf(value));
226     }
227 
228     public static void addRequiredTerm(
229         BooleanQuery booleanQuery, String field, String value) {
230 
231         addRequiredTerm(booleanQuery, field, value, false);
232     }
233 
234     public static void addRequiredTerm(
235         BooleanQuery booleanQuery, String field, String value, boolean like) {
236 
237         if (like) {
238             value = StringUtil.replace(
239                 value, StringPool.PERCENT, StringPool.STAR);
240 
241             value = value.toLowerCase();
242 
243             WildcardQuery wildcardQuery = new WildcardQuery(
244                 new Term(field, value));
245 
246             booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
247         }
248         else {
249             //text = KeywordsUtil.escape(value);
250 
251             Term term = new Term(field, value);
252             TermQuery termQuery = new TermQuery(term);
253 
254             booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
255         }
256     }
257 
258     public static void addModifiedDate(Document doc) {
259         doc.add(LuceneFields.getDate(Field.MODIFIED));
260     }
261 
262     public static void addTerm(
263             BooleanQuery booleanQuery, String field, long value)
264         throws ParseException {
265 
266         addTerm(booleanQuery, field, String.valueOf(value));
267     }
268 
269     public static void addTerm(
270             BooleanQuery booleanQuery, String field, String value)
271         throws ParseException {
272 
273         addTerm(booleanQuery, field, value, false);
274     }
275 
276     public static void addTerm(
277             BooleanQuery booleanQuery, String field, String value,
278             boolean like)
279         throws ParseException {
280 
281         if (Validator.isNull(value)) {
282             return;
283         }
284 
285         if (like) {
286             value = value.toLowerCase();
287 
288             StringBuilder sb = new StringBuilder();
289 
290             sb.append(StringPool.STAR);
291             sb.append(value);
292             sb.append(StringPool.STAR);
293 
294             WildcardQuery wildcardQuery = new WildcardQuery(
295                 new Term(field, sb.toString()));
296 
297             booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
298         }
299         else {
300             QueryParser queryParser = new QueryParser(
301                 field, LuceneUtil.getAnalyzer());
302 
303             try {
304                 Query query = queryParser.parse(value);
305 
306                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
307             }
308             catch (ParseException pe) {
309                 if (_log.isDebugEnabled()) {
310                     _log.debug(
311                         "ParseException thrown, reverting to literal search",
312                         pe);
313                 }
314 
315                 value = KeywordsUtil.escape(value);
316 
317                 Query query = queryParser.parse(value);
318 
319                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
320             }
321         }
322     }
323 
324     public static void checkLuceneDir(long companyId) {
325         if (PropsValues.INDEX_READ_ONLY) {
326             return;
327         }
328 
329         Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
330 
331         try {
332 
333             // LEP-6078
334 
335             if (luceneDir.fileExists("write.lock")) {
336                 luceneDir.deleteFile("write.lock");
337             }
338         }
339         catch (IOException ioe) {
340             _log.error("Unable to clear write lock", ioe);
341         }
342 
343         IndexWriter writer = null;
344 
345         // Lucene does not properly release its lock on the index when
346         // IndexWriter throws an exception
347 
348         try {
349             if (luceneDir.fileExists("segments.gen")) {
350                 writer = new IndexWriter(
351                     luceneDir, LuceneUtil.getAnalyzer(), false);
352             }
353             else {
354                 writer = new IndexWriter(
355                     luceneDir, LuceneUtil.getAnalyzer(), true);
356             }
357         }
358         catch (IOException ioe) {
359             _log.error("Check Lucene directory failed for " + companyId, ioe);
360         }
361         finally {
362             if (writer != null) {
363                 try {
364                     writer.close();
365                 }
366                 catch (IOException ioe) {
367                     _log.error(ioe);
368                 }
369             }
370         }
371     }
372 
373     public static void delete(long companyId) {
374         _instance._delete(companyId);
375     }
376 
377     public static void deleteDocuments(long companyId, Term term)
378         throws IOException {
379 
380         try {
381             _instance._sharedWriter.deleteDocuments(companyId, term);
382         }
383         catch (InterruptedException ie) {
384             _log.error(ie);
385         }
386     }
387 
388     public static Analyzer getAnalyzer() {
389         return _instance._getAnalyzer();
390     }
391 
392     public static Directory getLuceneDir(long companyId) {
393         return _instance._getLuceneDir(companyId);
394     }
395 
396     public static IndexReader getReader(long companyId) throws IOException {
397         return IndexReader.open(getLuceneDir(companyId));
398     }
399 
400     public static IndexSearcher getSearcher(long companyId)
401         throws IOException {
402 
403         return new IndexSearcher(getLuceneDir(companyId));
404     }
405 
406     public static IndexWriter getWriter(long companyId) throws IOException {
407         return getWriter(companyId, false);
408     }
409 
410     public static IndexWriter getWriter(long companyId, boolean create)
411         throws IOException {
412 
413         return _instance._sharedWriter.getWriter(companyId, create);
414     }
415 
416     public static void releaseLock(long companyId) {
417         _instance._sharedWriter.releaseLock(companyId);
418     }
419 
420     public static void write(long companyId) {
421         _instance._sharedWriter.write(companyId);
422     }
423 
424     public static void write(IndexWriter writer) throws IOException {
425         _instance._sharedWriter.write(writer);
426     }
427 
428     private LuceneUtil() {
429         String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
430 
431         if (Validator.isNotNull(analyzerName)) {
432             try {
433                 _analyzerClass = Class.forName(analyzerName);
434             }
435             catch (Exception e) {
436                 _log.error(e);
437             }
438         }
439 
440         // Dialect
441 
442         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
443             Connection con = null;
444 
445             try {
446                 con = DataAccess.getConnection();
447 
448                 String url = con.getMetaData().getURL();
449 
450                 int x = url.indexOf(":");
451                 int y = url.indexOf(":", x + 1);
452 
453                 String urlPrefix = url.substring(x + 1, y);
454 
455                 String dialectClass = PropsUtil.get(
456                     PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
457 
458                 if (dialectClass != null) {
459                     if (_log.isDebugEnabled()) {
460                         _log.debug("JDBC class implementation " + dialectClass);
461                     }
462                 }
463                 else {
464                     if (_log.isDebugEnabled()) {
465                         _log.debug("JDBC class implementation is null");
466                     }
467                 }
468 
469                 if (dialectClass != null) {
470                     _dialect =
471                         (Dialect)Class.forName(dialectClass).newInstance();
472                 }
473             }
474             catch (Exception e) {
475                 _log.error(e);
476             }
477             finally{
478                 DataAccess.cleanUp(con);
479             }
480 
481             if (_dialect == null) {
482                 _log.error("No JDBC dialect found");
483             }
484         }
485     }
486 
487     public void _delete(long companyId) {
488         if (PropsValues.INDEX_READ_ONLY) {
489             return;
490         }
491 
492         if (_log.isDebugEnabled()) {
493             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
494         }
495 
496         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
497             _deleteFile(companyId);
498         }
499         else if (PropsValues.LUCENE_STORE_TYPE.equals(
500                     _LUCENE_STORE_TYPE_JDBC)) {
501 
502             _deleteJdbc(companyId);
503         }
504         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
505             _deleteRam(companyId);
506         }
507         else {
508             throw new RuntimeException(
509                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
510         }
511     }
512 
513     private void _deleteFile(long companyId) {
514         String path = _getPath(companyId);
515 
516         try {
517             Directory directory = FSDirectory.getDirectory(path, false);
518 
519             directory.close();
520         }
521         catch (Exception e) {
522             if (_log.isWarnEnabled()) {
523                 _log.warn("Could not close directory " + path);
524             }
525         }
526 
527         FileUtil.deltree(path);
528     }
529 
530     private void _deleteJdbc(long companyId) {
531         String tableName = _getTableName(companyId);
532 
533         try {
534             Directory directory = _jdbcDirectories.remove(tableName);
535 
536             if (directory != null) {
537                 directory.close();
538             }
539         }
540         catch (Exception e) {
541             if (_log.isWarnEnabled()) {
542                 _log.warn("Could not close directory " + tableName);
543             }
544         }
545 
546         Connection con = null;
547         Statement s = null;
548 
549         try {
550             con = DataAccess.getConnection();
551 
552             s = con.createStatement();
553 
554             s.executeUpdate("DELETE FROM " + tableName);
555         }
556         catch (Exception e) {
557             if (_log.isWarnEnabled()) {
558                 _log.warn("Could not truncate " + tableName);
559             }
560         }
561         finally {
562             DataAccess.cleanUp(con, s);
563         }
564     }
565 
566     private void _deleteRam(long companyId) {
567     }
568 
569     private Analyzer _getAnalyzer() {
570         try {
571             return (Analyzer)_analyzerClass.newInstance();
572         }
573         catch (Exception e) {
574             throw new RuntimeException(e);
575         }
576     }
577 
578     private Directory _getLuceneDir(long companyId) {
579         if (_log.isDebugEnabled()) {
580             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
581         }
582 
583         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
584             return _getLuceneDirFile(companyId);
585         }
586         else if (PropsValues.LUCENE_STORE_TYPE.equals(
587                     _LUCENE_STORE_TYPE_JDBC)) {
588 
589             return _getLuceneDirJdbc(companyId);
590         }
591         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
592             return _getLuceneDirRam(companyId);
593         }
594         else {
595             throw new RuntimeException(
596                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
597         }
598     }
599 
600     private Directory _getLuceneDirFile(long companyId) {
601         Directory directory = null;
602 
603         String path = _getPath(companyId);
604 
605         try {
606             directory = FSDirectory.getDirectory(path, false);
607         }
608         catch (IOException ioe1) {
609             try {
610                 if (directory != null) {
611                     directory.close();
612                 }
613 
614                 directory = FSDirectory.getDirectory(path, true);
615             }
616             catch (IOException ioe2) {
617                 throw new RuntimeException(ioe2);
618             }
619         }
620 
621         return directory;
622     }
623 
624     private Directory _getLuceneDirJdbc(long companyId) {
625         JdbcDirectory directory = null;
626 
627         Thread currentThread = Thread.currentThread();
628 
629         ClassLoader contextClassLoader = currentThread.getContextClassLoader();
630 
631         try {
632             currentThread.setContextClassLoader(
633                 PortalClassLoaderUtil.getClassLoader());
634 
635             String tableName = _getTableName(companyId);
636 
637             directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
638 
639             if (directory != null) {
640                 return directory;
641             }
642 
643             try {
644                 DataSource ds = InfrastructureUtil.getDataSource();
645 
646                 directory = new JdbcDirectory(ds, _dialect, tableName);
647 
648                 _jdbcDirectories.put(tableName, directory);
649 
650                 if (!directory.tableExists()) {
651                     directory.create();
652                 }
653             }
654             catch (IOException ioe) {
655                 throw new RuntimeException(ioe);
656             }
657             catch (UnsupportedOperationException uoe) {
658                 if (_log.isWarnEnabled()) {
659                     _log.warn(
660                         "Database doesn't support the ability to check " +
661                             "whether a table exists");
662                 }
663 
664                 _manuallyCreateJdbcDirectory(directory, tableName);
665             }
666         }
667         finally {
668             currentThread.setContextClassLoader(contextClassLoader);
669         }
670 
671         return directory;
672     }
673 
674     private Directory _getLuceneDirRam(long companyId) {
675         String path = _getPath(companyId);
676 
677         Directory directory = _ramDirectories.get(path);
678 
679         if (directory == null) {
680             directory = new RAMDirectory();
681 
682             _ramDirectories.put(path, directory);
683         }
684 
685         return directory;
686     }
687 
688     private String _getPath(long companyId) {
689         StringBuilder sb = new StringBuilder();
690 
691         sb.append(PropsValues.LUCENE_DIR);
692         sb.append(companyId);
693         sb.append(StringPool.SLASH);
694 
695         return sb.toString();
696     }
697 
698     private String _getTableName(long companyId) {
699         return _LUCENE_TABLE_PREFIX + companyId;
700     }
701 
702     private void _manuallyCreateJdbcDirectory(
703         JdbcDirectory directory, String tableName) {
704 
705         // LEP-2181
706 
707         Connection con = null;
708         ResultSet rs = null;
709 
710         try {
711             con = DataAccess.getConnection();
712 
713             // Check if table exists
714 
715             DatabaseMetaData metaData = con.getMetaData();
716 
717             rs = metaData.getTables(null, null, tableName, null);
718 
719             if (!rs.next()) {
720                 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
721 
722                 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
723 
724                 Class<?> lockClass = directory.getSettings().getLockClass();
725 
726                 JdbcLock jdbcLock = null;
727 
728                 try {
729                     jdbcLock = (JdbcLock)lockClass.newInstance();
730                 }
731                 catch (Exception e) {
732                     throw new JdbcStoreException(
733                         "Failed to create lock class " + lockClass);
734                 }
735 
736                 jdbcLock.initializeDatabase(directory);
737             }
738         }
739         catch (Exception e) {
740             if (_log.isWarnEnabled()) {
741                 _log.warn("Could not create " + tableName);
742             }
743         }
744         finally {
745             DataAccess.cleanUp(con, null, rs);
746         }
747     }
748 
749     private static final String _LUCENE_STORE_TYPE_FILE = "file";
750 
751     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
752 
753     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
754 
755     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
756 
757     private static Log _log = LogFactory.getLog(LuceneUtil.class);
758 
759     private static LuceneUtil _instance = new LuceneUtil();
760 
761     private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
762     private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
763     private Dialect _dialect;
764     private Map<String, Directory> _jdbcDirectories =
765         new ConcurrentHashMap<String, Directory>();
766     private Map<String, Directory> _ramDirectories =
767         new ConcurrentHashMap<String, Directory>();
768 
769 }