1   /**
2    * Copyright (c) 2000-2009 Liferay, Inc. All rights reserved.
3    *
4    *
5    *
6    *
7    * The contents of this file are subject to the terms of the Liferay Enterprise
8    * Subscription License ("License"). You may not use this file except in
9    * compliance with the License. You can obtain a copy of the License by
10   * contacting Liferay, Inc. See the License for the specific language governing
11   * permissions and limitations under the License, including but not limited to
12   * distribution rights of the Software.
13   *
14   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  
23  package com.liferay.portal.search.lucene;
24  
25  import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26  import com.liferay.portal.kernel.log.Log;
27  import com.liferay.portal.kernel.log.LogFactoryUtil;
28  import com.liferay.portal.kernel.search.SearchEngineUtil;
29  import com.liferay.portal.kernel.util.FileUtil;
30  import com.liferay.portal.kernel.util.InfrastructureUtil;
31  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
32  import com.liferay.portal.kernel.util.StringPool;
33  import com.liferay.portal.kernel.util.StringUtil;
34  import com.liferay.portal.kernel.util.Validator;
35  import com.liferay.portal.util.PropsKeys;
36  import com.liferay.portal.util.PropsUtil;
37  import com.liferay.portal.util.PropsValues;
38  import com.liferay.util.lucene.KeywordsUtil;
39  
40  import java.io.IOException;
41  
42  import java.sql.Connection;
43  import java.sql.DatabaseMetaData;
44  import java.sql.ResultSet;
45  import java.sql.Statement;
46  
47  import java.util.Date;
48  import java.util.Map;
49  import java.util.concurrent.ConcurrentHashMap;
50  
51  import javax.sql.DataSource;
52  
53  import org.apache.lucene.analysis.Analyzer;
54  import org.apache.lucene.analysis.WhitespaceAnalyzer;
55  import org.apache.lucene.document.Document;
56  import org.apache.lucene.index.IndexReader;
57  import org.apache.lucene.index.IndexWriter;
58  import org.apache.lucene.index.Term;
59  import org.apache.lucene.queryParser.ParseException;
60  import org.apache.lucene.queryParser.QueryParser;
61  import org.apache.lucene.search.BooleanClause;
62  import org.apache.lucene.search.BooleanQuery;
63  import org.apache.lucene.search.IndexSearcher;
64  import org.apache.lucene.search.Query;
65  import org.apache.lucene.search.TermQuery;
66  import org.apache.lucene.search.WildcardQuery;
67  import org.apache.lucene.store.Directory;
68  import org.apache.lucene.store.FSDirectory;
69  import org.apache.lucene.store.RAMDirectory;
70  import org.apache.lucene.store.jdbc.JdbcDirectory;
71  import org.apache.lucene.store.jdbc.JdbcStoreException;
72  import org.apache.lucene.store.jdbc.dialect.Dialect;
73  import org.apache.lucene.store.jdbc.lock.JdbcLock;
74  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
75  
76  /**
77   * <a href="LuceneUtil.java.html"><b><i>View Source</i></b></a>
78   *
79   * @author Brian Wing Shun Chan
80   * @author Harry Mark
81   *
82   */
83  public class LuceneUtil {
84  
85      public static void acquireLock(long companyId) {
86          try {
87              _instance._sharedWriter.acquireLock(companyId, true);
88          }
89          catch (InterruptedException ie) {
90              _log.error(ie);
91          }
92      }
93  
94      public static void addDate(Document doc, String field, Date value) {
95          doc.add(LuceneFields.getDate(field, value));
96      }
97  
98      public static void addExactTerm(
99          BooleanQuery booleanQuery, String field, boolean value) {
100 
101         addExactTerm(booleanQuery, field, String.valueOf(value));
102     }
103 
104     public static void addExactTerm(
105         BooleanQuery booleanQuery, String field, Boolean value) {
106 
107         addExactTerm(booleanQuery, field, String.valueOf(value));
108     }
109 
110     public static void addExactTerm(
111         BooleanQuery booleanQuery, String field, double value) {
112 
113         addExactTerm(booleanQuery, field, String.valueOf(value));
114     }
115 
116     public static void addExactTerm(
117         BooleanQuery booleanQuery, String field, Double value) {
118 
119         addExactTerm(booleanQuery, field, String.valueOf(value));
120     }
121 
122     public static void addExactTerm(
123         BooleanQuery booleanQuery, String field, int value) {
124 
125         addExactTerm(booleanQuery, field, String.valueOf(value));
126     }
127 
128     public static void addExactTerm(
129         BooleanQuery booleanQuery, String field, Integer value) {
130 
131         addExactTerm(booleanQuery, field, String.valueOf(value));
132     }
133 
134     public static void addExactTerm(
135         BooleanQuery booleanQuery, String field, long value) {
136 
137         addExactTerm(booleanQuery, field, String.valueOf(value));
138     }
139 
140     public static void addExactTerm(
141         BooleanQuery booleanQuery, String field, Long value) {
142 
143         addExactTerm(booleanQuery, field, String.valueOf(value));
144     }
145 
146     public static void addExactTerm(
147         BooleanQuery booleanQuery, String field, short value) {
148 
149         addExactTerm(booleanQuery, field, String.valueOf(value));
150     }
151 
152     public static void addExactTerm(
153         BooleanQuery booleanQuery, String field, Short value) {
154 
155         addExactTerm(booleanQuery, field, String.valueOf(value));
156     }
157 
158     public static void addExactTerm(
159         BooleanQuery booleanQuery, String field, String value) {
160 
161         //text = KeywordsUtil.escape(value);
162 
163         Query query = new TermQuery(new Term(field, value));
164 
165         booleanQuery.add(query, BooleanClause.Occur.SHOULD);
166     }
167 
168     public static void addRequiredTerm(
169         BooleanQuery booleanQuery, String field, boolean value) {
170 
171         addRequiredTerm(booleanQuery, field, String.valueOf(value));
172     }
173 
174     public static void addRequiredTerm(
175         BooleanQuery booleanQuery, String field, Boolean value) {
176 
177         addRequiredTerm(booleanQuery, field, String.valueOf(value));
178     }
179 
180     public static void addRequiredTerm(
181         BooleanQuery booleanQuery, String field, double value) {
182 
183         addRequiredTerm(booleanQuery, field, String.valueOf(value));
184     }
185 
186     public static void addRequiredTerm(
187         BooleanQuery booleanQuery, String field, Double value) {
188 
189         addRequiredTerm(booleanQuery, field, String.valueOf(value));
190     }
191 
192     public static void addRequiredTerm(
193         BooleanQuery booleanQuery, String field, int value) {
194 
195         addRequiredTerm(booleanQuery, field, String.valueOf(value));
196     }
197 
198     public static void addRequiredTerm(
199         BooleanQuery booleanQuery, String field, Integer value) {
200 
201         addRequiredTerm(booleanQuery, field, String.valueOf(value));
202     }
203 
204     public static void addRequiredTerm(
205         BooleanQuery booleanQuery, String field, long value) {
206 
207         addRequiredTerm(booleanQuery, field, String.valueOf(value));
208     }
209 
210     public static void addRequiredTerm(
211         BooleanQuery booleanQuery, String field, Long value) {
212 
213         addRequiredTerm(booleanQuery, field, String.valueOf(value));
214     }
215 
216     public static void addRequiredTerm(
217         BooleanQuery booleanQuery, String field, short value) {
218 
219         addRequiredTerm(booleanQuery, field, String.valueOf(value));
220     }
221 
222     public static void addRequiredTerm(
223         BooleanQuery booleanQuery, String field, Short value) {
224 
225         addRequiredTerm(booleanQuery, field, String.valueOf(value));
226     }
227 
228     public static void addRequiredTerm(
229         BooleanQuery booleanQuery, String field, String value) {
230 
231         addRequiredTerm(booleanQuery, field, value, false);
232     }
233 
234     public static void addRequiredTerm(
235         BooleanQuery booleanQuery, String field, String value, boolean like) {
236 
237         if (like) {
238             value = StringUtil.replace(
239                 value, StringPool.PERCENT, StringPool.STAR);
240 
241             value = value.toLowerCase();
242 
243             WildcardQuery wildcardQuery = new WildcardQuery(
244                 new Term(field, value));
245 
246             booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
247         }
248         else {
249             //text = KeywordsUtil.escape(value);
250 
251             Term term = new Term(field, value);
252             TermQuery termQuery = new TermQuery(term);
253 
254             booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
255         }
256     }
257 
258     public static void addTerm(
259             BooleanQuery booleanQuery, String field, long value)
260         throws ParseException {
261 
262         addTerm(booleanQuery, field, String.valueOf(value));
263     }
264 
265     public static void addTerm(
266             BooleanQuery booleanQuery, String field, String value)
267         throws ParseException {
268 
269         addTerm(booleanQuery, field, value, false);
270     }
271 
272     public static void addTerm(
273             BooleanQuery booleanQuery, String field, String value,
274             boolean like)
275         throws ParseException {
276 
277         if (Validator.isNull(value)) {
278             return;
279         }
280 
281         if (like) {
282             value = value.toLowerCase();
283 
284             StringBuilder sb = new StringBuilder();
285 
286             sb.append(StringPool.STAR);
287             sb.append(value);
288             sb.append(StringPool.STAR);
289 
290             WildcardQuery wildcardQuery = new WildcardQuery(
291                 new Term(field, sb.toString()));
292 
293             booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
294         }
295         else {
296             QueryParser queryParser = new QueryParser(
297                 field, LuceneUtil.getAnalyzer());
298 
299             try {
300                 Query query = queryParser.parse(value);
301 
302                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
303             }
304             catch (ParseException pe) {
305                 if (_log.isDebugEnabled()) {
306                     _log.debug(
307                         "ParseException thrown, reverting to literal search",
308                         pe);
309                 }
310 
311                 value = KeywordsUtil.escape(value);
312 
313                 Query query = queryParser.parse(value);
314 
315                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
316             }
317         }
318     }
319 
320     public static void checkLuceneDir(long companyId) {
321         if (SearchEngineUtil.isIndexReadOnly()) {
322             return;
323         }
324 
325         Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
326 
327         try {
328 
329             // LEP-6078
330 
331             if (luceneDir.fileExists("write.lock")) {
332                 luceneDir.deleteFile("write.lock");
333             }
334         }
335         catch (IOException ioe) {
336             _log.error("Unable to clear write lock", ioe);
337         }
338 
339         IndexWriter writer = null;
340 
341         // Lucene does not properly release its lock on the index when
342         // IndexWriter throws an exception
343 
344         try {
345             if (luceneDir.fileExists("segments.gen")) {
346                 writer = new IndexWriter(
347                     luceneDir, LuceneUtil.getAnalyzer(), false);
348             }
349             else {
350                 writer = new IndexWriter(
351                     luceneDir, LuceneUtil.getAnalyzer(), true);
352             }
353         }
354         catch (IOException ioe) {
355             _log.error("Check Lucene directory failed for " + companyId, ioe);
356         }
357         finally {
358             if (writer != null) {
359                 try {
360                     writer.close();
361                 }
362                 catch (IOException ioe) {
363                     _log.error(ioe);
364                 }
365             }
366         }
367     }
368 
369     public static void delete(long companyId) {
370         _instance._delete(companyId);
371     }
372 
373     public static void deleteDocuments(long companyId, Term term)
374         throws IOException {
375 
376         try {
377             _instance._sharedWriter.deleteDocuments(companyId, term);
378         }
379         catch (InterruptedException ie) {
380             _log.error(ie);
381         }
382     }
383 
384     public static Analyzer getAnalyzer() {
385         return _instance._getAnalyzer();
386     }
387 
388     public static FSDirectory getDirectory(String path, boolean create)
389         throws IOException {
390 
391         return FSDirectory.getDirectory(path, false);
392     }
393 
394     public static Directory getLuceneDir(long companyId) {
395         return _instance._getLuceneDir(companyId);
396     }
397 
398     public static IndexReader getReader(long companyId) throws IOException {
399         return IndexReader.open(getLuceneDir(companyId));
400     }
401 
402     public static IndexSearcher getSearcher(long companyId)
403         throws IOException {
404 
405         return new IndexSearcher(getLuceneDir(companyId));
406     }
407 
408     public static IndexWriter getWriter(long companyId) throws IOException {
409         return getWriter(companyId, false);
410     }
411 
412     public static IndexWriter getWriter(long companyId, boolean create)
413         throws IOException {
414 
415         return _instance._sharedWriter.getWriter(companyId, create);
416     }
417 
418     public static void releaseLock(long companyId) {
419         _instance._sharedWriter.releaseLock(companyId);
420     }
421 
422     public static void write(long companyId) {
423         _instance._sharedWriter.write(companyId);
424     }
425 
426     public static void write(IndexWriter writer) throws IOException {
427         _instance._sharedWriter.write(writer);
428     }
429 
430     private LuceneUtil() {
431         String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
432 
433         if (Validator.isNotNull(analyzerName)) {
434             try {
435                 _analyzerClass = Class.forName(analyzerName);
436             }
437             catch (Exception e) {
438                 _log.error(e);
439             }
440         }
441 
442         // Dialect
443 
444         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
445             Connection con = null;
446 
447             try {
448                 con = DataAccess.getConnection();
449 
450                 String url = con.getMetaData().getURL();
451 
452                 int x = url.indexOf(":");
453                 int y = url.indexOf(":", x + 1);
454 
455                 String urlPrefix = url.substring(x + 1, y);
456 
457                 String dialectClass = PropsUtil.get(
458                     PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
459 
460                 if (dialectClass != null) {
461                     if (_log.isDebugEnabled()) {
462                         _log.debug("JDBC class implementation " + dialectClass);
463                     }
464                 }
465                 else {
466                     if (_log.isDebugEnabled()) {
467                         _log.debug("JDBC class implementation is null");
468                     }
469                 }
470 
471                 if (dialectClass != null) {
472                     _dialect =
473                         (Dialect)Class.forName(dialectClass).newInstance();
474                 }
475             }
476             catch (Exception e) {
477                 _log.error(e);
478             }
479             finally{
480                 DataAccess.cleanUp(con);
481             }
482 
483             if (_dialect == null) {
484                 _log.error("No JDBC dialect found");
485             }
486         }
487     }
488 
489     public void _delete(long companyId) {
490         if (SearchEngineUtil.isIndexReadOnly()) {
491             return;
492         }
493 
494         if (_log.isDebugEnabled()) {
495             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
496         }
497 
498         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
499             _deleteFile(companyId);
500         }
501         else if (PropsValues.LUCENE_STORE_TYPE.equals(
502                     _LUCENE_STORE_TYPE_JDBC)) {
503 
504             _deleteJdbc(companyId);
505         }
506         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
507             _deleteRam(companyId);
508         }
509         else {
510             throw new RuntimeException(
511                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
512         }
513     }
514 
515     private void _deleteFile(long companyId) {
516         String path = _getPath(companyId);
517 
518         try {
519             Directory directory = getDirectory(path, false);
520 
521             directory.close();
522         }
523         catch (Exception e) {
524             if (_log.isWarnEnabled()) {
525                 _log.warn("Could not close directory " + path);
526             }
527         }
528 
529         FileUtil.deltree(path);
530     }
531 
532     private void _deleteJdbc(long companyId) {
533         String tableName = _getTableName(companyId);
534 
535         try {
536             Directory directory = _jdbcDirectories.remove(tableName);
537 
538             if (directory != null) {
539                 directory.close();
540             }
541         }
542         catch (Exception e) {
543             if (_log.isWarnEnabled()) {
544                 _log.warn("Could not close directory " + tableName);
545             }
546         }
547 
548         Connection con = null;
549         Statement s = null;
550 
551         try {
552             con = DataAccess.getConnection();
553 
554             s = con.createStatement();
555 
556             s.executeUpdate("DELETE FROM " + tableName);
557         }
558         catch (Exception e) {
559             if (_log.isWarnEnabled()) {
560                 _log.warn("Could not truncate " + tableName);
561             }
562         }
563         finally {
564             DataAccess.cleanUp(con, s);
565         }
566     }
567 
568     private void _deleteRam(long companyId) {
569     }
570 
571     private Analyzer _getAnalyzer() {
572         try {
573             return (Analyzer)_analyzerClass.newInstance();
574         }
575         catch (Exception e) {
576             throw new RuntimeException(e);
577         }
578     }
579 
580     private Directory _getLuceneDir(long companyId) {
581         if (_log.isDebugEnabled()) {
582             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
583         }
584 
585         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
586             return _getLuceneDirFile(companyId);
587         }
588         else if (PropsValues.LUCENE_STORE_TYPE.equals(
589                     _LUCENE_STORE_TYPE_JDBC)) {
590 
591             return _getLuceneDirJdbc(companyId);
592         }
593         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
594             return _getLuceneDirRam(companyId);
595         }
596         else {
597             throw new RuntimeException(
598                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
599         }
600     }
601 
602     private Directory _getLuceneDirFile(long companyId) {
603         Directory directory = null;
604 
605         String path = _getPath(companyId);
606 
607         try {
608             directory = getDirectory(path, false);
609         }
610         catch (IOException ioe1) {
611             try {
612                 if (directory != null) {
613                     directory.close();
614                 }
615 
616                 directory = getDirectory(path, true);
617             }
618             catch (IOException ioe2) {
619                 throw new RuntimeException(ioe2);
620             }
621         }
622 
623         return directory;
624     }
625 
626     private Directory _getLuceneDirJdbc(long companyId) {
627         JdbcDirectory directory = null;
628 
629         Thread currentThread = Thread.currentThread();
630 
631         ClassLoader contextClassLoader = currentThread.getContextClassLoader();
632 
633         try {
634             currentThread.setContextClassLoader(
635                 PortalClassLoaderUtil.getClassLoader());
636 
637             String tableName = _getTableName(companyId);
638 
639             directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
640 
641             if (directory != null) {
642                 return directory;
643             }
644 
645             try {
646                 DataSource ds = InfrastructureUtil.getDataSource();
647 
648                 directory = new JdbcDirectory(ds, _dialect, tableName);
649 
650                 _jdbcDirectories.put(tableName, directory);
651 
652                 if (!directory.tableExists()) {
653                     directory.create();
654                 }
655             }
656             catch (IOException ioe) {
657                 throw new RuntimeException(ioe);
658             }
659             catch (UnsupportedOperationException uoe) {
660                 if (_log.isWarnEnabled()) {
661                     _log.warn(
662                         "Database doesn't support the ability to check " +
663                             "whether a table exists");
664                 }
665 
666                 _manuallyCreateJdbcDirectory(directory, tableName);
667             }
668         }
669         finally {
670             currentThread.setContextClassLoader(contextClassLoader);
671         }
672 
673         return directory;
674     }
675 
676     private Directory _getLuceneDirRam(long companyId) {
677         String path = _getPath(companyId);
678 
679         Directory directory = _ramDirectories.get(path);
680 
681         if (directory == null) {
682             directory = new RAMDirectory();
683 
684             _ramDirectories.put(path, directory);
685         }
686 
687         return directory;
688     }
689 
690     private String _getPath(long companyId) {
691         StringBuilder sb = new StringBuilder();
692 
693         sb.append(PropsValues.LUCENE_DIR);
694         sb.append(companyId);
695         sb.append(StringPool.SLASH);
696 
697         return sb.toString();
698     }
699 
700     private String _getTableName(long companyId) {
701         return _LUCENE_TABLE_PREFIX + companyId;
702     }
703 
704     private void _manuallyCreateJdbcDirectory(
705         JdbcDirectory directory, String tableName) {
706 
707         // LEP-2181
708 
709         Connection con = null;
710         ResultSet rs = null;
711 
712         try {
713             con = DataAccess.getConnection();
714 
715             // Check if table exists
716 
717             DatabaseMetaData metaData = con.getMetaData();
718 
719             rs = metaData.getTables(null, null, tableName, null);
720 
721             if (!rs.next()) {
722                 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
723 
724                 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
725 
726                 Class<?> lockClass = directory.getSettings().getLockClass();
727 
728                 JdbcLock jdbcLock = null;
729 
730                 try {
731                     jdbcLock = (JdbcLock)lockClass.newInstance();
732                 }
733                 catch (Exception e) {
734                     throw new JdbcStoreException(
735                         "Failed to create lock class " + lockClass);
736                 }
737 
738                 jdbcLock.initializeDatabase(directory);
739             }
740         }
741         catch (Exception e) {
742             if (_log.isWarnEnabled()) {
743                 _log.warn("Could not create " + tableName);
744             }
745         }
746         finally {
747             DataAccess.cleanUp(con, null, rs);
748         }
749     }
750 
751     private static final String _LUCENE_STORE_TYPE_FILE = "file";
752 
753     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
754 
755     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
756 
757     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
758 
759     private static Log _log = LogFactoryUtil.getLog(LuceneUtil.class);
760 
761     private static LuceneUtil _instance = new LuceneUtil();
762 
763     private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
764     private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
765     private Dialect _dialect;
766     private Map<String, Directory> _jdbcDirectories =
767         new ConcurrentHashMap<String, Directory>();
768     private Map<String, Directory> _ramDirectories =
769         new ConcurrentHashMap<String, Directory>();
770 
771 }