1
22
23 package com.liferay.portal.search.lucene;
24
25 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26 import com.liferay.portal.kernel.log.Log;
27 import com.liferay.portal.kernel.log.LogFactoryUtil;
28 import com.liferay.portal.kernel.search.SearchEngineUtil;
29 import com.liferay.portal.kernel.util.FileUtil;
30 import com.liferay.portal.kernel.util.InfrastructureUtil;
31 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
32 import com.liferay.portal.kernel.util.StringPool;
33 import com.liferay.portal.kernel.util.StringUtil;
34 import com.liferay.portal.kernel.util.Validator;
35 import com.liferay.portal.util.PropsKeys;
36 import com.liferay.portal.util.PropsUtil;
37 import com.liferay.portal.util.PropsValues;
38 import com.liferay.util.lucene.KeywordsUtil;
39
40 import java.io.IOException;
41
42 import java.sql.Connection;
43 import java.sql.DatabaseMetaData;
44 import java.sql.ResultSet;
45 import java.sql.Statement;
46
47 import java.util.Date;
48 import java.util.Map;
49 import java.util.concurrent.ConcurrentHashMap;
50
51 import javax.sql.DataSource;
52
53 import org.apache.lucene.analysis.Analyzer;
54 import org.apache.lucene.analysis.WhitespaceAnalyzer;
55 import org.apache.lucene.document.Document;
56 import org.apache.lucene.index.IndexReader;
57 import org.apache.lucene.index.IndexWriter;
58 import org.apache.lucene.index.Term;
59 import org.apache.lucene.queryParser.ParseException;
60 import org.apache.lucene.queryParser.QueryParser;
61 import org.apache.lucene.search.BooleanClause;
62 import org.apache.lucene.search.BooleanQuery;
63 import org.apache.lucene.search.IndexSearcher;
64 import org.apache.lucene.search.Query;
65 import org.apache.lucene.search.TermQuery;
66 import org.apache.lucene.search.WildcardQuery;
67 import org.apache.lucene.store.Directory;
68 import org.apache.lucene.store.FSDirectory;
69 import org.apache.lucene.store.RAMDirectory;
70 import org.apache.lucene.store.jdbc.JdbcDirectory;
71 import org.apache.lucene.store.jdbc.JdbcStoreException;
72 import org.apache.lucene.store.jdbc.dialect.Dialect;
73 import org.apache.lucene.store.jdbc.lock.JdbcLock;
74 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
75
76
83 public class LuceneUtil {
84
85 public static void acquireLock(long companyId) {
86 try {
87 _instance._sharedWriter.acquireLock(companyId, true);
88 }
89 catch (InterruptedException ie) {
90 _log.error(ie);
91 }
92 }
93
94 public static void addDate(Document doc, String field, Date value) {
95 doc.add(LuceneFields.getDate(field, value));
96 }
97
98 public static void addExactTerm(
99 BooleanQuery booleanQuery, String field, boolean value) {
100
101 addExactTerm(booleanQuery, field, String.valueOf(value));
102 }
103
104 public static void addExactTerm(
105 BooleanQuery booleanQuery, String field, Boolean value) {
106
107 addExactTerm(booleanQuery, field, String.valueOf(value));
108 }
109
110 public static void addExactTerm(
111 BooleanQuery booleanQuery, String field, double value) {
112
113 addExactTerm(booleanQuery, field, String.valueOf(value));
114 }
115
116 public static void addExactTerm(
117 BooleanQuery booleanQuery, String field, Double value) {
118
119 addExactTerm(booleanQuery, field, String.valueOf(value));
120 }
121
122 public static void addExactTerm(
123 BooleanQuery booleanQuery, String field, int value) {
124
125 addExactTerm(booleanQuery, field, String.valueOf(value));
126 }
127
128 public static void addExactTerm(
129 BooleanQuery booleanQuery, String field, Integer value) {
130
131 addExactTerm(booleanQuery, field, String.valueOf(value));
132 }
133
134 public static void addExactTerm(
135 BooleanQuery booleanQuery, String field, long value) {
136
137 addExactTerm(booleanQuery, field, String.valueOf(value));
138 }
139
140 public static void addExactTerm(
141 BooleanQuery booleanQuery, String field, Long value) {
142
143 addExactTerm(booleanQuery, field, String.valueOf(value));
144 }
145
146 public static void addExactTerm(
147 BooleanQuery booleanQuery, String field, short value) {
148
149 addExactTerm(booleanQuery, field, String.valueOf(value));
150 }
151
152 public static void addExactTerm(
153 BooleanQuery booleanQuery, String field, Short value) {
154
155 addExactTerm(booleanQuery, field, String.valueOf(value));
156 }
157
158 public static void addExactTerm(
159 BooleanQuery booleanQuery, String field, String value) {
160
161
163 Query query = new TermQuery(new Term(field, value));
164
165 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
166 }
167
168 public static void addRequiredTerm(
169 BooleanQuery booleanQuery, String field, boolean value) {
170
171 addRequiredTerm(booleanQuery, field, String.valueOf(value));
172 }
173
174 public static void addRequiredTerm(
175 BooleanQuery booleanQuery, String field, Boolean value) {
176
177 addRequiredTerm(booleanQuery, field, String.valueOf(value));
178 }
179
180 public static void addRequiredTerm(
181 BooleanQuery booleanQuery, String field, double value) {
182
183 addRequiredTerm(booleanQuery, field, String.valueOf(value));
184 }
185
186 public static void addRequiredTerm(
187 BooleanQuery booleanQuery, String field, Double value) {
188
189 addRequiredTerm(booleanQuery, field, String.valueOf(value));
190 }
191
192 public static void addRequiredTerm(
193 BooleanQuery booleanQuery, String field, int value) {
194
195 addRequiredTerm(booleanQuery, field, String.valueOf(value));
196 }
197
198 public static void addRequiredTerm(
199 BooleanQuery booleanQuery, String field, Integer value) {
200
201 addRequiredTerm(booleanQuery, field, String.valueOf(value));
202 }
203
204 public static void addRequiredTerm(
205 BooleanQuery booleanQuery, String field, long value) {
206
207 addRequiredTerm(booleanQuery, field, String.valueOf(value));
208 }
209
210 public static void addRequiredTerm(
211 BooleanQuery booleanQuery, String field, Long value) {
212
213 addRequiredTerm(booleanQuery, field, String.valueOf(value));
214 }
215
216 public static void addRequiredTerm(
217 BooleanQuery booleanQuery, String field, short value) {
218
219 addRequiredTerm(booleanQuery, field, String.valueOf(value));
220 }
221
222 public static void addRequiredTerm(
223 BooleanQuery booleanQuery, String field, Short value) {
224
225 addRequiredTerm(booleanQuery, field, String.valueOf(value));
226 }
227
228 public static void addRequiredTerm(
229 BooleanQuery booleanQuery, String field, String value) {
230
231 addRequiredTerm(booleanQuery, field, value, false);
232 }
233
234 public static void addRequiredTerm(
235 BooleanQuery booleanQuery, String field, String value, boolean like) {
236
237 if (like) {
238 value = StringUtil.replace(
239 value, StringPool.PERCENT, StringPool.STAR);
240
241 value = value.toLowerCase();
242
243 WildcardQuery wildcardQuery = new WildcardQuery(
244 new Term(field, value));
245
246 booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
247 }
248 else {
249
251 Term term = new Term(field, value);
252 TermQuery termQuery = new TermQuery(term);
253
254 booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
255 }
256 }
257
258 public static void addTerm(
259 BooleanQuery booleanQuery, String field, long value)
260 throws ParseException {
261
262 addTerm(booleanQuery, field, String.valueOf(value));
263 }
264
265 public static void addTerm(
266 BooleanQuery booleanQuery, String field, String value)
267 throws ParseException {
268
269 addTerm(booleanQuery, field, value, false);
270 }
271
272 public static void addTerm(
273 BooleanQuery booleanQuery, String field, String value,
274 boolean like)
275 throws ParseException {
276
277 if (Validator.isNull(value)) {
278 return;
279 }
280
281 if (like) {
282 value = value.toLowerCase();
283
284 StringBuilder sb = new StringBuilder();
285
286 sb.append(StringPool.STAR);
287 sb.append(value);
288 sb.append(StringPool.STAR);
289
290 WildcardQuery wildcardQuery = new WildcardQuery(
291 new Term(field, sb.toString()));
292
293 booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
294 }
295 else {
296 QueryParser queryParser = new QueryParser(
297 field, LuceneUtil.getAnalyzer());
298
299 try {
300 Query query = queryParser.parse(value);
301
302 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
303 }
304 catch (ParseException pe) {
305 if (_log.isDebugEnabled()) {
306 _log.debug(
307 "ParseException thrown, reverting to literal search",
308 pe);
309 }
310
311 value = KeywordsUtil.escape(value);
312
313 Query query = queryParser.parse(value);
314
315 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
316 }
317 }
318 }
319
320 public static void checkLuceneDir(long companyId) {
321 if (SearchEngineUtil.isIndexReadOnly()) {
322 return;
323 }
324
325 Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
326
327 try {
328
329
331 if (luceneDir.fileExists("write.lock")) {
332 luceneDir.deleteFile("write.lock");
333 }
334 }
335 catch (IOException ioe) {
336 _log.error("Unable to clear write lock", ioe);
337 }
338
339 IndexWriter writer = null;
340
341
344 try {
345 if (luceneDir.fileExists("segments.gen")) {
346 writer = new IndexWriter(
347 luceneDir, LuceneUtil.getAnalyzer(), false);
348 }
349 else {
350 writer = new IndexWriter(
351 luceneDir, LuceneUtil.getAnalyzer(), true);
352 }
353 }
354 catch (IOException ioe) {
355 _log.error("Check Lucene directory failed for " + companyId, ioe);
356 }
357 finally {
358 if (writer != null) {
359 try {
360 writer.close();
361 }
362 catch (IOException ioe) {
363 _log.error(ioe);
364 }
365 }
366 }
367 }
368
369 public static void delete(long companyId) {
370 _instance._delete(companyId);
371 }
372
373 public static void deleteDocuments(long companyId, Term term)
374 throws IOException {
375
376 try {
377 _instance._sharedWriter.deleteDocuments(companyId, term);
378 }
379 catch (InterruptedException ie) {
380 _log.error(ie);
381 }
382 }
383
384 public static Analyzer getAnalyzer() {
385 return _instance._getAnalyzer();
386 }
387
388 public static FSDirectory getDirectory(String path, boolean create)
389 throws IOException {
390
391 return FSDirectory.getDirectory(path, false);
392 }
393
394 public static Directory getLuceneDir(long companyId) {
395 return _instance._getLuceneDir(companyId);
396 }
397
398 public static IndexReader getReader(long companyId) throws IOException {
399 return IndexReader.open(getLuceneDir(companyId));
400 }
401
402 public static IndexSearcher getSearcher(long companyId)
403 throws IOException {
404
405 return new IndexSearcher(getLuceneDir(companyId));
406 }
407
408 public static IndexWriter getWriter(long companyId) throws IOException {
409 return getWriter(companyId, false);
410 }
411
412 public static IndexWriter getWriter(long companyId, boolean create)
413 throws IOException {
414
415 return _instance._sharedWriter.getWriter(companyId, create);
416 }
417
418 public static void releaseLock(long companyId) {
419 _instance._sharedWriter.releaseLock(companyId);
420 }
421
422 public static void write(long companyId) {
423 _instance._sharedWriter.write(companyId);
424 }
425
426 public static void write(IndexWriter writer) throws IOException {
427 _instance._sharedWriter.write(writer);
428 }
429
430 private LuceneUtil() {
431 String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
432
433 if (Validator.isNotNull(analyzerName)) {
434 try {
435 _analyzerClass = Class.forName(analyzerName);
436 }
437 catch (Exception e) {
438 _log.error(e);
439 }
440 }
441
442
444 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
445 Connection con = null;
446
447 try {
448 con = DataAccess.getConnection();
449
450 String url = con.getMetaData().getURL();
451
452 int x = url.indexOf(":");
453 int y = url.indexOf(":", x + 1);
454
455 String urlPrefix = url.substring(x + 1, y);
456
457 String dialectClass = PropsUtil.get(
458 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
459
460 if (dialectClass != null) {
461 if (_log.isDebugEnabled()) {
462 _log.debug("JDBC class implementation " + dialectClass);
463 }
464 }
465 else {
466 if (_log.isDebugEnabled()) {
467 _log.debug("JDBC class implementation is null");
468 }
469 }
470
471 if (dialectClass != null) {
472 _dialect =
473 (Dialect)Class.forName(dialectClass).newInstance();
474 }
475 }
476 catch (Exception e) {
477 _log.error(e);
478 }
479 finally{
480 DataAccess.cleanUp(con);
481 }
482
483 if (_dialect == null) {
484 _log.error("No JDBC dialect found");
485 }
486 }
487 }
488
489 public void _delete(long companyId) {
490 if (SearchEngineUtil.isIndexReadOnly()) {
491 return;
492 }
493
494 if (_log.isDebugEnabled()) {
495 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
496 }
497
498 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
499 _deleteFile(companyId);
500 }
501 else if (PropsValues.LUCENE_STORE_TYPE.equals(
502 _LUCENE_STORE_TYPE_JDBC)) {
503
504 _deleteJdbc(companyId);
505 }
506 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
507 _deleteRam(companyId);
508 }
509 else {
510 throw new RuntimeException(
511 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
512 }
513 }
514
515 private void _deleteFile(long companyId) {
516 String path = _getPath(companyId);
517
518 try {
519 Directory directory = getDirectory(path, false);
520
521 directory.close();
522 }
523 catch (Exception e) {
524 if (_log.isWarnEnabled()) {
525 _log.warn("Could not close directory " + path);
526 }
527 }
528
529 FileUtil.deltree(path);
530 }
531
532 private void _deleteJdbc(long companyId) {
533 String tableName = _getTableName(companyId);
534
535 try {
536 Directory directory = _jdbcDirectories.remove(tableName);
537
538 if (directory != null) {
539 directory.close();
540 }
541 }
542 catch (Exception e) {
543 if (_log.isWarnEnabled()) {
544 _log.warn("Could not close directory " + tableName);
545 }
546 }
547
548 Connection con = null;
549 Statement s = null;
550
551 try {
552 con = DataAccess.getConnection();
553
554 s = con.createStatement();
555
556 s.executeUpdate("DELETE FROM " + tableName);
557 }
558 catch (Exception e) {
559 if (_log.isWarnEnabled()) {
560 _log.warn("Could not truncate " + tableName);
561 }
562 }
563 finally {
564 DataAccess.cleanUp(con, s);
565 }
566 }
567
568 private void _deleteRam(long companyId) {
569 }
570
571 private Analyzer _getAnalyzer() {
572 try {
573 return (Analyzer)_analyzerClass.newInstance();
574 }
575 catch (Exception e) {
576 throw new RuntimeException(e);
577 }
578 }
579
580 private Directory _getLuceneDir(long companyId) {
581 if (_log.isDebugEnabled()) {
582 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
583 }
584
585 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
586 return _getLuceneDirFile(companyId);
587 }
588 else if (PropsValues.LUCENE_STORE_TYPE.equals(
589 _LUCENE_STORE_TYPE_JDBC)) {
590
591 return _getLuceneDirJdbc(companyId);
592 }
593 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
594 return _getLuceneDirRam(companyId);
595 }
596 else {
597 throw new RuntimeException(
598 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
599 }
600 }
601
602 private Directory _getLuceneDirFile(long companyId) {
603 Directory directory = null;
604
605 String path = _getPath(companyId);
606
607 try {
608 directory = getDirectory(path, false);
609 }
610 catch (IOException ioe1) {
611 try {
612 if (directory != null) {
613 directory.close();
614 }
615
616 directory = getDirectory(path, true);
617 }
618 catch (IOException ioe2) {
619 throw new RuntimeException(ioe2);
620 }
621 }
622
623 return directory;
624 }
625
626 private Directory _getLuceneDirJdbc(long companyId) {
627 JdbcDirectory directory = null;
628
629 Thread currentThread = Thread.currentThread();
630
631 ClassLoader contextClassLoader = currentThread.getContextClassLoader();
632
633 try {
634 currentThread.setContextClassLoader(
635 PortalClassLoaderUtil.getClassLoader());
636
637 String tableName = _getTableName(companyId);
638
639 directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
640
641 if (directory != null) {
642 return directory;
643 }
644
645 try {
646 DataSource ds = InfrastructureUtil.getDataSource();
647
648 directory = new JdbcDirectory(ds, _dialect, tableName);
649
650 _jdbcDirectories.put(tableName, directory);
651
652 if (!directory.tableExists()) {
653 directory.create();
654 }
655 }
656 catch (IOException ioe) {
657 throw new RuntimeException(ioe);
658 }
659 catch (UnsupportedOperationException uoe) {
660 if (_log.isWarnEnabled()) {
661 _log.warn(
662 "Database doesn't support the ability to check " +
663 "whether a table exists");
664 }
665
666 _manuallyCreateJdbcDirectory(directory, tableName);
667 }
668 }
669 finally {
670 currentThread.setContextClassLoader(contextClassLoader);
671 }
672
673 return directory;
674 }
675
676 private Directory _getLuceneDirRam(long companyId) {
677 String path = _getPath(companyId);
678
679 Directory directory = _ramDirectories.get(path);
680
681 if (directory == null) {
682 directory = new RAMDirectory();
683
684 _ramDirectories.put(path, directory);
685 }
686
687 return directory;
688 }
689
690 private String _getPath(long companyId) {
691 StringBuilder sb = new StringBuilder();
692
693 sb.append(PropsValues.LUCENE_DIR);
694 sb.append(companyId);
695 sb.append(StringPool.SLASH);
696
697 return sb.toString();
698 }
699
700 private String _getTableName(long companyId) {
701 return _LUCENE_TABLE_PREFIX + companyId;
702 }
703
704 private void _manuallyCreateJdbcDirectory(
705 JdbcDirectory directory, String tableName) {
706
707
709 Connection con = null;
710 ResultSet rs = null;
711
712 try {
713 con = DataAccess.getConnection();
714
715
717 DatabaseMetaData metaData = con.getMetaData();
718
719 rs = metaData.getTables(null, null, tableName, null);
720
721 if (!rs.next()) {
722 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
723
724 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
725
726 Class<?> lockClass = directory.getSettings().getLockClass();
727
728 JdbcLock jdbcLock = null;
729
730 try {
731 jdbcLock = (JdbcLock)lockClass.newInstance();
732 }
733 catch (Exception e) {
734 throw new JdbcStoreException(
735 "Failed to create lock class " + lockClass);
736 }
737
738 jdbcLock.initializeDatabase(directory);
739 }
740 }
741 catch (Exception e) {
742 if (_log.isWarnEnabled()) {
743 _log.warn("Could not create " + tableName);
744 }
745 }
746 finally {
747 DataAccess.cleanUp(con, null, rs);
748 }
749 }
750
751 private static final String _LUCENE_STORE_TYPE_FILE = "file";
752
753 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
754
755 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
756
757 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
758
759 private static Log _log = LogFactoryUtil.getLog(LuceneUtil.class);
760
761 private static LuceneUtil _instance = new LuceneUtil();
762
763 private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
764 private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
765 private Dialect _dialect;
766 private Map<String, Directory> _jdbcDirectories =
767 new ConcurrentHashMap<String, Directory>();
768 private Map<String, Directory> _ramDirectories =
769 new ConcurrentHashMap<String, Directory>();
770
771 }