1
22
23 package com.liferay.portal.search.lucene;
24
25 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26 import com.liferay.portal.kernel.search.Field;
27 import com.liferay.portal.kernel.util.FileUtil;
28 import com.liferay.portal.kernel.util.InfrastructureUtil;
29 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
30 import com.liferay.portal.kernel.util.StringPool;
31 import com.liferay.portal.kernel.util.StringUtil;
32 import com.liferay.portal.kernel.util.Validator;
33 import com.liferay.portal.util.PropsKeys;
34 import com.liferay.portal.util.PropsUtil;
35 import com.liferay.portal.util.PropsValues;
36 import com.liferay.util.lucene.KeywordsUtil;
37
38 import java.io.IOException;
39
40 import java.sql.Connection;
41 import java.sql.DatabaseMetaData;
42 import java.sql.ResultSet;
43 import java.sql.Statement;
44
45 import java.util.Date;
46 import java.util.Map;
47 import java.util.concurrent.ConcurrentHashMap;
48
49 import javax.sql.DataSource;
50
51 import org.apache.commons.logging.Log;
52 import org.apache.commons.logging.LogFactory;
53 import org.apache.lucene.analysis.Analyzer;
54 import org.apache.lucene.analysis.WhitespaceAnalyzer;
55 import org.apache.lucene.document.Document;
56 import org.apache.lucene.index.IndexReader;
57 import org.apache.lucene.index.IndexWriter;
58 import org.apache.lucene.index.Term;
59 import org.apache.lucene.queryParser.ParseException;
60 import org.apache.lucene.queryParser.QueryParser;
61 import org.apache.lucene.search.BooleanClause;
62 import org.apache.lucene.search.BooleanQuery;
63 import org.apache.lucene.search.IndexSearcher;
64 import org.apache.lucene.search.Query;
65 import org.apache.lucene.search.TermQuery;
66 import org.apache.lucene.search.WildcardQuery;
67 import org.apache.lucene.store.Directory;
68 import org.apache.lucene.store.FSDirectory;
69 import org.apache.lucene.store.RAMDirectory;
70 import org.apache.lucene.store.jdbc.JdbcDirectory;
71 import org.apache.lucene.store.jdbc.JdbcStoreException;
72 import org.apache.lucene.store.jdbc.dialect.Dialect;
73 import org.apache.lucene.store.jdbc.lock.JdbcLock;
74 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
75
76
83 public class LuceneUtil {
84
85 public static void acquireLock(long companyId) {
86 try {
87 _instance._sharedWriter.acquireLock(companyId, true);
88 }
89 catch (InterruptedException ie) {
90 _log.error(ie);
91 }
92 }
93
94 public static void addDate(Document doc, String field, Date value) {
95 doc.add(LuceneFields.getDate(field, value));
96 }
97
98 public static void addExactTerm(
99 BooleanQuery booleanQuery, String field, boolean value) {
100
101 addExactTerm(booleanQuery, field, String.valueOf(value));
102 }
103
104 public static void addExactTerm(
105 BooleanQuery booleanQuery, String field, Boolean value) {
106
107 addExactTerm(booleanQuery, field, String.valueOf(value));
108 }
109
110 public static void addExactTerm(
111 BooleanQuery booleanQuery, String field, double value) {
112
113 addExactTerm(booleanQuery, field, String.valueOf(value));
114 }
115
116 public static void addExactTerm(
117 BooleanQuery booleanQuery, String field, Double value) {
118
119 addExactTerm(booleanQuery, field, String.valueOf(value));
120 }
121
122 public static void addExactTerm(
123 BooleanQuery booleanQuery, String field, int value) {
124
125 addExactTerm(booleanQuery, field, String.valueOf(value));
126 }
127
128 public static void addExactTerm(
129 BooleanQuery booleanQuery, String field, Integer value) {
130
131 addExactTerm(booleanQuery, field, String.valueOf(value));
132 }
133
134 public static void addExactTerm(
135 BooleanQuery booleanQuery, String field, long value) {
136
137 addExactTerm(booleanQuery, field, String.valueOf(value));
138 }
139
140 public static void addExactTerm(
141 BooleanQuery booleanQuery, String field, Long value) {
142
143 addExactTerm(booleanQuery, field, String.valueOf(value));
144 }
145
146 public static void addExactTerm(
147 BooleanQuery booleanQuery, String field, short value) {
148
149 addExactTerm(booleanQuery, field, String.valueOf(value));
150 }
151
152 public static void addExactTerm(
153 BooleanQuery booleanQuery, String field, Short value) {
154
155 addExactTerm(booleanQuery, field, String.valueOf(value));
156 }
157
158 public static void addExactTerm(
159 BooleanQuery booleanQuery, String field, String value) {
160
161
163 Query query = new TermQuery(new Term(field, value));
164
165 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
166 }
167
168 public static void addRequiredTerm(
169 BooleanQuery booleanQuery, String field, boolean value) {
170
171 addRequiredTerm(booleanQuery, field, String.valueOf(value));
172 }
173
174 public static void addRequiredTerm(
175 BooleanQuery booleanQuery, String field, Boolean value) {
176
177 addRequiredTerm(booleanQuery, field, String.valueOf(value));
178 }
179
180 public static void addRequiredTerm(
181 BooleanQuery booleanQuery, String field, double value) {
182
183 addRequiredTerm(booleanQuery, field, String.valueOf(value));
184 }
185
186 public static void addRequiredTerm(
187 BooleanQuery booleanQuery, String field, Double value) {
188
189 addRequiredTerm(booleanQuery, field, String.valueOf(value));
190 }
191
192 public static void addRequiredTerm(
193 BooleanQuery booleanQuery, String field, int value) {
194
195 addRequiredTerm(booleanQuery, field, String.valueOf(value));
196 }
197
198 public static void addRequiredTerm(
199 BooleanQuery booleanQuery, String field, Integer value) {
200
201 addRequiredTerm(booleanQuery, field, String.valueOf(value));
202 }
203
204 public static void addRequiredTerm(
205 BooleanQuery booleanQuery, String field, long value) {
206
207 addRequiredTerm(booleanQuery, field, String.valueOf(value));
208 }
209
210 public static void addRequiredTerm(
211 BooleanQuery booleanQuery, String field, Long value) {
212
213 addRequiredTerm(booleanQuery, field, String.valueOf(value));
214 }
215
216 public static void addRequiredTerm(
217 BooleanQuery booleanQuery, String field, short value) {
218
219 addRequiredTerm(booleanQuery, field, String.valueOf(value));
220 }
221
222 public static void addRequiredTerm(
223 BooleanQuery booleanQuery, String field, Short value) {
224
225 addRequiredTerm(booleanQuery, field, String.valueOf(value));
226 }
227
228 public static void addRequiredTerm(
229 BooleanQuery booleanQuery, String field, String value) {
230
231 addRequiredTerm(booleanQuery, field, value, false);
232 }
233
234 public static void addRequiredTerm(
235 BooleanQuery booleanQuery, String field, String value, boolean like) {
236
237 if (like) {
238 value = StringUtil.replace(
239 value, StringPool.PERCENT, StringPool.STAR);
240
241 value = value.toLowerCase();
242
243 WildcardQuery wildcardQuery = new WildcardQuery(
244 new Term(field, value));
245
246 booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
247 }
248 else {
249
251 Term term = new Term(field, value);
252 TermQuery termQuery = new TermQuery(term);
253
254 booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
255 }
256 }
257
258 public static void addModifiedDate(Document doc) {
259 doc.add(LuceneFields.getDate(Field.MODIFIED));
260 }
261
262 public static void addTerm(
263 BooleanQuery booleanQuery, String field, long value)
264 throws ParseException {
265
266 addTerm(booleanQuery, field, String.valueOf(value));
267 }
268
269 public static void addTerm(
270 BooleanQuery booleanQuery, String field, String value)
271 throws ParseException {
272
273 addTerm(booleanQuery, field, value, false);
274 }
275
276 public static void addTerm(
277 BooleanQuery booleanQuery, String field, String value,
278 boolean like)
279 throws ParseException {
280
281 if (Validator.isNull(value)) {
282 return;
283 }
284
285 if (like) {
286 value = value.toLowerCase();
287
288 StringBuilder sb = new StringBuilder();
289
290 sb.append(StringPool.STAR);
291 sb.append(value);
292 sb.append(StringPool.STAR);
293
294 WildcardQuery wildcardQuery = new WildcardQuery(
295 new Term(field, sb.toString()));
296
297 booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
298 }
299 else {
300 QueryParser queryParser = new QueryParser(
301 field, LuceneUtil.getAnalyzer());
302
303 try {
304 Query query = queryParser.parse(value);
305
306 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
307 }
308 catch (ParseException pe) {
309 if (_log.isDebugEnabled()) {
310 _log.debug(
311 "ParseException thrown, reverting to literal search",
312 pe);
313 }
314
315 value = KeywordsUtil.escape(value);
316
317 Query query = queryParser.parse(value);
318
319 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
320 }
321 }
322 }
323
324 public static void checkLuceneDir(long companyId) {
325 if (PropsValues.INDEX_READ_ONLY) {
326 return;
327 }
328
329 Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
330
331 try {
332
333
335 if (luceneDir.fileExists("write.lock")) {
336 luceneDir.deleteFile("write.lock");
337 }
338 }
339 catch (IOException ioe) {
340 _log.error("Unable to clear write lock", ioe);
341 }
342
343 IndexWriter writer = null;
344
345
348 try {
349 if (luceneDir.fileExists("segments.gen")) {
350 writer = new IndexWriter(
351 luceneDir, LuceneUtil.getAnalyzer(), false);
352 }
353 else {
354 writer = new IndexWriter(
355 luceneDir, LuceneUtil.getAnalyzer(), true);
356 }
357 }
358 catch (IOException ioe) {
359 _log.error("Check Lucene directory failed for " + companyId, ioe);
360 }
361 finally {
362 if (writer != null) {
363 try {
364 writer.close();
365 }
366 catch (IOException ioe) {
367 _log.error(ioe);
368 }
369 }
370 }
371 }
372
373 public static void delete(long companyId) {
374 _instance._delete(companyId);
375 }
376
377 public static void deleteDocuments(long companyId, Term term)
378 throws IOException {
379
380 try {
381 _instance._sharedWriter.deleteDocuments(companyId, term);
382 }
383 catch (InterruptedException ie) {
384 _log.error(ie);
385 }
386 }
387
388 public static Analyzer getAnalyzer() {
389 return _instance._getAnalyzer();
390 }
391
392 public static Directory getLuceneDir(long companyId) {
393 return _instance._getLuceneDir(companyId);
394 }
395
396 public static IndexReader getReader(long companyId) throws IOException {
397 return IndexReader.open(getLuceneDir(companyId));
398 }
399
400 public static IndexSearcher getSearcher(long companyId)
401 throws IOException {
402
403 return new IndexSearcher(getLuceneDir(companyId));
404 }
405
406 public static IndexWriter getWriter(long companyId) throws IOException {
407 return getWriter(companyId, false);
408 }
409
410 public static IndexWriter getWriter(long companyId, boolean create)
411 throws IOException {
412
413 return _instance._sharedWriter.getWriter(companyId, create);
414 }
415
416 public static void releaseLock(long companyId) {
417 _instance._sharedWriter.releaseLock(companyId);
418 }
419
420 public static void write(long companyId) {
421 _instance._sharedWriter.write(companyId);
422 }
423
424 public static void write(IndexWriter writer) throws IOException {
425 _instance._sharedWriter.write(writer);
426 }
427
428 private LuceneUtil() {
429 String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
430
431 if (Validator.isNotNull(analyzerName)) {
432 try {
433 _analyzerClass = Class.forName(analyzerName);
434 }
435 catch (Exception e) {
436 _log.error(e);
437 }
438 }
439
440
442 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
443 Connection con = null;
444
445 try {
446 con = DataAccess.getConnection();
447
448 String url = con.getMetaData().getURL();
449
450 int x = url.indexOf(":");
451 int y = url.indexOf(":", x + 1);
452
453 String urlPrefix = url.substring(x + 1, y);
454
455 String dialectClass = PropsUtil.get(
456 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
457
458 if (dialectClass != null) {
459 if (_log.isDebugEnabled()) {
460 _log.debug("JDBC class implementation " + dialectClass);
461 }
462 }
463 else {
464 if (_log.isDebugEnabled()) {
465 _log.debug("JDBC class implementation is null");
466 }
467 }
468
469 if (dialectClass != null) {
470 _dialect =
471 (Dialect)Class.forName(dialectClass).newInstance();
472 }
473 }
474 catch (Exception e) {
475 _log.error(e);
476 }
477 finally{
478 DataAccess.cleanUp(con);
479 }
480
481 if (_dialect == null) {
482 _log.error("No JDBC dialect found");
483 }
484 }
485 }
486
487 public void _delete(long companyId) {
488 if (PropsValues.INDEX_READ_ONLY) {
489 return;
490 }
491
492 if (_log.isDebugEnabled()) {
493 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
494 }
495
496 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
497 _deleteFile(companyId);
498 }
499 else if (PropsValues.LUCENE_STORE_TYPE.equals(
500 _LUCENE_STORE_TYPE_JDBC)) {
501
502 _deleteJdbc(companyId);
503 }
504 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
505 _deleteRam(companyId);
506 }
507 else {
508 throw new RuntimeException(
509 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
510 }
511 }
512
513 private void _deleteFile(long companyId) {
514 String path = _getPath(companyId);
515
516 try {
517 Directory directory = FSDirectory.getDirectory(path, false);
518
519 directory.close();
520 }
521 catch (Exception e) {
522 if (_log.isWarnEnabled()) {
523 _log.warn("Could not close directory " + path);
524 }
525 }
526
527 FileUtil.deltree(path);
528 }
529
530 private void _deleteJdbc(long companyId) {
531 String tableName = _getTableName(companyId);
532
533 try {
534 Directory directory = _jdbcDirectories.remove(tableName);
535
536 if (directory != null) {
537 directory.close();
538 }
539 }
540 catch (Exception e) {
541 if (_log.isWarnEnabled()) {
542 _log.warn("Could not close directory " + tableName);
543 }
544 }
545
546 Connection con = null;
547 Statement s = null;
548
549 try {
550 con = DataAccess.getConnection();
551
552 s = con.createStatement();
553
554 s.executeUpdate("DELETE FROM " + tableName);
555 }
556 catch (Exception e) {
557 if (_log.isWarnEnabled()) {
558 _log.warn("Could not truncate " + tableName);
559 }
560 }
561 finally {
562 DataAccess.cleanUp(con, s);
563 }
564 }
565
566 private void _deleteRam(long companyId) {
567 }
568
569 private Analyzer _getAnalyzer() {
570 try {
571 return (Analyzer)_analyzerClass.newInstance();
572 }
573 catch (Exception e) {
574 throw new RuntimeException(e);
575 }
576 }
577
578 private Directory _getLuceneDir(long companyId) {
579 if (_log.isDebugEnabled()) {
580 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
581 }
582
583 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
584 return _getLuceneDirFile(companyId);
585 }
586 else if (PropsValues.LUCENE_STORE_TYPE.equals(
587 _LUCENE_STORE_TYPE_JDBC)) {
588
589 return _getLuceneDirJdbc(companyId);
590 }
591 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
592 return _getLuceneDirRam(companyId);
593 }
594 else {
595 throw new RuntimeException(
596 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
597 }
598 }
599
600 private Directory _getLuceneDirFile(long companyId) {
601 Directory directory = null;
602
603 String path = _getPath(companyId);
604
605 try {
606 directory = FSDirectory.getDirectory(path, false);
607 }
608 catch (IOException ioe1) {
609 try {
610 if (directory != null) {
611 directory.close();
612 }
613
614 directory = FSDirectory.getDirectory(path, true);
615 }
616 catch (IOException ioe2) {
617 throw new RuntimeException(ioe2);
618 }
619 }
620
621 return directory;
622 }
623
624 private Directory _getLuceneDirJdbc(long companyId) {
625 JdbcDirectory directory = null;
626
627 Thread currentThread = Thread.currentThread();
628
629 ClassLoader contextClassLoader = currentThread.getContextClassLoader();
630
631 try {
632 currentThread.setContextClassLoader(
633 PortalClassLoaderUtil.getClassLoader());
634
635 String tableName = _getTableName(companyId);
636
637 directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
638
639 if (directory != null) {
640 return directory;
641 }
642
643 try {
644 DataSource ds = InfrastructureUtil.getDataSource();
645
646 directory = new JdbcDirectory(ds, _dialect, tableName);
647
648 _jdbcDirectories.put(tableName, directory);
649
650 if (!directory.tableExists()) {
651 directory.create();
652 }
653 }
654 catch (IOException ioe) {
655 throw new RuntimeException(ioe);
656 }
657 catch (UnsupportedOperationException uoe) {
658 if (_log.isWarnEnabled()) {
659 _log.warn(
660 "Database doesn't support the ability to check " +
661 "whether a table exists");
662 }
663
664 _manuallyCreateJdbcDirectory(directory, tableName);
665 }
666 }
667 finally {
668 currentThread.setContextClassLoader(contextClassLoader);
669 }
670
671 return directory;
672 }
673
674 private Directory _getLuceneDirRam(long companyId) {
675 String path = _getPath(companyId);
676
677 Directory directory = _ramDirectories.get(path);
678
679 if (directory == null) {
680 directory = new RAMDirectory();
681
682 _ramDirectories.put(path, directory);
683 }
684
685 return directory;
686 }
687
688 private String _getPath(long companyId) {
689 StringBuilder sb = new StringBuilder();
690
691 sb.append(PropsValues.LUCENE_DIR);
692 sb.append(companyId);
693 sb.append(StringPool.SLASH);
694
695 return sb.toString();
696 }
697
698 private String _getTableName(long companyId) {
699 return _LUCENE_TABLE_PREFIX + companyId;
700 }
701
702 private void _manuallyCreateJdbcDirectory(
703 JdbcDirectory directory, String tableName) {
704
705
707 Connection con = null;
708 ResultSet rs = null;
709
710 try {
711 con = DataAccess.getConnection();
712
713
715 DatabaseMetaData metaData = con.getMetaData();
716
717 rs = metaData.getTables(null, null, tableName, null);
718
719 if (!rs.next()) {
720 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
721
722 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
723
724 Class<?> lockClass = directory.getSettings().getLockClass();
725
726 JdbcLock jdbcLock = null;
727
728 try {
729 jdbcLock = (JdbcLock)lockClass.newInstance();
730 }
731 catch (Exception e) {
732 throw new JdbcStoreException(
733 "Failed to create lock class " + lockClass);
734 }
735
736 jdbcLock.initializeDatabase(directory);
737 }
738 }
739 catch (Exception e) {
740 if (_log.isWarnEnabled()) {
741 _log.warn("Could not create " + tableName);
742 }
743 }
744 finally {
745 DataAccess.cleanUp(con, null, rs);
746 }
747 }
748
749 private static final String _LUCENE_STORE_TYPE_FILE = "file";
750
751 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
752
753 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
754
755 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
756
757 private static Log _log = LogFactory.getLog(LuceneUtil.class);
758
759 private static LuceneUtil _instance = new LuceneUtil();
760
761 private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
762 private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
763 private Dialect _dialect;
764 private Map<String, Directory> _jdbcDirectories =
765 new ConcurrentHashMap<String, Directory>();
766 private Map<String, Directory> _ramDirectories =
767 new ConcurrentHashMap<String, Directory>();
768
769 }