001
014
015 package com.liferay.portal.search.lucene;
016
017 import com.liferay.portal.kernel.log.Log;
018 import com.liferay.portal.kernel.log.LogFactoryUtil;
019 import com.liferay.portal.kernel.search.SearchEngineUtil;
020 import com.liferay.portal.kernel.util.FileUtil;
021 import com.liferay.portal.kernel.util.InstanceFactory;
022 import com.liferay.portal.kernel.util.StringPool;
023 import com.liferay.portal.search.lucene.dump.DumpIndexDeletionPolicy;
024 import com.liferay.portal.search.lucene.dump.IndexCommitSerializationUtil;
025 import com.liferay.portal.util.ClassLoaderUtil;
026 import com.liferay.portal.util.PropsValues;
027
028 import java.io.File;
029 import java.io.IOException;
030 import java.io.InputStream;
031 import java.io.OutputStream;
032
033 import java.util.Map;
034 import java.util.concurrent.ConcurrentHashMap;
035 import java.util.concurrent.Executors;
036 import java.util.concurrent.ScheduledExecutorService;
037 import java.util.concurrent.TimeUnit;
038 import java.util.concurrent.locks.Lock;
039 import java.util.concurrent.locks.ReentrantLock;
040
041 import org.apache.lucene.analysis.Analyzer;
042 import org.apache.lucene.analysis.LimitTokenCountAnalyzer;
043 import org.apache.lucene.document.Document;
044 import org.apache.lucene.index.IndexReader;
045 import org.apache.lucene.index.IndexWriter;
046 import org.apache.lucene.index.IndexWriterConfig;
047 import org.apache.lucene.index.LogMergePolicy;
048 import org.apache.lucene.index.MergePolicy;
049 import org.apache.lucene.index.Term;
050 import org.apache.lucene.search.IndexSearcher;
051 import org.apache.lucene.search.MatchAllDocsQuery;
052 import org.apache.lucene.search.ScoreDoc;
053 import org.apache.lucene.search.TopDocs;
054 import org.apache.lucene.store.Directory;
055 import org.apache.lucene.store.FSDirectory;
056 import org.apache.lucene.store.MMapDirectory;
057 import org.apache.lucene.store.RAMDirectory;
058
059
066 public class IndexAccessorImpl implements IndexAccessor {
067
068 public IndexAccessorImpl(long companyId) {
069 _companyId = companyId;
070
071 _checkLuceneDir();
072 _initIndexWriter();
073 _initCommitScheduler();
074 }
075
076 public void addDocument(Document document) throws IOException {
077 if (SearchEngineUtil.isIndexReadOnly()) {
078 return;
079 }
080
081 _write(null, document);
082 }
083
084 public void close() {
085 try {
086 _indexWriter.close();
087 }
088 catch (Exception e) {
089 _log.error("Closing Lucene writer failed for " + _companyId, e);
090 }
091 }
092
093 public void delete() {
094 if (SearchEngineUtil.isIndexReadOnly()) {
095 return;
096 }
097
098 _deleteDirectory();
099 }
100
101 public void deleteDocuments(Term term) throws IOException {
102 if (SearchEngineUtil.isIndexReadOnly()) {
103 return;
104 }
105
106 try {
107 _indexWriter.deleteDocuments(term);
108
109 _batchCount++;
110 }
111 finally {
112 _commit();
113 }
114 }
115
116 public void dumpIndex(OutputStream outputStream) throws IOException {
117 _dumpIndexDeletionPolicy.dump(outputStream, _indexWriter, _commitLock);
118 }
119
120 public long getCompanyId() {
121 return _companyId;
122 }
123
124 public long getLastGeneration() {
125 return _dumpIndexDeletionPolicy.getLastGeneration();
126 }
127
128 public Directory getLuceneDir() {
129 if (_log.isDebugEnabled()) {
130 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
131 }
132
133 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
134 return _getLuceneDirFile();
135 }
136 else if (PropsValues.LUCENE_STORE_TYPE.equals(
137 _LUCENE_STORE_TYPE_JDBC)) {
138
139 throw new IllegalArgumentException(
140 "Store type JDBC is no longer supported in favor of SOLR");
141 }
142 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
143 return _getLuceneDirRam();
144 }
145 else {
146 throw new RuntimeException(
147 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
148 }
149 }
150
151 public void loadIndex(InputStream inputStream) throws IOException {
152 File tempFile = FileUtil.createTempFile();
153
154 Directory tempDirectory = FSDirectory.open(tempFile);
155
156 IndexCommitSerializationUtil.deserializeIndex(
157 inputStream, tempDirectory);
158
159 _deleteDirectory();
160
161 IndexReader indexReader = IndexReader.open(tempDirectory, false);
162
163 IndexSearcher indexSearcher = new IndexSearcher(indexReader);
164
165 try {
166 TopDocs topDocs = indexSearcher.search(
167 new MatchAllDocsQuery(), indexReader.numDocs());
168
169 ScoreDoc[] scoreDocs = topDocs.scoreDocs;
170
171 for (ScoreDoc scoreDoc : scoreDocs) {
172 Document document = indexSearcher.doc(scoreDoc.doc);
173
174 addDocument(document);
175 }
176 }
177 catch (IllegalArgumentException iae) {
178 if (_log.isDebugEnabled()) {
179 _log.debug(iae.getMessage());
180 }
181 }
182
183 indexSearcher.close();
184
185 indexReader.flush();
186 indexReader.close();
187
188 tempDirectory.close();
189
190 FileUtil.deltree(tempFile);
191 }
192
193 public void updateDocument(Term term, Document document)
194 throws IOException {
195
196 if (SearchEngineUtil.isIndexReadOnly()) {
197 return;
198 }
199
200 if (_log.isDebugEnabled()) {
201 _log.debug("Indexing " + document);
202 }
203
204 _write(term, document);
205 }
206
207 private void _checkLuceneDir() {
208 if (SearchEngineUtil.isIndexReadOnly()) {
209 return;
210 }
211
212 try {
213 Directory directory = getLuceneDir();
214
215 if (IndexWriter.isLocked(directory)) {
216 IndexWriter.unlock(directory);
217 }
218 }
219 catch (Exception e) {
220 _log.error("Check Lucene directory failed for " + _companyId, e);
221 }
222 }
223
224 private void _commit() throws IOException {
225 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
226 (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
227
228 _doCommit();
229 }
230 }
231
232 private void _deleteDirectory() {
233 if (_log.isDebugEnabled()) {
234 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
235 }
236
237 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
238 _deleteFile();
239 }
240 else if (PropsValues.LUCENE_STORE_TYPE.equals(
241 _LUCENE_STORE_TYPE_JDBC)) {
242
243 throw new IllegalArgumentException(
244 "Store type JDBC is no longer supported in favor of SOLR");
245 }
246 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
247 _deleteRam();
248 }
249 else {
250 throw new RuntimeException(
251 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
252 }
253 }
254
255 private void _deleteFile() {
256 String path = _getPath();
257
258 try {
259 _indexWriter.deleteAll();
260
261
262
263 _indexWriter.commit();
264 }
265 catch (Exception e) {
266 if (_log.isWarnEnabled()) {
267 _log.warn("Could not delete index in directory " + path);
268 }
269 }
270 }
271
272 private void _deleteRam() {
273 }
274
275 private void _doCommit() throws IOException {
276 if (_indexWriter != null) {
277 _commitLock.lock();
278
279 try {
280 _indexWriter.commit();
281 }
282 finally {
283 _commitLock.unlock();
284 }
285 }
286
287 _batchCount = 0;
288 }
289
290 private FSDirectory _getDirectory(String path) throws IOException {
291 if (PropsValues.LUCENE_STORE_TYPE_FILE_FORCE_MMAP) {
292 return new MMapDirectory(new File(path));
293 }
294 else {
295 return FSDirectory.open(new File(path));
296 }
297 }
298
299 private Directory _getLuceneDirFile() {
300 Directory directory = null;
301
302 String path = _getPath();
303
304 try {
305 directory = _getDirectory(path);
306 }
307 catch (IOException ioe) {
308 if (directory != null) {
309 try {
310 directory.close();
311 }
312 catch (Exception e) {
313 }
314 }
315 }
316
317 return directory;
318 }
319
320 private Directory _getLuceneDirRam() {
321 String path = _getPath();
322
323 Directory directory = _ramDirectories.get(path);
324
325 if (directory == null) {
326 directory = new RAMDirectory();
327
328 _ramDirectories.put(path, directory);
329 }
330
331 return directory;
332 }
333
334 private MergePolicy _getMergePolicy() throws Exception {
335 ClassLoader classLoader = ClassLoaderUtil.getPortalClassLoader();
336
337 MergePolicy mergePolicy = (MergePolicy)InstanceFactory.newInstance(
338 classLoader, PropsValues.LUCENE_MERGE_POLICY);
339
340 if (mergePolicy instanceof LogMergePolicy) {
341 LogMergePolicy logMergePolicy = (LogMergePolicy)mergePolicy;
342
343 logMergePolicy.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
344 }
345
346 return mergePolicy;
347 }
348
349 private String _getPath() {
350 return PropsValues.LUCENE_DIR.concat(String.valueOf(_companyId)).concat(
351 StringPool.SLASH);
352 }
353
354 private void _initCommitScheduler() {
355 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
356 (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
357
358 return;
359 }
360
361 ScheduledExecutorService scheduledExecutorService =
362 Executors.newSingleThreadScheduledExecutor();
363
364 Runnable runnable = new Runnable() {
365
366 public void run() {
367 try {
368 if (_batchCount > 0) {
369 _doCommit();
370 }
371 }
372 catch (IOException ioe) {
373 _log.error("Could not run scheduled commit", ioe);
374 }
375 }
376
377 };
378
379 scheduledExecutorService.scheduleWithFixedDelay(
380 runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
381 TimeUnit.MILLISECONDS);
382 }
383
384 private void _initIndexWriter() {
385 try {
386 Analyzer analyzer = new LimitTokenCountAnalyzer(
387 LuceneHelperUtil.getAnalyzer(),
388 PropsValues.LUCENE_ANALYZER_MAX_TOKENS);
389
390 IndexWriterConfig indexWriterConfig = new IndexWriterConfig(
391 LuceneHelperUtil.getVersion(), analyzer);
392
393 indexWriterConfig.setIndexDeletionPolicy(_dumpIndexDeletionPolicy);
394 indexWriterConfig.setMergePolicy(_getMergePolicy());
395 indexWriterConfig.setRAMBufferSizeMB(
396 PropsValues.LUCENE_BUFFER_SIZE);
397
398 _indexWriter = new IndexWriter(getLuceneDir(), indexWriterConfig);
399
400 if (!IndexReader.indexExists(getLuceneDir())) {
401
402
403
404 if (_log.isDebugEnabled()) {
405 _log.debug("Creating missing index");
406 }
407
408 _doCommit();
409 }
410 }
411 catch (Exception e) {
412 _log.error(
413 "Initializing Lucene writer failed for " + _companyId, e);
414 }
415 }
416
417 private void _write(Term term, Document document) throws IOException {
418 try {
419 if (term != null) {
420 _indexWriter.updateDocument(term, document);
421 }
422 else {
423 _indexWriter.addDocument(document);
424 }
425
426 _batchCount++;
427 }
428 finally {
429 _commit();
430 }
431 }
432
433 private static final String _LUCENE_STORE_TYPE_FILE = "file";
434
435 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
436
437 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
438
439 private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
440
441 private volatile int _batchCount;
442 private Lock _commitLock = new ReentrantLock();
443 private long _companyId;
444 private DumpIndexDeletionPolicy _dumpIndexDeletionPolicy =
445 new DumpIndexDeletionPolicy();
446 private IndexWriter _indexWriter;
447 private Map<String, Directory> _ramDirectories =
448 new ConcurrentHashMap<String, Directory>();
449
450 }