001    /**
002     * Copyright (c) 2000-2013 Liferay, Inc. All rights reserved.
003     *
004     * This library is free software; you can redistribute it and/or modify it under
005     * the terms of the GNU Lesser General Public License as published by the Free
006     * Software Foundation; either version 2.1 of the License, or (at your option)
007     * any later version.
008     *
009     * This library is distributed in the hope that it will be useful, but WITHOUT
010     * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
011     * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
012     * details.
013     */
014    
015    package com.liferay.portal.search.lucene;
016    
017    import com.liferay.portal.kernel.log.Log;
018    import com.liferay.portal.kernel.log.LogFactoryUtil;
019    import com.liferay.portal.kernel.search.SearchEngineUtil;
020    import com.liferay.portal.kernel.util.FileUtil;
021    import com.liferay.portal.kernel.util.InstanceFactory;
022    import com.liferay.portal.kernel.util.StringPool;
023    import com.liferay.portal.search.lucene.dump.DumpIndexDeletionPolicy;
024    import com.liferay.portal.search.lucene.dump.IndexCommitSerializationUtil;
025    import com.liferay.portal.util.ClassLoaderUtil;
026    import com.liferay.portal.util.PropsValues;
027    
028    import java.io.File;
029    import java.io.IOException;
030    import java.io.InputStream;
031    import java.io.OutputStream;
032    
033    import java.util.Map;
034    import java.util.concurrent.ConcurrentHashMap;
035    import java.util.concurrent.Executors;
036    import java.util.concurrent.ScheduledExecutorService;
037    import java.util.concurrent.TimeUnit;
038    import java.util.concurrent.locks.Lock;
039    import java.util.concurrent.locks.ReentrantLock;
040    
041    import org.apache.lucene.analysis.Analyzer;
042    import org.apache.lucene.analysis.LimitTokenCountAnalyzer;
043    import org.apache.lucene.document.Document;
044    import org.apache.lucene.index.IndexReader;
045    import org.apache.lucene.index.IndexWriter;
046    import org.apache.lucene.index.IndexWriterConfig;
047    import org.apache.lucene.index.LogMergePolicy;
048    import org.apache.lucene.index.MergePolicy;
049    import org.apache.lucene.index.Term;
050    import org.apache.lucene.search.IndexSearcher;
051    import org.apache.lucene.search.MatchAllDocsQuery;
052    import org.apache.lucene.search.ScoreDoc;
053    import org.apache.lucene.search.TopDocs;
054    import org.apache.lucene.store.Directory;
055    import org.apache.lucene.store.FSDirectory;
056    import org.apache.lucene.store.MMapDirectory;
057    import org.apache.lucene.store.RAMDirectory;
058    
059    /**
060     * @author Harry Mark
061     * @author Brian Wing Shun Chan
062     * @author Bruno Farache
063     * @author Shuyang Zhou
064     * @author Mate Thurzo
065     */
066    public class IndexAccessorImpl implements IndexAccessor {
067    
068            public IndexAccessorImpl(long companyId) {
069                    _companyId = companyId;
070    
071                    _checkLuceneDir();
072                    _initIndexWriter();
073                    _initCommitScheduler();
074            }
075    
076            public void addDocument(Document document) throws IOException {
077                    if (SearchEngineUtil.isIndexReadOnly()) {
078                            return;
079                    }
080    
081                    _write(null, document);
082            }
083    
084            public void close() {
085                    try {
086                            _indexWriter.close();
087                    }
088                    catch (Exception e) {
089                            _log.error("Closing Lucene writer failed for " + _companyId, e);
090                    }
091            }
092    
093            public void delete() {
094                    if (SearchEngineUtil.isIndexReadOnly()) {
095                            return;
096                    }
097    
098                    _deleteDirectory();
099            }
100    
101            public void deleteDocuments(Term term) throws IOException {
102                    if (SearchEngineUtil.isIndexReadOnly()) {
103                            return;
104                    }
105    
106                    try {
107                            _indexWriter.deleteDocuments(term);
108    
109                            _batchCount++;
110                    }
111                    finally {
112                            _commit();
113                    }
114            }
115    
116            public void dumpIndex(OutputStream outputStream) throws IOException {
117                    _dumpIndexDeletionPolicy.dump(outputStream, _indexWriter, _commitLock);
118            }
119    
120            public long getCompanyId() {
121                    return _companyId;
122            }
123    
124            public long getLastGeneration() {
125                    return _dumpIndexDeletionPolicy.getLastGeneration();
126            }
127    
128            public Directory getLuceneDir() {
129                    if (_log.isDebugEnabled()) {
130                            _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
131                    }
132    
133                    if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
134                            return _getLuceneDirFile();
135                    }
136                    else if (PropsValues.LUCENE_STORE_TYPE.equals(
137                                            _LUCENE_STORE_TYPE_JDBC)) {
138    
139                            throw new IllegalArgumentException(
140                                    "Store type JDBC is no longer supported in favor of SOLR");
141                    }
142                    else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
143                            return _getLuceneDirRam();
144                    }
145                    else {
146                            throw new RuntimeException(
147                                    "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
148                    }
149            }
150    
151            public void loadIndex(InputStream inputStream) throws IOException {
152                    File tempFile = FileUtil.createTempFile();
153    
154                    Directory tempDirectory = FSDirectory.open(tempFile);
155    
156                    IndexCommitSerializationUtil.deserializeIndex(
157                            inputStream, tempDirectory);
158    
159                    _deleteDirectory();
160    
161                    IndexReader indexReader = IndexReader.open(tempDirectory, false);
162    
163                    IndexSearcher indexSearcher = new IndexSearcher(indexReader);
164    
165                    try {
166                            TopDocs topDocs = indexSearcher.search(
167                                    new MatchAllDocsQuery(), indexReader.numDocs());
168    
169                            ScoreDoc[] scoreDocs = topDocs.scoreDocs;
170    
171                            for (ScoreDoc scoreDoc : scoreDocs) {
172                                    Document document = indexSearcher.doc(scoreDoc.doc);
173    
174                                    addDocument(document);
175                            }
176                    }
177                    catch (IllegalArgumentException iae) {
178                            if (_log.isDebugEnabled()) {
179                                    _log.debug(iae.getMessage());
180                            }
181                    }
182    
183                    indexSearcher.close();
184    
185                    indexReader.flush();
186                    indexReader.close();
187    
188                    tempDirectory.close();
189    
190                    FileUtil.deltree(tempFile);
191            }
192    
193            public void updateDocument(Term term, Document document)
194                    throws IOException {
195    
196                    if (SearchEngineUtil.isIndexReadOnly()) {
197                            return;
198                    }
199    
200                    if (_log.isDebugEnabled()) {
201                            _log.debug("Indexing " + document);
202                    }
203    
204                    _write(term, document);
205            }
206    
207            private void _checkLuceneDir() {
208                    if (SearchEngineUtil.isIndexReadOnly()) {
209                            return;
210                    }
211    
212                    try {
213                            Directory directory = getLuceneDir();
214    
215                            if (IndexWriter.isLocked(directory)) {
216                                    IndexWriter.unlock(directory);
217                            }
218                    }
219                    catch (Exception e) {
220                            _log.error("Check Lucene directory failed for " + _companyId, e);
221                    }
222            }
223    
224            private void _commit() throws IOException {
225                    if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
226                            (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
227    
228                            _doCommit();
229                    }
230            }
231    
232            private void _deleteDirectory() {
233                    if (_log.isDebugEnabled()) {
234                            _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
235                    }
236    
237                    if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
238                            _deleteFile();
239                    }
240                    else if (PropsValues.LUCENE_STORE_TYPE.equals(
241                                            _LUCENE_STORE_TYPE_JDBC)) {
242    
243                            throw new IllegalArgumentException(
244                                    "Store type JDBC is no longer supported in favor of SOLR");
245                    }
246                    else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
247                            _deleteRam();
248                    }
249                    else {
250                            throw new RuntimeException(
251                                    "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
252                    }
253            }
254    
255            private void _deleteFile() {
256                    String path = _getPath();
257    
258                    try {
259                            _indexWriter.deleteAll();
260    
261                            // Ensuring that all the changes has been applied to the index
262    
263                            _indexWriter.commit();
264                    }
265                    catch (Exception e) {
266                            if (_log.isWarnEnabled()) {
267                                    _log.warn("Could not delete index in directory " + path);
268                            }
269                    }
270            }
271    
272            private void _deleteRam() {
273            }
274    
275            private void _doCommit() throws IOException {
276                    if (_indexWriter != null) {
277                            _commitLock.lock();
278    
279                            try {
280                                    _indexWriter.commit();
281                            }
282                            finally {
283                                    _commitLock.unlock();
284                            }
285                    }
286    
287                    _batchCount = 0;
288            }
289    
290            private FSDirectory _getDirectory(String path) throws IOException {
291                    if (PropsValues.LUCENE_STORE_TYPE_FILE_FORCE_MMAP) {
292                            return new MMapDirectory(new File(path));
293                    }
294                    else {
295                            return FSDirectory.open(new File(path));
296                    }
297            }
298    
299            private Directory _getLuceneDirFile() {
300                    Directory directory = null;
301    
302                    String path = _getPath();
303    
304                    try {
305                            directory = _getDirectory(path);
306                    }
307                    catch (IOException ioe) {
308                            if (directory != null) {
309                                    try {
310                                            directory.close();
311                                    }
312                                    catch (Exception e) {
313                                    }
314                            }
315                    }
316    
317                    return directory;
318            }
319    
320            private Directory _getLuceneDirRam() {
321                    String path = _getPath();
322    
323                    Directory directory = _ramDirectories.get(path);
324    
325                    if (directory == null) {
326                            directory = new RAMDirectory();
327    
328                            _ramDirectories.put(path, directory);
329                    }
330    
331                    return directory;
332            }
333    
334            private MergePolicy _getMergePolicy() throws Exception {
335                    ClassLoader classLoader = ClassLoaderUtil.getPortalClassLoader();
336    
337                    MergePolicy mergePolicy = (MergePolicy)InstanceFactory.newInstance(
338                            classLoader, PropsValues.LUCENE_MERGE_POLICY);
339    
340                    if (mergePolicy instanceof LogMergePolicy) {
341                            LogMergePolicy logMergePolicy = (LogMergePolicy)mergePolicy;
342    
343                            logMergePolicy.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
344                    }
345    
346                    return mergePolicy;
347            }
348    
349            private String _getPath() {
350                    return PropsValues.LUCENE_DIR.concat(String.valueOf(_companyId)).concat(
351                            StringPool.SLASH);
352            }
353    
354            private void _initCommitScheduler() {
355                    if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
356                            (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
357    
358                            return;
359                    }
360    
361                    ScheduledExecutorService scheduledExecutorService =
362                            Executors.newSingleThreadScheduledExecutor();
363    
364                    Runnable runnable = new Runnable() {
365    
366                            public void run() {
367                                    try {
368                                            if (_batchCount > 0) {
369                                                    _doCommit();
370                                            }
371                                    }
372                                    catch (IOException ioe) {
373                                            _log.error("Could not run scheduled commit", ioe);
374                                    }
375                            }
376    
377                    };
378    
379                    scheduledExecutorService.scheduleWithFixedDelay(
380                            runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
381                            TimeUnit.MILLISECONDS);
382            }
383    
384            private void _initIndexWriter() {
385                    try {
386                            Analyzer analyzer = new LimitTokenCountAnalyzer(
387                                    LuceneHelperUtil.getAnalyzer(),
388                                    PropsValues.LUCENE_ANALYZER_MAX_TOKENS);
389    
390                            IndexWriterConfig indexWriterConfig = new IndexWriterConfig(
391                                    LuceneHelperUtil.getVersion(), analyzer);
392    
393                            indexWriterConfig.setIndexDeletionPolicy(_dumpIndexDeletionPolicy);
394                            indexWriterConfig.setMergePolicy(_getMergePolicy());
395                            indexWriterConfig.setRAMBufferSizeMB(
396                                    PropsValues.LUCENE_BUFFER_SIZE);
397    
398                            _indexWriter = new IndexWriter(getLuceneDir(), indexWriterConfig);
399    
400                            if (!IndexReader.indexExists(getLuceneDir())) {
401    
402                                    // Workaround for LUCENE-2386
403    
404                                    if (_log.isDebugEnabled()) {
405                                            _log.debug("Creating missing index");
406                                    }
407    
408                                    _doCommit();
409                            }
410                    }
411                    catch (Exception e) {
412                            _log.error(
413                                    "Initializing Lucene writer failed for " + _companyId, e);
414                    }
415            }
416    
417            private void _write(Term term, Document document) throws IOException {
418                    try {
419                            if (term != null) {
420                                    _indexWriter.updateDocument(term, document);
421                            }
422                            else {
423                                    _indexWriter.addDocument(document);
424                            }
425    
426                            _batchCount++;
427                    }
428                    finally {
429                            _commit();
430                    }
431            }
432    
433            private static final String _LUCENE_STORE_TYPE_FILE = "file";
434    
435            private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
436    
437            private static final String _LUCENE_STORE_TYPE_RAM = "ram";
438    
439            private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
440    
441            private volatile int _batchCount;
442            private Lock _commitLock = new ReentrantLock();
443            private long _companyId;
444            private DumpIndexDeletionPolicy _dumpIndexDeletionPolicy =
445                    new DumpIndexDeletionPolicy();
446            private IndexWriter _indexWriter;
447            private Map<String, Directory> _ramDirectories =
448                    new ConcurrentHashMap<String, Directory>();
449    
450    }