001    /**
002     * Copyright (c) 2000-2012 Liferay, Inc. All rights reserved.
003     *
004     * The contents of this file are subject to the terms of the Liferay Enterprise
005     * Subscription License ("License"). You may not use this file except in
006     * compliance with the License. You can obtain a copy of the License by
007     * contacting Liferay, Inc. See the License for the specific language governing
008     * permissions and limitations under the License, including but not limited to
009     * distribution rights of the Software.
010     *
011     *
012     *
013     */
014    
015    package com.liferay.portal.search.lucene;
016    
017    import com.liferay.portal.kernel.log.Log;
018    import com.liferay.portal.kernel.log.LogFactoryUtil;
019    import com.liferay.portal.kernel.search.SearchEngineUtil;
020    import com.liferay.portal.kernel.util.FileUtil;
021    import com.liferay.portal.kernel.util.StringPool;
022    import com.liferay.portal.search.lucene.dump.DumpIndexDeletionPolicy;
023    import com.liferay.portal.search.lucene.dump.IndexCommitSerializationUtil;
024    import com.liferay.portal.util.PropsValues;
025    
026    import java.io.File;
027    import java.io.IOException;
028    import java.io.InputStream;
029    import java.io.OutputStream;
030    
031    import java.util.Map;
032    import java.util.concurrent.ConcurrentHashMap;
033    import java.util.concurrent.Executors;
034    import java.util.concurrent.ScheduledExecutorService;
035    import java.util.concurrent.TimeUnit;
036    import java.util.concurrent.locks.Lock;
037    import java.util.concurrent.locks.ReentrantLock;
038    
039    import org.apache.lucene.document.Document;
040    import org.apache.lucene.index.IndexWriter;
041    import org.apache.lucene.index.Term;
042    import org.apache.lucene.store.Directory;
043    import org.apache.lucene.store.FSDirectory;
044    import org.apache.lucene.store.MMapDirectory;
045    import org.apache.lucene.store.RAMDirectory;
046    
047    /**
048     * @author Harry Mark
049     * @author Brian Wing Shun Chan
050     * @author Bruno Farache
051     * @author Shuyang Zhou
052     */
053    public class IndexAccessorImpl implements IndexAccessor {
054    
055            public IndexAccessorImpl(long companyId) {
056                    _companyId = companyId;
057    
058                    _checkLuceneDir();
059                    _initIndexWriter();
060                    _initCommitScheduler();
061            }
062    
063            public void addDocument(Document document) throws IOException {
064                    if (SearchEngineUtil.isIndexReadOnly()) {
065                            return;
066                    }
067    
068                    _write(null, document);
069            }
070    
071            public void close() {
072                    try {
073                            _indexWriter.close();
074                    }
075                    catch (Exception e) {
076                            _log.error("Closing Lucene writer failed for " + _companyId, e);
077                    }
078            }
079    
080            public void delete() {
081                    if (SearchEngineUtil.isIndexReadOnly()) {
082                            return;
083                    }
084    
085                    close();
086    
087                    _deleteDirectory();
088    
089                    _initIndexWriter();
090            }
091    
092            public void deleteDocuments(Term term) throws IOException {
093                    if (SearchEngineUtil.isIndexReadOnly()) {
094                            return;
095                    }
096    
097                    try {
098                            _indexWriter.deleteDocuments(term);
099    
100                            _batchCount++;
101                    }
102                    finally {
103                            _commit();
104                    }
105            }
106    
107            public void dumpIndex(OutputStream outputStream) throws IOException {
108                    _dumpIndexDeletionPolicy.dump(outputStream, _indexWriter, _commitLock);
109            }
110    
111            public long getCompanyId() {
112                    return _companyId;
113            }
114    
115            public long getLastGeneration() {
116                    return _dumpIndexDeletionPolicy.getLastGeneration();
117            }
118    
119            public Directory getLuceneDir() {
120                    if (_log.isDebugEnabled()) {
121                            _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
122                    }
123    
124                    if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
125                            return _getLuceneDirFile();
126                    }
127                    else if (PropsValues.LUCENE_STORE_TYPE.equals(
128                                            _LUCENE_STORE_TYPE_JDBC)) {
129    
130                            throw new IllegalArgumentException(
131                                    "Store type JDBC is no longer supported in favor of SOLR");
132                    }
133                    else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
134                            return _getLuceneDirRam();
135                    }
136                    else {
137                            throw new RuntimeException(
138                                    "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
139                    }
140            }
141    
142            public void loadIndex(InputStream inputStream) throws IOException {
143                    File tempFile = FileUtil.createTempFile();
144    
145                    Directory tempDirectory = FSDirectory.open(tempFile);
146    
147                    IndexCommitSerializationUtil.deserializeIndex(
148                            inputStream, tempDirectory);
149    
150                    close();
151    
152                    _deleteDirectory();
153    
154                    Directory.copy(tempDirectory, getLuceneDir(), true);
155    
156                    _initIndexWriter();
157    
158                    tempDirectory.close();
159    
160                    FileUtil.deltree(tempFile);
161            }
162    
163            public void updateDocument(Term term, Document document)
164                    throws IOException {
165    
166                    if (SearchEngineUtil.isIndexReadOnly()) {
167                            return;
168                    }
169    
170                    if (_log.isDebugEnabled()) {
171                            _log.debug("Indexing " + document);
172                    }
173    
174                    _write(term, document);
175            }
176    
177            private void _checkLuceneDir() {
178                    if (SearchEngineUtil.isIndexReadOnly()) {
179                            return;
180                    }
181    
182                    try {
183                            Directory directory = getLuceneDir();
184    
185                            if (IndexWriter.isLocked(directory)) {
186                                    IndexWriter.unlock(directory);
187                            }
188                    }
189                    catch (Exception e) {
190                            _log.error("Check Lucene directory failed for " + _companyId, e);
191                    }
192            }
193    
194            private void _commit() throws IOException {
195                    if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
196                            (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
197    
198                            _doCommit();
199                    }
200            }
201    
202            private void _deleteDirectory() {
203                    if (_log.isDebugEnabled()) {
204                            _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
205                    }
206    
207                    if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
208                            _deleteFile();
209                    }
210                    else if (PropsValues.LUCENE_STORE_TYPE.equals(
211                                            _LUCENE_STORE_TYPE_JDBC)) {
212    
213                            throw new IllegalArgumentException(
214                                    "Store type JDBC is no longer supported in favor of SOLR");
215                    }
216                    else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
217                            _deleteRam();
218                    }
219                    else {
220                            throw new RuntimeException(
221                                    "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
222                    }
223            }
224    
225            private void _deleteFile() {
226                    String path = _getPath();
227    
228                    try {
229                            Directory directory = _getDirectory(path);
230    
231                            directory.close();
232                    }
233                    catch (Exception e) {
234                            if (_log.isWarnEnabled()) {
235                                    _log.warn("Could not close directory " + path);
236                            }
237                    }
238    
239                    FileUtil.deltree(path);
240            }
241    
242            private void _deleteRam() {
243            }
244    
245            private void _doCommit() throws IOException {
246                    if (_indexWriter != null) {
247                            _commitLock.lock();
248    
249                            try {
250                                    _indexWriter.commit();
251                            }
252                            finally {
253                                    _commitLock.unlock();
254                            }
255                    }
256    
257                    _batchCount = 0;
258            }
259    
260            private FSDirectory _getDirectory(String path) throws IOException {
261                    if (PropsValues.LUCENE_STORE_TYPE_FILE_FORCE_MMAP) {
262                            return new MMapDirectory(new File(path));
263                    }
264                    else {
265                            return FSDirectory.open(new File(path));
266                    }
267            }
268    
269            private Directory _getLuceneDirFile() {
270                    Directory directory = null;
271    
272                    String path = _getPath();
273    
274                    try {
275                            directory = _getDirectory(path);
276                    }
277                    catch (IOException ioe1) {
278                            if (directory != null) {
279                                    try {
280                                            directory.close();
281                                    }
282                                    catch (Exception e) {
283                                    }
284                            }
285                    }
286    
287                    return directory;
288            }
289    
290            private Directory _getLuceneDirRam() {
291                    String path = _getPath();
292    
293                    Directory directory = _ramDirectories.get(path);
294    
295                    if (directory == null) {
296                            directory = new RAMDirectory();
297    
298                            _ramDirectories.put(path, directory);
299                    }
300    
301                    return directory;
302            }
303    
304            private String _getPath() {
305                    return PropsValues.LUCENE_DIR.concat(String.valueOf(_companyId)).concat(
306                            StringPool.SLASH);
307            }
308    
309            private void _initCommitScheduler() {
310                    if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
311                            (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
312    
313                            return;
314                    }
315    
316                    ScheduledExecutorService scheduledExecutorService =
317                            Executors.newSingleThreadScheduledExecutor();
318    
319                    Runnable runnable = new Runnable() {
320    
321                            public void run() {
322                                    try {
323                                            if (_batchCount > 0) {
324                                                    _doCommit();
325                                            }
326                                    }
327                                    catch (IOException ioe) {
328                                            _log.error("Could not run scheduled commit", ioe);
329                                    }
330                            }
331    
332                    };
333    
334                    scheduledExecutorService.scheduleWithFixedDelay(
335                            runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
336                            TimeUnit.MILLISECONDS);
337            }
338    
339            private void _initIndexWriter() {
340                    try {
341                            _indexWriter = new IndexWriter(
342                                    getLuceneDir(), LuceneHelperUtil.getAnalyzer(),
343                                    _dumpIndexDeletionPolicy, IndexWriter.MaxFieldLength.LIMITED);
344    
345                            _indexWriter.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
346                            _indexWriter.setRAMBufferSizeMB(PropsValues.LUCENE_BUFFER_SIZE);
347                    }
348                    catch (Exception e) {
349                            _log.error(
350                                    "Initializing Lucene writer failed for " + _companyId, e);
351                    }
352            }
353    
354            private void _write(Term term, Document document) throws IOException {
355                    try {
356                            if (term != null) {
357                                    _indexWriter.updateDocument(term, document);
358                            }
359                            else {
360                                    _indexWriter.addDocument(document);
361                            }
362    
363                            _optimizeCount++;
364    
365                            if ((PropsValues.LUCENE_OPTIMIZE_INTERVAL == 0) ||
366                                    (_optimizeCount >= PropsValues.LUCENE_OPTIMIZE_INTERVAL)) {
367    
368                                    _indexWriter.optimize();
369    
370                                    _optimizeCount = 0;
371                            }
372    
373                            _batchCount++;
374                    }
375                    finally {
376                            _commit();
377                    }
378            }
379    
380            private static final String _LUCENE_STORE_TYPE_FILE = "file";
381    
382            private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
383    
384            private static final String _LUCENE_STORE_TYPE_RAM = "ram";
385    
386            private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
387    
388            private volatile int _batchCount;
389            private Lock _commitLock = new ReentrantLock();
390            private long _companyId;
391            private DumpIndexDeletionPolicy _dumpIndexDeletionPolicy =
392                    new DumpIndexDeletionPolicy();
393            private IndexWriter _indexWriter;
394            private int _optimizeCount;
395            private Map<String, Directory> _ramDirectories =
396                    new ConcurrentHashMap<String, Directory>();
397    
398    }