001
014
015 package com.liferay.portal.search.lucene;
016
017 import com.liferay.portal.kernel.log.Log;
018 import com.liferay.portal.kernel.log.LogFactoryUtil;
019 import com.liferay.portal.kernel.search.SearchEngineUtil;
020 import com.liferay.portal.kernel.util.FileUtil;
021 import com.liferay.portal.kernel.util.StringPool;
022 import com.liferay.portal.search.lucene.dump.DumpIndexDeletionPolicy;
023 import com.liferay.portal.search.lucene.dump.IndexCommitSerializationUtil;
024 import com.liferay.portal.util.PropsValues;
025
026 import java.io.File;
027 import java.io.IOException;
028 import java.io.InputStream;
029 import java.io.OutputStream;
030
031 import java.util.Map;
032 import java.util.concurrent.ConcurrentHashMap;
033 import java.util.concurrent.Executors;
034 import java.util.concurrent.ScheduledExecutorService;
035 import java.util.concurrent.TimeUnit;
036 import java.util.concurrent.locks.Lock;
037 import java.util.concurrent.locks.ReentrantLock;
038
039 import org.apache.lucene.document.Document;
040 import org.apache.lucene.index.IndexWriter;
041 import org.apache.lucene.index.Term;
042 import org.apache.lucene.store.Directory;
043 import org.apache.lucene.store.FSDirectory;
044 import org.apache.lucene.store.MMapDirectory;
045 import org.apache.lucene.store.RAMDirectory;
046
047
053 public class IndexAccessorImpl implements IndexAccessor {
054
055 public IndexAccessorImpl(long companyId) {
056 _companyId = companyId;
057
058 _checkLuceneDir();
059 _initIndexWriter();
060 _initCommitScheduler();
061 }
062
063 public void addDocument(Document document) throws IOException {
064 if (SearchEngineUtil.isIndexReadOnly()) {
065 return;
066 }
067
068 _write(null, document);
069 }
070
071 public void close() {
072 try {
073 _indexWriter.close();
074 }
075 catch (Exception e) {
076 _log.error("Closing Lucene writer failed for " + _companyId, e);
077 }
078 }
079
080 public void delete() {
081 if (SearchEngineUtil.isIndexReadOnly()) {
082 return;
083 }
084
085 close();
086
087 _deleteDirectory();
088
089 _initIndexWriter();
090 }
091
092 public void deleteDocuments(Term term) throws IOException {
093 if (SearchEngineUtil.isIndexReadOnly()) {
094 return;
095 }
096
097 try {
098 _indexWriter.deleteDocuments(term);
099
100 _batchCount++;
101 }
102 finally {
103 _commit();
104 }
105 }
106
107 public void dumpIndex(OutputStream outputStream) throws IOException {
108 _dumpIndexDeletionPolicy.dump(outputStream, _indexWriter, _commitLock);
109 }
110
111 public long getCompanyId() {
112 return _companyId;
113 }
114
115 public long getLastGeneration() {
116 return _dumpIndexDeletionPolicy.getLastGeneration();
117 }
118
119 public Directory getLuceneDir() {
120 if (_log.isDebugEnabled()) {
121 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
122 }
123
124 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
125 return _getLuceneDirFile();
126 }
127 else if (PropsValues.LUCENE_STORE_TYPE.equals(
128 _LUCENE_STORE_TYPE_JDBC)) {
129
130 throw new IllegalArgumentException(
131 "Store type JDBC is no longer supported in favor of SOLR");
132 }
133 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
134 return _getLuceneDirRam();
135 }
136 else {
137 throw new RuntimeException(
138 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
139 }
140 }
141
142 public void loadIndex(InputStream inputStream) throws IOException {
143 File tempFile = FileUtil.createTempFile();
144
145 Directory tempDirectory = FSDirectory.open(tempFile);
146
147 IndexCommitSerializationUtil.deserializeIndex(
148 inputStream, tempDirectory);
149
150 close();
151
152 _deleteDirectory();
153
154 Directory.copy(tempDirectory, getLuceneDir(), true);
155
156 _initIndexWriter();
157
158 tempDirectory.close();
159
160 FileUtil.deltree(tempFile);
161 }
162
163 public void updateDocument(Term term, Document document)
164 throws IOException {
165
166 if (SearchEngineUtil.isIndexReadOnly()) {
167 return;
168 }
169
170 if (_log.isDebugEnabled()) {
171 _log.debug("Indexing " + document);
172 }
173
174 _write(term, document);
175 }
176
177 private void _checkLuceneDir() {
178 if (SearchEngineUtil.isIndexReadOnly()) {
179 return;
180 }
181
182 try {
183 Directory directory = getLuceneDir();
184
185 if (IndexWriter.isLocked(directory)) {
186 IndexWriter.unlock(directory);
187 }
188 }
189 catch (Exception e) {
190 _log.error("Check Lucene directory failed for " + _companyId, e);
191 }
192 }
193
194 private void _commit() throws IOException {
195 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
196 (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
197
198 _doCommit();
199 }
200 }
201
202 private void _deleteDirectory() {
203 if (_log.isDebugEnabled()) {
204 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
205 }
206
207 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
208 _deleteFile();
209 }
210 else if (PropsValues.LUCENE_STORE_TYPE.equals(
211 _LUCENE_STORE_TYPE_JDBC)) {
212
213 throw new IllegalArgumentException(
214 "Store type JDBC is no longer supported in favor of SOLR");
215 }
216 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
217 _deleteRam();
218 }
219 else {
220 throw new RuntimeException(
221 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
222 }
223 }
224
225 private void _deleteFile() {
226 String path = _getPath();
227
228 try {
229 Directory directory = _getDirectory(path);
230
231 directory.close();
232 }
233 catch (Exception e) {
234 if (_log.isWarnEnabled()) {
235 _log.warn("Could not close directory " + path);
236 }
237 }
238
239 FileUtil.deltree(path);
240 }
241
242 private void _deleteRam() {
243 }
244
245 private void _doCommit() throws IOException {
246 if (_indexWriter != null) {
247 _commitLock.lock();
248
249 try {
250 _indexWriter.commit();
251 }
252 finally {
253 _commitLock.unlock();
254 }
255 }
256
257 _batchCount = 0;
258 }
259
260 private FSDirectory _getDirectory(String path) throws IOException {
261 if (PropsValues.LUCENE_STORE_TYPE_FILE_FORCE_MMAP) {
262 return new MMapDirectory(new File(path));
263 }
264 else {
265 return FSDirectory.open(new File(path));
266 }
267 }
268
269 private Directory _getLuceneDirFile() {
270 Directory directory = null;
271
272 String path = _getPath();
273
274 try {
275 directory = _getDirectory(path);
276 }
277 catch (IOException ioe1) {
278 if (directory != null) {
279 try {
280 directory.close();
281 }
282 catch (Exception e) {
283 }
284 }
285 }
286
287 return directory;
288 }
289
290 private Directory _getLuceneDirRam() {
291 String path = _getPath();
292
293 Directory directory = _ramDirectories.get(path);
294
295 if (directory == null) {
296 directory = new RAMDirectory();
297
298 _ramDirectories.put(path, directory);
299 }
300
301 return directory;
302 }
303
304 private String _getPath() {
305 return PropsValues.LUCENE_DIR.concat(String.valueOf(_companyId)).concat(
306 StringPool.SLASH);
307 }
308
309 private void _initCommitScheduler() {
310 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
311 (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
312
313 return;
314 }
315
316 ScheduledExecutorService scheduledExecutorService =
317 Executors.newSingleThreadScheduledExecutor();
318
319 Runnable runnable = new Runnable() {
320
321 public void run() {
322 try {
323 if (_batchCount > 0) {
324 _doCommit();
325 }
326 }
327 catch (IOException ioe) {
328 _log.error("Could not run scheduled commit", ioe);
329 }
330 }
331
332 };
333
334 scheduledExecutorService.scheduleWithFixedDelay(
335 runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
336 TimeUnit.MILLISECONDS);
337 }
338
339 private void _initIndexWriter() {
340 try {
341 _indexWriter = new IndexWriter(
342 getLuceneDir(), LuceneHelperUtil.getAnalyzer(),
343 _dumpIndexDeletionPolicy, IndexWriter.MaxFieldLength.LIMITED);
344
345 _indexWriter.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
346 _indexWriter.setRAMBufferSizeMB(PropsValues.LUCENE_BUFFER_SIZE);
347 }
348 catch (Exception e) {
349 _log.error(
350 "Initializing Lucene writer failed for " + _companyId, e);
351 }
352 }
353
354 private void _write(Term term, Document document) throws IOException {
355 try {
356 if (term != null) {
357 _indexWriter.updateDocument(term, document);
358 }
359 else {
360 _indexWriter.addDocument(document);
361 }
362
363 _optimizeCount++;
364
365 if ((PropsValues.LUCENE_OPTIMIZE_INTERVAL == 0) ||
366 (_optimizeCount >= PropsValues.LUCENE_OPTIMIZE_INTERVAL)) {
367
368 _indexWriter.optimize();
369
370 _optimizeCount = 0;
371 }
372
373 _batchCount++;
374 }
375 finally {
376 _commit();
377 }
378 }
379
380 private static final String _LUCENE_STORE_TYPE_FILE = "file";
381
382 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
383
384 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
385
386 private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
387
388 private volatile int _batchCount;
389 private Lock _commitLock = new ReentrantLock();
390 private long _companyId;
391 private DumpIndexDeletionPolicy _dumpIndexDeletionPolicy =
392 new DumpIndexDeletionPolicy();
393 private IndexWriter _indexWriter;
394 private int _optimizeCount;
395 private Map<String, Directory> _ramDirectories =
396 new ConcurrentHashMap<String, Directory>();
397
398 }