1   /**
2    * Copyright (c) 2000-2009 Liferay, Inc. All rights reserved.
3    *
4    * Permission is hereby granted, free of charge, to any person obtaining a copy
5    * of this software and associated documentation files (the "Software"), to deal
6    * in the Software without restriction, including without limitation the rights
7    * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8    * copies of the Software, and to permit persons to whom the Software is
9    * furnished to do so, subject to the following conditions:
10   *
11   * The above copyright notice and this permission notice shall be included in
12   * all copies or substantial portions of the Software.
13   *
14   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  
23  package com.liferay.portal.search.lucene;
24  
25  import com.liferay.portal.SystemException;
26  import com.liferay.portal.kernel.log.Log;
27  import com.liferay.portal.kernel.log.LogFactoryUtil;
28  import com.liferay.portal.kernel.search.SearchEngineUtil;
29  import com.liferay.portal.kernel.util.GetterUtil;
30  import com.liferay.portal.model.Company;
31  import com.liferay.portal.model.CompanyConstants;
32  import com.liferay.portal.service.CompanyLocalServiceUtil;
33  import com.liferay.portal.util.PropsKeys;
34  import com.liferay.portal.util.PropsUtil;
35  import com.liferay.util.SystemProperties;
36  
37  import java.io.File;
38  import java.io.IOException;
39  
40  import java.util.HashMap;
41  import java.util.List;
42  import java.util.Map;
43  import java.util.concurrent.Semaphore;
44  
45  import org.apache.lucene.analysis.SimpleAnalyzer;
46  import org.apache.lucene.index.IndexReader;
47  import org.apache.lucene.index.IndexWriter;
48  import org.apache.lucene.index.Term;
49  import org.apache.lucene.store.Directory;
50  import org.apache.lucene.store.FSDirectory;
51  
52  /**
53   * <a href="IndexWriterFactory.java.html"><b><i>View Source</i></b></a>
54   *
55   * <p>
56   * Lucene only allows one IndexWriter to be open at a time. However, multiple
57   * threads can use this single IndexWriter. This class manages a global
58   * IndexWriter and uses reference counting to determine when it can be closed.
59   * </p>
60   *
61   * <p>
62   * To delete documents, IndexReaders are used but cannot delete while another
63   * IndexWriter or IndexReader has the write lock. A semaphore is used to
64   * serialize delete and add operations. If the shared IndexWriter is open,
65   * concurrent add operations are permitted.
66   * </p>
67   *
68   * @author Harry Mark
69   * @author Brian Wing Shun Chan
70   *
71   */
72  public class IndexWriterFactory {
73  
74      public IndexWriterFactory() {
75          if (SearchEngineUtil.isIndexReadOnly()) {
76              return;
77          }
78  
79          // Create semaphores for all companies
80  
81          try {
82              List<Company> companies = CompanyLocalServiceUtil.getCompanies(
83                  false);
84  
85              for (Company company : companies) {
86                  _lockLookup.put(company.getCompanyId(), new Semaphore(1));
87              }
88  
89              _lockLookup.put(CompanyConstants.SYSTEM, new Semaphore(1));
90          }
91          catch (SystemException se) {
92              _log.error(se);
93          }
94      }
95  
96      public void acquireLock(long companyId, boolean needExclusive)
97          throws InterruptedException {
98  
99          if (SearchEngineUtil.isIndexReadOnly()) {
100             return;
101         }
102 
103         Semaphore lock = _lockLookup.get(companyId);
104 
105         if (lock != null) {
106 
107             // Exclusive checking is used to prevent greedy IndexWriter sharing.
108             // This registers a need for exclusive lock, and causes IndexWriters
109             // to wait in FIFO order.
110 
111             if (needExclusive) {
112                 synchronized (_lockLookup) {
113                     _needExclusiveLock++;
114                 }
115             }
116 
117             try {
118                 lock.acquire();
119             }
120             finally {
121                 if (needExclusive) {
122                     synchronized (_lockLookup) {
123                         _needExclusiveLock--;
124                     }
125                 }
126             }
127         }
128         else {
129             if (_log.isWarnEnabled()) {
130                 _log.warn("IndexWriterFactory lock not found for " + companyId);
131             }
132         }
133     }
134 
135     public void deleteDocuments(long companyId, Term term)
136         throws InterruptedException, IOException {
137 
138         if (SearchEngineUtil.isIndexReadOnly()) {
139             return;
140         }
141 
142         try {
143             acquireLock(companyId, true);
144 
145             IndexReader reader = null;
146 
147             try {
148                 reader = IndexReader.open(LuceneUtil.getLuceneDir(companyId));
149 
150                 reader.deleteDocuments(term);
151             }
152             finally {
153                 if (reader != null) {
154                     reader.close();
155                 }
156             }
157         }
158         finally {
159             releaseLock(companyId);
160         }
161     }
162 
163     public IndexWriter getWriter(long companyId, boolean create)
164         throws IOException {
165 
166         if (SearchEngineUtil.isIndexReadOnly()) {
167             return getReadOnlyIndexWriter();
168         }
169 
170         boolean hasError = false;
171         boolean newWriter = false;
172 
173         try {
174 
175             // If others need an exclusive lock, then wait to acquire lock
176             // before proceeding. This prevents starvation.
177 
178             if (_needExclusiveLock > 0) {
179                 acquireLock(companyId, false);
180                 releaseLock(companyId);
181             }
182 
183             synchronized (this) {
184                 IndexWriterData writerData = _writerLookup.get(companyId);
185 
186                 if (writerData == null) {
187                     newWriter = true;
188 
189                     acquireLock(companyId, false);
190 
191                     IndexWriter writer = new IndexWriter(
192                         LuceneUtil.getLuceneDir(companyId),
193                         LuceneUtil.getAnalyzer(), create);
194 
195                     writer.setMergeFactor(_MERGE_FACTOR);
196 
197                     writerData = new IndexWriterData(companyId, writer, 0);
198 
199                     _writerLookup.put(companyId, writerData);
200                 }
201 
202                 writerData.setCount(writerData.getCount() + 1);
203 
204                 return writerData.getWriter();
205             }
206         }
207         catch (Exception e) {
208             hasError = true;
209 
210             _log.error("Unable to create a new writer", e);
211 
212             throw new IOException("Unable to create a new writer");
213         }
214         finally {
215             if (hasError && newWriter) {
216                 try {
217                     releaseLock(companyId);
218                 }
219                 catch (Exception e) {
220                 }
221             }
222         }
223     }
224 
225     public void releaseLock(long companyId) {
226         if (SearchEngineUtil.isIndexReadOnly()) {
227             return;
228         }
229 
230         Semaphore lock = _lockLookup.get(companyId);
231 
232         if (lock != null) {
233             lock.release();
234         }
235     }
236 
237     public void write(long companyId) {
238         if (SearchEngineUtil.isIndexReadOnly()) {
239             return;
240         }
241 
242         IndexWriterData writerData = _writerLookup.get(companyId);
243 
244         if (writerData != null) {
245             decrement(writerData);
246         }
247         else {
248             if (_log.isWarnEnabled()) {
249                 _log.warn("IndexWriterData not found for " + companyId);
250             }
251         }
252     }
253 
254     public void write(IndexWriter writer) throws IOException {
255         if (SearchEngineUtil.isIndexReadOnly()) {
256             return;
257         }
258 
259         boolean writerFound = false;
260 
261         synchronized (this) {
262             if (!_writerLookup.isEmpty()) {
263                 for (IndexWriterData writerData : _writerLookup.values()) {
264                     if (writerData.getWriter() == writer) {
265                         writerFound = true;
266 
267                         decrement(writerData);
268 
269                         break;
270                     }
271                 }
272             }
273         }
274 
275         if (!writerFound) {
276             try {
277                 _optimizeCount++;
278 
279                 if ((_OPTIMIZE_INTERVAL == 0) ||
280                     (_optimizeCount >= _OPTIMIZE_INTERVAL)) {
281 
282                     writer.optimize();
283 
284                     _optimizeCount = 0;
285                 }
286             }
287             finally {
288                 writer.close();
289             }
290         }
291     }
292 
293     protected void decrement(IndexWriterData writerData) {
294         if (writerData.getCount() > 0) {
295             writerData.setCount(writerData.getCount() - 1);
296 
297             if (writerData.getCount() == 0) {
298                 _writerLookup.remove(writerData.getCompanyId());
299 
300                 try {
301                     IndexWriter writer = writerData.getWriter();
302 
303                     try {
304                         _optimizeCount++;
305 
306                         if ((_OPTIMIZE_INTERVAL == 0) ||
307                             (_optimizeCount >= _OPTIMIZE_INTERVAL)) {
308 
309                             writer.optimize();
310 
311                             _optimizeCount = 0;
312                         }
313                     }
314                     finally {
315                         writer.close();
316                     }
317                 }
318                 catch (Exception e) {
319                     _log.error(e, e);
320                 }
321                 finally {
322                     releaseLock(writerData.getCompanyId());
323                 }
324             }
325         }
326     }
327 
328     protected IndexWriter getReadOnlyIndexWriter() {
329         if (_readOnlyIndexWriter == null) {
330             try {
331                 if (_log.isInfoEnabled()) {
332                     _log.info("Disabling writing to index for this process");
333                 }
334 
335                 _readOnlyIndexWriter = new ReadOnlyIndexWriter(
336                     getReadOnlyLuceneDir(), new SimpleAnalyzer(), true);
337             }
338             catch (IOException ioe) {
339                 throw new RuntimeException(ioe);
340             }
341         }
342 
343         return _readOnlyIndexWriter;
344     }
345 
346     protected Directory getReadOnlyLuceneDir() throws IOException {
347         if (_readOnlyLuceneDir == null) {
348             String tmpDir = SystemProperties.get(SystemProperties.TMP_DIR);
349 
350             File dir = new File(tmpDir + "/liferay/lucene/empty");
351 
352             dir.mkdir();
353 
354             _readOnlyLuceneDir = LuceneUtil.getDirectory(dir.getPath(), false);
355         }
356 
357         return _readOnlyLuceneDir;
358     }
359 
360     private static final int _MERGE_FACTOR = GetterUtil.getInteger(
361         PropsUtil.get(PropsKeys.LUCENE_MERGE_FACTOR));
362 
363     private static final int _OPTIMIZE_INTERVAL = GetterUtil.getInteger(
364         PropsUtil.get(PropsKeys.LUCENE_OPTIMIZE_INTERVAL));
365 
366     private static Log _log = LogFactoryUtil.getLog(IndexWriterFactory.class);
367 
368     private FSDirectory _readOnlyLuceneDir = null;
369     private IndexWriter _readOnlyIndexWriter = null;
370     private Map<Long, Semaphore> _lockLookup = new HashMap<Long, Semaphore>();
371     private Map<Long, IndexWriterData> _writerLookup =
372         new HashMap<Long, IndexWriterData>();
373     private int _needExclusiveLock = 0;
374     private int _optimizeCount = 0;
375 
376 }