1   /**
2    * Copyright (c) 2000-2009 Liferay, Inc. All rights reserved.
3    *
4    *
5    *
6    *
7    * The contents of this file are subject to the terms of the Liferay Enterprise
8    * Subscription License ("License"). You may not use this file except in
9    * compliance with the License. You can obtain a copy of the License by
10   * contacting Liferay, Inc. See the License for the specific language governing
11   * permissions and limitations under the License, including but not limited to
12   * distribution rights of the Software.
13   *
14   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  
23  package com.liferay.portal.search.lucene;
24  
25  import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26  import com.liferay.portal.kernel.log.Log;
27  import com.liferay.portal.kernel.log.LogFactoryUtil;
28  import com.liferay.portal.kernel.search.SearchEngineUtil;
29  import com.liferay.portal.kernel.util.FileUtil;
30  import com.liferay.portal.kernel.util.InfrastructureUtil;
31  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
32  import com.liferay.portal.kernel.util.PropsKeys;
33  import com.liferay.portal.kernel.util.StringPool;
34  import com.liferay.portal.util.PropsUtil;
35  import com.liferay.portal.util.PropsValues;
36  
37  import java.io.File;
38  import java.io.IOException;
39  
40  import java.sql.Connection;
41  import java.sql.DatabaseMetaData;
42  import java.sql.ResultSet;
43  import java.sql.Statement;
44  
45  import java.util.Map;
46  import java.util.concurrent.ConcurrentHashMap;
47  import java.util.concurrent.Executors;
48  import java.util.concurrent.ScheduledExecutorService;
49  import java.util.concurrent.TimeUnit;
50  
51  import javax.sql.DataSource;
52  
53  import org.apache.lucene.document.Document;
54  import org.apache.lucene.index.IndexWriter;
55  import org.apache.lucene.index.Term;
56  import org.apache.lucene.store.Directory;
57  import org.apache.lucene.store.FSDirectory;
58  import org.apache.lucene.store.RAMDirectory;
59  import org.apache.lucene.store.jdbc.JdbcDirectory;
60  import org.apache.lucene.store.jdbc.JdbcStoreException;
61  import org.apache.lucene.store.jdbc.dialect.Dialect;
62  import org.apache.lucene.store.jdbc.lock.JdbcLock;
63  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
64  
65  /**
66   * <a href="IndexAccessorImpl.java.html"><b><i>View Source</i></b></a>
67   *
68   * @author Harry Mark
69   * @author Brian Wing Shun Chan
70   * @author Bruno Farache
71   */
72  public class IndexAccessorImpl implements IndexAccessor {
73  
74      public IndexAccessorImpl(long companyId) {
75          _companyId = companyId;
76  
77          _checkLuceneDir();
78          _initIndexWriter();
79          _initCommitScheduler();
80          _initDialect();
81      }
82  
83      public void addDocument(Document document) throws IOException {
84          if (SearchEngineUtil.isIndexReadOnly()) {
85              return;
86          }
87  
88          _write(null, document);
89      }
90  
91      public void close() {
92          try {
93              _indexWriter.close();
94          }
95          catch(Exception e) {
96              _log.error(
97                  "Closing Lucene writer failed for " + _companyId, e);
98          }
99      }
100 
101     public void delete() {
102         if (SearchEngineUtil.isIndexReadOnly()) {
103             return;
104         }
105 
106         close();
107 
108         if (_log.isDebugEnabled()) {
109             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
110         }
111 
112         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
113             _deleteFile();
114         }
115         else if (PropsValues.LUCENE_STORE_TYPE.equals(
116                     _LUCENE_STORE_TYPE_JDBC)) {
117 
118             _deleteJdbc();
119         }
120         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
121             _deleteRam();
122         }
123         else {
124             throw new RuntimeException(
125                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
126         }
127 
128         _initIndexWriter();
129     }
130 
131     public void deleteDocuments(Term term) throws IOException {
132         if (SearchEngineUtil.isIndexReadOnly()) {
133             return;
134         }
135 
136         try {
137             _indexWriter.deleteDocuments(term);
138 
139             _batchCount++;
140         }
141         finally {
142             _commit();
143         }
144     }
145 
146     public long getCompanyId() {
147         return _companyId;
148     }
149 
150     public Directory getLuceneDir() {
151         if (_log.isDebugEnabled()) {
152             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
153         }
154 
155         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
156             return _getLuceneDirFile();
157         }
158         else if (PropsValues.LUCENE_STORE_TYPE.equals(
159                     _LUCENE_STORE_TYPE_JDBC)) {
160 
161             return _getLuceneDirJdbc();
162         }
163         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
164             return _getLuceneDirRam();
165         }
166         else {
167             throw new RuntimeException(
168                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
169         }
170     }
171 
172     public void updateDocument(Term term, Document document)
173         throws IOException {
174 
175         if (SearchEngineUtil.isIndexReadOnly()) {
176             return;
177         }
178 
179         _write(term, document);
180     }
181 
182     private void _checkLuceneDir() {
183         if (SearchEngineUtil.isIndexReadOnly()) {
184             return;
185         }
186 
187         try {
188             Directory directory = getLuceneDir();
189 
190             if (IndexWriter.isLocked(directory)) {
191                 IndexWriter.unlock(directory);
192             }
193         }
194         catch (Exception e) {
195             _log.error("Check Lucene directory failed for " + _companyId, e);
196         }
197     }
198 
199     private void _commit() throws IOException {
200         if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
201             (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
202 
203             _doCommit();
204         }
205     }
206 
207     private void _deleteFile() {
208         String path = _getPath();
209 
210         try {
211             Directory directory = _getDirectory(path);
212 
213             directory.close();
214         }
215         catch (Exception e) {
216             if (_log.isWarnEnabled()) {
217                 _log.warn("Could not close directory " + path);
218             }
219         }
220 
221         FileUtil.deltree(path);
222     }
223 
224     private void _deleteJdbc() {
225         String tableName = _getTableName();
226 
227         try {
228             Directory directory = _jdbcDirectories.remove(tableName);
229 
230             if (directory != null) {
231                 directory.close();
232             }
233         }
234         catch (Exception e) {
235             if (_log.isWarnEnabled()) {
236                 _log.warn("Could not close directory " + tableName);
237             }
238         }
239 
240         Connection con = null;
241         Statement s = null;
242 
243         try {
244             con = DataAccess.getConnection();
245 
246             s = con.createStatement();
247 
248             s.executeUpdate("DELETE FROM " + tableName);
249         }
250         catch (Exception e) {
251             if (_log.isWarnEnabled()) {
252                 _log.warn("Could not truncate " + tableName);
253             }
254         }
255         finally {
256             DataAccess.cleanUp(con, s);
257         }
258     }
259 
260     private void _deleteRam() {
261     }
262 
263     private void _doCommit() throws IOException {
264         if (_indexWriter != null) {
265             _indexWriter.commit();
266         }
267 
268         _batchCount = 0;
269     }
270 
271     private FSDirectory _getDirectory(String path) throws IOException {
272         return FSDirectory.open(new File(path));
273     }
274 
275     private Directory _getLuceneDirFile() {
276         Directory directory = null;
277 
278         String path = _getPath();
279 
280         try {
281             directory = _getDirectory(path);
282         }
283         catch (IOException ioe1) {
284             if (directory != null) {
285                 try {
286                     directory.close();
287                 }
288                 catch (Exception e) {
289                 }
290             }
291         }
292 
293         return directory;
294     }
295 
296     private Directory _getLuceneDirJdbc() {
297         JdbcDirectory jdbcDirectory = null;
298 
299         Thread currentThread = Thread.currentThread();
300 
301         ClassLoader contextClassLoader = currentThread.getContextClassLoader();
302 
303         try {
304             currentThread.setContextClassLoader(
305                 PortalClassLoaderUtil.getClassLoader());
306 
307             String tableName = _getTableName();
308 
309             jdbcDirectory = (JdbcDirectory)_jdbcDirectories.get(tableName);
310 
311             if (jdbcDirectory != null) {
312                 return jdbcDirectory;
313             }
314 
315             try {
316                 DataSource dataSource = InfrastructureUtil.getDataSource();
317 
318                 jdbcDirectory = new JdbcDirectory(
319                     dataSource, _dialect, tableName);
320 
321                 _jdbcDirectories.put(tableName, jdbcDirectory);
322 
323                 if (!jdbcDirectory.tableExists()) {
324                     jdbcDirectory.create();
325                 }
326             }
327             catch (IOException ioe) {
328                 throw new RuntimeException(ioe);
329             }
330             catch (UnsupportedOperationException uoe) {
331                 if (_log.isWarnEnabled()) {
332                     _log.warn(
333                         "Database doesn't support the ability to check " +
334                             "whether a table exists");
335                 }
336 
337                 _manuallyCreateJdbcDirectory(jdbcDirectory, tableName);
338             }
339         }
340         finally {
341             currentThread.setContextClassLoader(contextClassLoader);
342         }
343 
344         return jdbcDirectory;
345     }
346 
347     private Directory _getLuceneDirRam() {
348         String path = _getPath();
349 
350         Directory directory = _ramDirectories.get(path);
351 
352         if (directory == null) {
353             directory = new RAMDirectory();
354 
355             _ramDirectories.put(path, directory);
356         }
357 
358         return directory;
359     }
360 
361     private String _getPath() {
362         StringBuilder sb = new StringBuilder();
363 
364         sb.append(PropsValues.LUCENE_DIR);
365         sb.append(_companyId);
366         sb.append(StringPool.SLASH);
367 
368         return sb.toString();
369     }
370 
371     private String _getTableName() {
372         return _LUCENE_TABLE_PREFIX + _companyId;
373     }
374 
375     private void _initCommitScheduler() {
376         if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
377             (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
378 
379             return;
380         }
381 
382         ScheduledExecutorService scheduledExecutorService =
383             Executors.newSingleThreadScheduledExecutor();
384 
385         Runnable runnable = new Runnable() {
386 
387             public void run() {
388                 try {
389                     _doCommit();
390                 }
391                 catch (IOException ioe) {
392                     _log.error("Could not run scheduled commit", ioe);
393                 }
394             }
395 
396         };
397 
398         scheduledExecutorService.scheduleWithFixedDelay(
399             runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
400             TimeUnit.MILLISECONDS);
401     }
402 
403     private void _initDialect() {
404         if (!PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
405             return;
406         }
407 
408         Connection con = null;
409 
410         try {
411             con = DataAccess.getConnection();
412 
413             String url = con.getMetaData().getURL();
414 
415             int x = url.indexOf(StringPool.COLON);
416             int y = url.indexOf(StringPool.COLON, x + 1);
417 
418             String urlPrefix = url.substring(x + 1, y);
419 
420             String dialectClass = PropsUtil.get(
421                 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
422 
423             if (dialectClass != null) {
424                 if (_log.isDebugEnabled()) {
425                     _log.debug("JDBC class implementation " + dialectClass);
426                 }
427             }
428             else {
429                 if (_log.isDebugEnabled()) {
430                     _log.debug("JDBC class implementation is null");
431                 }
432             }
433 
434             if (dialectClass != null) {
435                 _dialect = (Dialect)Class.forName(dialectClass).newInstance();
436             }
437         }
438         catch (Exception e) {
439             _log.error(e);
440         }
441         finally{
442             DataAccess.cleanUp(con);
443         }
444 
445         if (_dialect == null) {
446             _log.error("No JDBC dialect found");
447         }
448     }
449 
450     private void _initIndexWriter() {
451         try {
452             _indexWriter = new IndexWriter(
453                 getLuceneDir(), LuceneHelperUtil.getAnalyzer(),
454                 IndexWriter.MaxFieldLength.LIMITED);
455 
456             _indexWriter.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
457             _indexWriter.setRAMBufferSizeMB(PropsValues.LUCENE_BUFFER_SIZE);
458         }
459         catch (Exception e) {
460             _log.error(
461                 "Initializing Lucene writer failed for " + _companyId, e);
462         }
463     }
464 
465     private void _manuallyCreateJdbcDirectory(
466         JdbcDirectory jdbcDirectory, String tableName) {
467 
468         // LEP-2181
469 
470         Connection con = null;
471         ResultSet rs = null;
472 
473         try {
474             con = DataAccess.getConnection();
475 
476             // Check if table exists
477 
478             DatabaseMetaData metaData = con.getMetaData();
479 
480             rs = metaData.getTables(null, null, tableName, null);
481 
482             if (!rs.next()) {
483                 JdbcTemplate jdbcTemplate = jdbcDirectory.getJdbcTemplate();
484 
485                 jdbcTemplate.executeUpdate(
486                     jdbcDirectory.getTable().sqlCreate());
487 
488                 Class<?> lockClass = jdbcDirectory.getSettings().getLockClass();
489 
490                 JdbcLock jdbcLock = null;
491 
492                 try {
493                     jdbcLock = (JdbcLock)lockClass.newInstance();
494                 }
495                 catch (Exception e) {
496                     throw new JdbcStoreException(
497                         "Could not create lock class " + lockClass);
498                 }
499 
500                 jdbcLock.initializeDatabase(jdbcDirectory);
501             }
502         }
503         catch (Exception e) {
504             if (_log.isWarnEnabled()) {
505                 _log.warn("Could not create " + tableName);
506             }
507         }
508         finally {
509             DataAccess.cleanUp(con, null, rs);
510         }
511     }
512 
513     private void _write(Term term, Document document) throws IOException {
514         try {
515             if (term != null) {
516                 _indexWriter.updateDocument(term, document);
517             }
518             else {
519                 _indexWriter.addDocument(document);
520             }
521 
522             _optimizeCount++;
523 
524             if ((PropsValues.LUCENE_OPTIMIZE_INTERVAL == 0) ||
525                 (_optimizeCount >= PropsValues.LUCENE_OPTIMIZE_INTERVAL)) {
526 
527                 _indexWriter.optimize();
528 
529                 _optimizeCount = 0;
530             }
531 
532             _batchCount++;
533         }
534         finally {
535             _commit();
536         }
537     }
538 
539     private static final String _LUCENE_STORE_TYPE_FILE = "file";
540 
541     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
542 
543     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
544 
545     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
546 
547     private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
548 
549     private int _batchCount;
550     private long _companyId;
551     private Dialect _dialect;
552     private IndexWriter _indexWriter;
553     private Map<String, Directory> _jdbcDirectories =
554         new ConcurrentHashMap<String, Directory>();
555     private int _optimizeCount;
556     private Map<String, Directory> _ramDirectories =
557         new ConcurrentHashMap<String, Directory>();
558 
559 }