1   /**
2    * Copyright (c) 2000-2008 Liferay, Inc. All rights reserved.
3    *
4    * Permission is hereby granted, free of charge, to any person obtaining a copy
5    * of this software and associated documentation files (the "Software"), to deal
6    * in the Software without restriction, including without limitation the rights
7    * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8    * copies of the Software, and to permit persons to whom the Software is
9    * furnished to do so, subject to the following conditions:
10   *
11   * The above copyright notice and this permission notice shall be included in
12   * all copies or substantial portions of the Software.
13   *
14   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  
23  package com.liferay.portal.search.lucene;
24  
25  import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26  import com.liferay.portal.kernel.search.Field;
27  import com.liferay.portal.kernel.util.FileUtil;
28  import com.liferay.portal.kernel.util.InfrastructureUtil;
29  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
30  import com.liferay.portal.kernel.util.StringPool;
31  import com.liferay.portal.kernel.util.Validator;
32  import com.liferay.portal.util.PropsKeys;
33  import com.liferay.portal.util.PropsUtil;
34  import com.liferay.portal.util.PropsValues;
35  import com.liferay.util.lucene.KeywordsUtil;
36  
37  import java.io.IOException;
38  
39  import java.sql.Connection;
40  import java.sql.DatabaseMetaData;
41  import java.sql.ResultSet;
42  import java.sql.Statement;
43  
44  import java.util.Date;
45  import java.util.Map;
46  import java.util.concurrent.ConcurrentHashMap;
47  
48  import javax.sql.DataSource;
49  
50  import org.apache.commons.logging.Log;
51  import org.apache.commons.logging.LogFactory;
52  import org.apache.lucene.analysis.Analyzer;
53  import org.apache.lucene.analysis.WhitespaceAnalyzer;
54  import org.apache.lucene.document.Document;
55  import org.apache.lucene.index.IndexReader;
56  import org.apache.lucene.index.IndexWriter;
57  import org.apache.lucene.index.Term;
58  import org.apache.lucene.queryParser.ParseException;
59  import org.apache.lucene.queryParser.QueryParser;
60  import org.apache.lucene.search.BooleanClause;
61  import org.apache.lucene.search.BooleanQuery;
62  import org.apache.lucene.search.IndexSearcher;
63  import org.apache.lucene.search.Query;
64  import org.apache.lucene.search.TermQuery;
65  import org.apache.lucene.store.Directory;
66  import org.apache.lucene.store.FSDirectory;
67  import org.apache.lucene.store.RAMDirectory;
68  import org.apache.lucene.store.jdbc.JdbcDirectory;
69  import org.apache.lucene.store.jdbc.JdbcStoreException;
70  import org.apache.lucene.store.jdbc.dialect.Dialect;
71  import org.apache.lucene.store.jdbc.lock.JdbcLock;
72  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
73  
74  /**
75   * <a href="LuceneUtil.java.html"><b><i>View Source</i></b></a>
76   *
77   * @author Brian Wing Shun Chan
78   * @author Harry Mark
79   *
80   */
81  public class LuceneUtil {
82  
83      public static void acquireLock(long companyId) {
84          try {
85              _instance._sharedWriter.acquireLock(companyId, true);
86          }
87          catch (InterruptedException ie) {
88              _log.error(ie);
89          }
90      }
91  
92      public static void addDate(Document doc, String field, Date value) {
93          doc.add(LuceneFields.getDate(field, value));
94      }
95  
96      public static void addExactTerm(
97          BooleanQuery booleanQuery, String field, long value) {
98  
99          addExactTerm(booleanQuery, field, String.valueOf(value));
100     }
101 
102     public static void addExactTerm(
103         BooleanQuery booleanQuery, String field, String value) {
104 
105         //text = KeywordsUtil.escape(value);
106 
107         Query query = new TermQuery(new Term(field, value));
108 
109         booleanQuery.add(query, BooleanClause.Occur.SHOULD);
110     }
111 
112     public static void addKeyword(Document doc, String field, double value) {
113         addKeyword(doc, field, String.valueOf(value));
114     }
115 
116     public static void addKeyword(Document doc, String field, long value) {
117         addKeyword(doc, field, String.valueOf(value));
118     }
119 
120     public static void addKeyword(Document doc, String field, String value) {
121         if (Validator.isNotNull(value)) {
122             doc.add(LuceneFields.getKeyword(field, value));
123         }
124     }
125 
126     public static void addKeyword(Document doc, String field, String[] values) {
127         if (values == null) {
128             return;
129         }
130 
131         for (int i = 0; i < values.length; i++) {
132             addKeyword(doc, field, values[i]);
133         }
134     }
135 
136     public static void addRequiredTerm(
137         BooleanQuery booleanQuery, String field, long value) {
138 
139         addRequiredTerm(booleanQuery, field, String.valueOf(value));
140     }
141 
142     public static void addRequiredTerm(
143         BooleanQuery booleanQuery, String field, String value) {
144 
145         //text = KeywordsUtil.escape(value);
146 
147         Term term = new Term(field, value);
148         TermQuery termQuery = new TermQuery(term);
149 
150         booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
151     }
152 
153     public static void addModifiedDate(Document doc) {
154         doc.add(LuceneFields.getDate(Field.MODIFIED));
155     }
156 
157     public static void addTerm(
158             BooleanQuery booleanQuery, String field, long value)
159         throws ParseException {
160 
161         addTerm(booleanQuery, field, String.valueOf(value));
162     }
163 
164     public static void addTerm(
165             BooleanQuery booleanQuery, String field, String value)
166         throws ParseException {
167 
168         if (Validator.isNotNull(value)) {
169             QueryParser queryParser = new QueryParser(
170                 field, LuceneUtil.getAnalyzer());
171 
172             try {
173                 Query query = queryParser.parse(value);
174 
175                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
176             }
177             catch (ParseException pe) {
178                 if (_log.isDebugEnabled()) {
179                     _log.debug(
180                         "ParseException thrown, reverting to literal search",
181                         pe);
182                 }
183 
184                 value = KeywordsUtil.escape(value);
185 
186                 Query query = queryParser.parse(value);
187 
188                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
189             }
190         }
191     }
192 
193     public static void addText(Document doc, String field, long value) {
194         addText(doc, field, String.valueOf(value));
195     }
196 
197     public static void addText(Document doc, String field, String value) {
198         if (Validator.isNotNull(value)) {
199             doc.add(LuceneFields.getText(field, value));
200         }
201     }
202 
203     public static void checkLuceneDir(long companyId) {
204         if (PropsValues.INDEX_READ_ONLY) {
205             return;
206         }
207 
208         Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
209 
210         try {
211 
212             // LEP-6078
213 
214             if (luceneDir.fileExists("write.lock")) {
215                 luceneDir.deleteFile("write.lock");
216             }
217         }
218         catch (IOException ioe) {
219             _log.error("Unable to clear write lock", ioe);
220         }
221 
222         IndexWriter writer = null;
223 
224         // Lucene does not properly release its lock on the index when
225         // IndexWriter throws an exception
226 
227         try {
228             if (luceneDir.fileExists("segments.gen")) {
229                 writer = new IndexWriter(
230                     luceneDir, LuceneUtil.getAnalyzer(), false);
231             }
232             else {
233                 writer = new IndexWriter(
234                     luceneDir, LuceneUtil.getAnalyzer(), true);
235             }
236         }
237         catch (IOException ioe) {
238             _log.error("Check Lucene directory failed for " + companyId, ioe);
239         }
240         finally {
241             if (writer != null) {
242                 try {
243                     writer.close();
244                 }
245                 catch (IOException ioe) {
246                     _log.error(ioe);
247                 }
248             }
249         }
250     }
251 
252     public static void delete(long companyId) {
253         _instance._delete(companyId);
254     }
255 
256     public static void deleteDocuments(long companyId, Term term)
257         throws IOException {
258 
259         try {
260             _instance._sharedWriter.deleteDocuments(companyId, term);
261         }
262         catch (InterruptedException ie) {
263             _log.error(ie);
264         }
265     }
266 
267     public static Analyzer getAnalyzer() {
268         return _instance._getAnalyzer();
269     }
270 
271     public static Directory getLuceneDir(long companyId) {
272         return _instance._getLuceneDir(companyId);
273     }
274 
275     public static IndexReader getReader(long companyId) throws IOException {
276         return IndexReader.open(getLuceneDir(companyId));
277     }
278 
279     public static IndexSearcher getSearcher(long companyId)
280         throws IOException {
281 
282         return new IndexSearcher(getLuceneDir(companyId));
283     }
284 
285     public static IndexWriter getWriter(long companyId) throws IOException {
286         return getWriter(companyId, false);
287     }
288 
289     public static IndexWriter getWriter(long companyId, boolean create)
290         throws IOException {
291 
292         return _instance._sharedWriter.getWriter(companyId, create);
293     }
294 
295     public static void releaseLock(long companyId) {
296         _instance._sharedWriter.releaseLock(companyId);
297     }
298 
299     public static void write(long companyId) {
300         _instance._sharedWriter.write(companyId);
301     }
302 
303     public static void write(IndexWriter writer) throws IOException {
304         _instance._sharedWriter.write(writer);
305     }
306 
307     private LuceneUtil() {
308         String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
309 
310         if (Validator.isNotNull(analyzerName)) {
311             try {
312                 _analyzerClass = Class.forName(analyzerName);
313             }
314             catch (Exception e) {
315                 _log.error(e);
316             }
317         }
318 
319         // Dialect
320 
321         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
322             Connection con = null;
323 
324             try {
325                 con = DataAccess.getConnection();
326 
327                 String url = con.getMetaData().getURL();
328 
329                 int x = url.indexOf(":");
330                 int y = url.indexOf(":", x + 1);
331 
332                 String urlPrefix = url.substring(x + 1, y);
333 
334                 String dialectClass = PropsUtil.get(
335                     PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
336 
337                 if (dialectClass != null) {
338                     if (_log.isDebugEnabled()) {
339                         _log.debug("JDBC class implementation " + dialectClass);
340                     }
341                 }
342                 else {
343                     if (_log.isDebugEnabled()) {
344                         _log.debug("JDBC class implementation is null");
345                     }
346                 }
347 
348                 if (dialectClass != null) {
349                     _dialect =
350                         (Dialect)Class.forName(dialectClass).newInstance();
351                 }
352             }
353             catch (Exception e) {
354                 _log.error(e);
355             }
356             finally{
357                 DataAccess.cleanUp(con);
358             }
359 
360             if (_dialect == null) {
361                 _log.error("No JDBC dialect found");
362             }
363         }
364     }
365 
366     public void _delete(long companyId) {
367         if (PropsValues.INDEX_READ_ONLY) {
368             return;
369         }
370 
371         if (_log.isDebugEnabled()) {
372             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
373         }
374 
375         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
376             _deleteFile(companyId);
377         }
378         else if (PropsValues.LUCENE_STORE_TYPE.equals(
379                     _LUCENE_STORE_TYPE_JDBC)) {
380 
381             _deleteJdbc(companyId);
382         }
383         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
384             _deleteRam(companyId);
385         }
386         else {
387             throw new RuntimeException(
388                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
389         }
390     }
391 
392     private void _deleteFile(long companyId) {
393         String path = _getPath(companyId);
394 
395         try {
396             Directory directory = FSDirectory.getDirectory(path, false);
397 
398             directory.close();
399         }
400         catch (Exception e) {
401             if (_log.isWarnEnabled()) {
402                 _log.warn("Could not close directory " + path);
403             }
404         }
405 
406         FileUtil.deltree(path);
407     }
408 
409     private void _deleteJdbc(long companyId) {
410         String tableName = _getTableName(companyId);
411 
412         try {
413             Directory directory = _jdbcDirectories.remove(tableName);
414 
415             if (directory != null) {
416                 directory.close();
417             }
418         }
419         catch (Exception e) {
420             if (_log.isWarnEnabled()) {
421                 _log.warn("Could not close directory " + tableName);
422             }
423         }
424 
425         Connection con = null;
426         Statement s = null;
427 
428         try {
429             con = DataAccess.getConnection();
430 
431             s = con.createStatement();
432 
433             s.executeUpdate("DELETE FROM " + tableName);
434         }
435         catch (Exception e) {
436             if (_log.isWarnEnabled()) {
437                 _log.warn("Could not truncate " + tableName);
438             }
439         }
440         finally {
441             DataAccess.cleanUp(con, s);
442         }
443     }
444 
445     private void _deleteRam(long companyId) {
446     }
447 
448     private Analyzer _getAnalyzer() {
449         try {
450             return (Analyzer)_analyzerClass.newInstance();
451         }
452         catch (Exception e) {
453             throw new RuntimeException(e);
454         }
455     }
456 
457     private Directory _getLuceneDir(long companyId) {
458         if (_log.isDebugEnabled()) {
459             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
460         }
461 
462         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
463             return _getLuceneDirFile(companyId);
464         }
465         else if (PropsValues.LUCENE_STORE_TYPE.equals(
466                     _LUCENE_STORE_TYPE_JDBC)) {
467 
468             return _getLuceneDirJdbc(companyId);
469         }
470         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
471             return _getLuceneDirRam(companyId);
472         }
473         else {
474             throw new RuntimeException(
475                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
476         }
477     }
478 
479     private Directory _getLuceneDirFile(long companyId) {
480         Directory directory = null;
481 
482         String path = _getPath(companyId);
483 
484         try {
485             directory = FSDirectory.getDirectory(path, false);
486         }
487         catch (IOException ioe1) {
488             try {
489                 if (directory != null) {
490                     directory.close();
491                 }
492 
493                 directory = FSDirectory.getDirectory(path, true);
494             }
495             catch (IOException ioe2) {
496                 throw new RuntimeException(ioe2);
497             }
498         }
499 
500         return directory;
501     }
502 
503     private Directory _getLuceneDirJdbc(long companyId) {
504         JdbcDirectory directory = null;
505 
506         ClassLoader contextClassLoader =
507             Thread.currentThread().getContextClassLoader();
508 
509         try {
510             Thread.currentThread().setContextClassLoader(
511                 PortalClassLoaderUtil.getClassLoader());
512 
513             String tableName = _getTableName(companyId);
514 
515             directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
516 
517             if (directory != null) {
518                 return directory;
519             }
520 
521             try {
522                 DataSource ds = InfrastructureUtil.getDataSource();
523 
524                 directory = new JdbcDirectory(ds, _dialect, tableName);
525 
526                 _jdbcDirectories.put(tableName, directory);
527 
528                 if (!directory.tableExists()) {
529                     directory.create();
530                 }
531             }
532             catch (IOException ioe) {
533                 throw new RuntimeException(ioe);
534             }
535             catch (UnsupportedOperationException uoe) {
536                 if (_log.isWarnEnabled()) {
537                     _log.warn(
538                         "Database doesn't support the ability to check " +
539                             "whether a table exists");
540                 }
541 
542                 _manuallyCreateJdbcDirectory(directory, tableName);
543             }
544         }
545         finally {
546             Thread.currentThread().setContextClassLoader(contextClassLoader);
547         }
548 
549         return directory;
550     }
551 
552     private Directory _getLuceneDirRam(long companyId) {
553         String path = _getPath(companyId);
554 
555         Directory directory = _ramDirectories.get(path);
556 
557         if (directory == null) {
558             directory = new RAMDirectory();
559 
560             _ramDirectories.put(path, directory);
561         }
562 
563         return directory;
564     }
565 
566     private String _getPath(long companyId) {
567         StringBuilder sb = new StringBuilder();
568 
569         sb.append(PropsValues.LUCENE_DIR);
570         sb.append(companyId);
571         sb.append(StringPool.SLASH);
572 
573         return sb.toString();
574     }
575 
576     private String _getTableName(long companyId) {
577         return _LUCENE_TABLE_PREFIX + companyId;
578     }
579 
580     private void _manuallyCreateJdbcDirectory(
581         JdbcDirectory directory, String tableName) {
582 
583         // LEP-2181
584 
585         Connection con = null;
586         ResultSet rs = null;
587 
588         try {
589             con = DataAccess.getConnection();
590 
591             // Check if table exists
592 
593             DatabaseMetaData metaData = con.getMetaData();
594 
595             rs = metaData.getTables(null, null, tableName, null);
596 
597             if (!rs.next()) {
598                 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
599 
600                 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
601 
602                 Class<?> lockClass = directory.getSettings().getLockClass();
603 
604                 JdbcLock jdbcLock = null;
605 
606                 try {
607                     jdbcLock = (JdbcLock)lockClass.newInstance();
608                 }
609                 catch (Exception e) {
610                     throw new JdbcStoreException(
611                         "Failed to create lock class " + lockClass);
612                 }
613 
614                 jdbcLock.initializeDatabase(directory);
615             }
616         }
617         catch (Exception e) {
618             if (_log.isWarnEnabled()) {
619                 _log.warn("Could not create " + tableName);
620             }
621         }
622         finally {
623             DataAccess.cleanUp(con, null, rs);
624         }
625     }
626 
627     private static final String _LUCENE_STORE_TYPE_FILE = "file";
628 
629     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
630 
631     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
632 
633     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
634 
635     private static Log _log = LogFactory.getLog(LuceneUtil.class);
636 
637     private static LuceneUtil _instance = new LuceneUtil();
638 
639     private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
640     private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
641     private Dialect _dialect;
642     private Map<String, Directory> _jdbcDirectories =
643         new ConcurrentHashMap<String, Directory>();
644     private Map<String, Directory> _ramDirectories =
645         new ConcurrentHashMap<String, Directory>();
646 
647 }