1   /**
2    * Copyright (c) 2000-2009 Liferay, Inc. All rights reserved.
3    *
4    * The contents of this file are subject to the terms of the Liferay Enterprise
5    * Subscription License ("License"). You may not use this file except in
6    * compliance with the License. You can obtain a copy of the License by
7    * contacting Liferay, Inc. See the License for the specific language governing
8    * permissions and limitations under the License, including but not limited to
9    * distribution rights of the Software.
10   *
11   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
12   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
13   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
14   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
15   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
16   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
17   * SOFTWARE.
18   */
19  
20  package com.liferay.portal.search.lucene;
21  
22  import com.liferay.portal.kernel.dao.jdbc.DataAccess;
23  import com.liferay.portal.kernel.log.Log;
24  import com.liferay.portal.kernel.log.LogFactoryUtil;
25  import com.liferay.portal.kernel.util.FileUtil;
26  import com.liferay.portal.kernel.util.InfrastructureUtil;
27  import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
28  import com.liferay.portal.kernel.util.StringPool;
29  import com.liferay.portal.kernel.util.StringUtil;
30  import com.liferay.portal.kernel.util.Validator;
31  import com.liferay.portal.util.PropsKeys;
32  import com.liferay.portal.util.PropsUtil;
33  import com.liferay.portal.util.PropsValues;
34  import com.liferay.util.lucene.KeywordsUtil;
35  
36  import java.io.IOException;
37  
38  import java.sql.Connection;
39  import java.sql.DatabaseMetaData;
40  import java.sql.ResultSet;
41  import java.sql.Statement;
42  
43  import java.util.Date;
44  import java.util.Map;
45  import java.util.concurrent.ConcurrentHashMap;
46  
47  import javax.sql.DataSource;
48  
49  import org.apache.lucene.analysis.Analyzer;
50  import org.apache.lucene.analysis.WhitespaceAnalyzer;
51  import org.apache.lucene.document.Document;
52  import org.apache.lucene.index.IndexReader;
53  import org.apache.lucene.index.IndexWriter;
54  import org.apache.lucene.index.Term;
55  import org.apache.lucene.queryParser.ParseException;
56  import org.apache.lucene.queryParser.QueryParser;
57  import org.apache.lucene.search.BooleanClause;
58  import org.apache.lucene.search.BooleanQuery;
59  import org.apache.lucene.search.IndexSearcher;
60  import org.apache.lucene.search.Query;
61  import org.apache.lucene.search.TermQuery;
62  import org.apache.lucene.search.WildcardQuery;
63  import org.apache.lucene.store.Directory;
64  import org.apache.lucene.store.FSDirectory;
65  import org.apache.lucene.store.RAMDirectory;
66  import org.apache.lucene.store.jdbc.JdbcDirectory;
67  import org.apache.lucene.store.jdbc.JdbcStoreException;
68  import org.apache.lucene.store.jdbc.dialect.Dialect;
69  import org.apache.lucene.store.jdbc.lock.JdbcLock;
70  import org.apache.lucene.store.jdbc.support.JdbcTemplate;
71  
72  /**
73   * <a href="LuceneUtil.java.html"><b><i>View Source</i></b></a>
74   *
75   * @author Brian Wing Shun Chan
76   * @author Harry Mark
77   *
78   */
79  public class LuceneUtil {
80  
81      public static void acquireLock(long companyId) {
82          try {
83              _instance._sharedWriter.acquireLock(companyId, true);
84          }
85          catch (InterruptedException ie) {
86              _log.error(ie);
87          }
88      }
89  
90      public static void addDate(Document doc, String field, Date value) {
91          doc.add(LuceneFields.getDate(field, value));
92      }
93  
94      public static void addExactTerm(
95          BooleanQuery booleanQuery, String field, boolean value) {
96  
97          addExactTerm(booleanQuery, field, String.valueOf(value));
98      }
99  
100     public static void addExactTerm(
101         BooleanQuery booleanQuery, String field, Boolean value) {
102 
103         addExactTerm(booleanQuery, field, String.valueOf(value));
104     }
105 
106     public static void addExactTerm(
107         BooleanQuery booleanQuery, String field, double value) {
108 
109         addExactTerm(booleanQuery, field, String.valueOf(value));
110     }
111 
112     public static void addExactTerm(
113         BooleanQuery booleanQuery, String field, Double value) {
114 
115         addExactTerm(booleanQuery, field, String.valueOf(value));
116     }
117 
118     public static void addExactTerm(
119         BooleanQuery booleanQuery, String field, int value) {
120 
121         addExactTerm(booleanQuery, field, String.valueOf(value));
122     }
123 
124     public static void addExactTerm(
125         BooleanQuery booleanQuery, String field, Integer value) {
126 
127         addExactTerm(booleanQuery, field, String.valueOf(value));
128     }
129 
130     public static void addExactTerm(
131         BooleanQuery booleanQuery, String field, long value) {
132 
133         addExactTerm(booleanQuery, field, String.valueOf(value));
134     }
135 
136     public static void addExactTerm(
137         BooleanQuery booleanQuery, String field, Long value) {
138 
139         addExactTerm(booleanQuery, field, String.valueOf(value));
140     }
141 
142     public static void addExactTerm(
143         BooleanQuery booleanQuery, String field, short value) {
144 
145         addExactTerm(booleanQuery, field, String.valueOf(value));
146     }
147 
148     public static void addExactTerm(
149         BooleanQuery booleanQuery, String field, Short value) {
150 
151         addExactTerm(booleanQuery, field, String.valueOf(value));
152     }
153 
154     public static void addExactTerm(
155         BooleanQuery booleanQuery, String field, String value) {
156 
157         //text = KeywordsUtil.escape(value);
158 
159         Query query = new TermQuery(new Term(field, value));
160 
161         booleanQuery.add(query, BooleanClause.Occur.SHOULD);
162     }
163 
164     public static void addRequiredTerm(
165         BooleanQuery booleanQuery, String field, boolean value) {
166 
167         addRequiredTerm(booleanQuery, field, String.valueOf(value));
168     }
169 
170     public static void addRequiredTerm(
171         BooleanQuery booleanQuery, String field, Boolean value) {
172 
173         addRequiredTerm(booleanQuery, field, String.valueOf(value));
174     }
175 
176     public static void addRequiredTerm(
177         BooleanQuery booleanQuery, String field, double value) {
178 
179         addRequiredTerm(booleanQuery, field, String.valueOf(value));
180     }
181 
182     public static void addRequiredTerm(
183         BooleanQuery booleanQuery, String field, Double value) {
184 
185         addRequiredTerm(booleanQuery, field, String.valueOf(value));
186     }
187 
188     public static void addRequiredTerm(
189         BooleanQuery booleanQuery, String field, int value) {
190 
191         addRequiredTerm(booleanQuery, field, String.valueOf(value));
192     }
193 
194     public static void addRequiredTerm(
195         BooleanQuery booleanQuery, String field, Integer value) {
196 
197         addRequiredTerm(booleanQuery, field, String.valueOf(value));
198     }
199 
200     public static void addRequiredTerm(
201         BooleanQuery booleanQuery, String field, long value) {
202 
203         addRequiredTerm(booleanQuery, field, String.valueOf(value));
204     }
205 
206     public static void addRequiredTerm(
207         BooleanQuery booleanQuery, String field, Long value) {
208 
209         addRequiredTerm(booleanQuery, field, String.valueOf(value));
210     }
211 
212     public static void addRequiredTerm(
213         BooleanQuery booleanQuery, String field, short value) {
214 
215         addRequiredTerm(booleanQuery, field, String.valueOf(value));
216     }
217 
218     public static void addRequiredTerm(
219         BooleanQuery booleanQuery, String field, Short value) {
220 
221         addRequiredTerm(booleanQuery, field, String.valueOf(value));
222     }
223 
224     public static void addRequiredTerm(
225         BooleanQuery booleanQuery, String field, String value) {
226 
227         addRequiredTerm(booleanQuery, field, value, false);
228     }
229 
230     public static void addRequiredTerm(
231         BooleanQuery booleanQuery, String field, String value, boolean like) {
232 
233         if (like) {
234             value = StringUtil.replace(
235                 value, StringPool.PERCENT, StringPool.STAR);
236 
237             value = value.toLowerCase();
238 
239             WildcardQuery wildcardQuery = new WildcardQuery(
240                 new Term(field, value));
241 
242             booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
243         }
244         else {
245             //text = KeywordsUtil.escape(value);
246 
247             Term term = new Term(field, value);
248             TermQuery termQuery = new TermQuery(term);
249 
250             booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
251         }
252     }
253 
254     public static void addTerm(
255             BooleanQuery booleanQuery, String field, long value)
256         throws ParseException {
257 
258         addTerm(booleanQuery, field, String.valueOf(value));
259     }
260 
261     public static void addTerm(
262             BooleanQuery booleanQuery, String field, String value)
263         throws ParseException {
264 
265         addTerm(booleanQuery, field, value, false);
266     }
267 
268     public static void addTerm(
269             BooleanQuery booleanQuery, String field, String value,
270             boolean like)
271         throws ParseException {
272 
273         if (Validator.isNull(value)) {
274             return;
275         }
276 
277         if (like) {
278             value = value.toLowerCase();
279 
280             StringBuilder sb = new StringBuilder();
281 
282             sb.append(StringPool.STAR);
283             sb.append(value);
284             sb.append(StringPool.STAR);
285 
286             WildcardQuery wildcardQuery = new WildcardQuery(
287                 new Term(field, sb.toString()));
288 
289             booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
290         }
291         else {
292             QueryParser queryParser = new QueryParser(
293                 field, LuceneUtil.getAnalyzer());
294 
295             try {
296                 Query query = queryParser.parse(value);
297 
298                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
299             }
300             catch (ParseException pe) {
301                 if (_log.isDebugEnabled()) {
302                     _log.debug(
303                         "ParseException thrown, reverting to literal search",
304                         pe);
305                 }
306 
307                 value = KeywordsUtil.escape(value);
308 
309                 Query query = queryParser.parse(value);
310 
311                 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
312             }
313         }
314     }
315 
316     public static void checkLuceneDir(long companyId) {
317         if (PropsValues.INDEX_READ_ONLY) {
318             return;
319         }
320 
321         Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
322 
323         try {
324 
325             // LEP-6078
326 
327             if (luceneDir.fileExists("write.lock")) {
328                 luceneDir.deleteFile("write.lock");
329             }
330         }
331         catch (IOException ioe) {
332             _log.error("Unable to clear write lock", ioe);
333         }
334 
335         IndexWriter writer = null;
336 
337         // Lucene does not properly release its lock on the index when
338         // IndexWriter throws an exception
339 
340         try {
341             if (luceneDir.fileExists("segments.gen")) {
342                 writer = new IndexWriter(
343                     luceneDir, LuceneUtil.getAnalyzer(), false);
344             }
345             else {
346                 writer = new IndexWriter(
347                     luceneDir, LuceneUtil.getAnalyzer(), true);
348             }
349         }
350         catch (IOException ioe) {
351             _log.error("Check Lucene directory failed for " + companyId, ioe);
352         }
353         finally {
354             if (writer != null) {
355                 try {
356                     writer.close();
357                 }
358                 catch (IOException ioe) {
359                     _log.error(ioe);
360                 }
361             }
362         }
363     }
364 
365     public static void delete(long companyId) {
366         _instance._delete(companyId);
367     }
368 
369     public static void deleteDocuments(long companyId, Term term)
370         throws IOException {
371 
372         try {
373             _instance._sharedWriter.deleteDocuments(companyId, term);
374         }
375         catch (InterruptedException ie) {
376             _log.error(ie);
377         }
378     }
379 
380     public static Analyzer getAnalyzer() {
381         return _instance._getAnalyzer();
382     }
383 
384     public static Directory getLuceneDir(long companyId) {
385         return _instance._getLuceneDir(companyId);
386     }
387 
388     public static IndexReader getReader(long companyId) throws IOException {
389         return IndexReader.open(getLuceneDir(companyId));
390     }
391 
392     public static IndexSearcher getSearcher(long companyId)
393         throws IOException {
394 
395         return new IndexSearcher(getLuceneDir(companyId));
396     }
397 
398     public static IndexWriter getWriter(long companyId) throws IOException {
399         return getWriter(companyId, false);
400     }
401 
402     public static IndexWriter getWriter(long companyId, boolean create)
403         throws IOException {
404 
405         return _instance._sharedWriter.getWriter(companyId, create);
406     }
407 
408     public static void releaseLock(long companyId) {
409         _instance._sharedWriter.releaseLock(companyId);
410     }
411 
412     public static void write(long companyId) {
413         _instance._sharedWriter.write(companyId);
414     }
415 
416     public static void write(IndexWriter writer) throws IOException {
417         _instance._sharedWriter.write(writer);
418     }
419 
420     private LuceneUtil() {
421         String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
422 
423         if (Validator.isNotNull(analyzerName)) {
424             try {
425                 _analyzerClass = Class.forName(analyzerName);
426             }
427             catch (Exception e) {
428                 _log.error(e);
429             }
430         }
431 
432         // Dialect
433 
434         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
435             Connection con = null;
436 
437             try {
438                 con = DataAccess.getConnection();
439 
440                 String url = con.getMetaData().getURL();
441 
442                 int x = url.indexOf(":");
443                 int y = url.indexOf(":", x + 1);
444 
445                 String urlPrefix = url.substring(x + 1, y);
446 
447                 String dialectClass = PropsUtil.get(
448                     PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
449 
450                 if (dialectClass != null) {
451                     if (_log.isDebugEnabled()) {
452                         _log.debug("JDBC class implementation " + dialectClass);
453                     }
454                 }
455                 else {
456                     if (_log.isDebugEnabled()) {
457                         _log.debug("JDBC class implementation is null");
458                     }
459                 }
460 
461                 if (dialectClass != null) {
462                     _dialect =
463                         (Dialect)Class.forName(dialectClass).newInstance();
464                 }
465             }
466             catch (Exception e) {
467                 _log.error(e);
468             }
469             finally{
470                 DataAccess.cleanUp(con);
471             }
472 
473             if (_dialect == null) {
474                 _log.error("No JDBC dialect found");
475             }
476         }
477     }
478 
479     public void _delete(long companyId) {
480         if (PropsValues.INDEX_READ_ONLY) {
481             return;
482         }
483 
484         if (_log.isDebugEnabled()) {
485             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
486         }
487 
488         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
489             _deleteFile(companyId);
490         }
491         else if (PropsValues.LUCENE_STORE_TYPE.equals(
492                     _LUCENE_STORE_TYPE_JDBC)) {
493 
494             _deleteJdbc(companyId);
495         }
496         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
497             _deleteRam(companyId);
498         }
499         else {
500             throw new RuntimeException(
501                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
502         }
503     }
504 
505     private void _deleteFile(long companyId) {
506         String path = _getPath(companyId);
507 
508         try {
509             Directory directory = FSDirectory.getDirectory(path, false);
510 
511             directory.close();
512         }
513         catch (Exception e) {
514             if (_log.isWarnEnabled()) {
515                 _log.warn("Could not close directory " + path);
516             }
517         }
518 
519         FileUtil.deltree(path);
520     }
521 
522     private void _deleteJdbc(long companyId) {
523         String tableName = _getTableName(companyId);
524 
525         try {
526             Directory directory = _jdbcDirectories.remove(tableName);
527 
528             if (directory != null) {
529                 directory.close();
530             }
531         }
532         catch (Exception e) {
533             if (_log.isWarnEnabled()) {
534                 _log.warn("Could not close directory " + tableName);
535             }
536         }
537 
538         Connection con = null;
539         Statement s = null;
540 
541         try {
542             con = DataAccess.getConnection();
543 
544             s = con.createStatement();
545 
546             s.executeUpdate("DELETE FROM " + tableName);
547         }
548         catch (Exception e) {
549             if (_log.isWarnEnabled()) {
550                 _log.warn("Could not truncate " + tableName);
551             }
552         }
553         finally {
554             DataAccess.cleanUp(con, s);
555         }
556     }
557 
558     private void _deleteRam(long companyId) {
559     }
560 
561     private Analyzer _getAnalyzer() {
562         try {
563             return (Analyzer)_analyzerClass.newInstance();
564         }
565         catch (Exception e) {
566             throw new RuntimeException(e);
567         }
568     }
569 
570     private Directory _getLuceneDir(long companyId) {
571         if (_log.isDebugEnabled()) {
572             _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
573         }
574 
575         if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
576             return _getLuceneDirFile(companyId);
577         }
578         else if (PropsValues.LUCENE_STORE_TYPE.equals(
579                     _LUCENE_STORE_TYPE_JDBC)) {
580 
581             return _getLuceneDirJdbc(companyId);
582         }
583         else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
584             return _getLuceneDirRam(companyId);
585         }
586         else {
587             throw new RuntimeException(
588                 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
589         }
590     }
591 
592     private Directory _getLuceneDirFile(long companyId) {
593         Directory directory = null;
594 
595         String path = _getPath(companyId);
596 
597         try {
598             directory = FSDirectory.getDirectory(path, false);
599         }
600         catch (IOException ioe1) {
601             try {
602                 if (directory != null) {
603                     directory.close();
604                 }
605 
606                 directory = FSDirectory.getDirectory(path, true);
607             }
608             catch (IOException ioe2) {
609                 throw new RuntimeException(ioe2);
610             }
611         }
612 
613         return directory;
614     }
615 
616     private Directory _getLuceneDirJdbc(long companyId) {
617         JdbcDirectory directory = null;
618 
619         Thread currentThread = Thread.currentThread();
620 
621         ClassLoader contextClassLoader = currentThread.getContextClassLoader();
622 
623         try {
624             currentThread.setContextClassLoader(
625                 PortalClassLoaderUtil.getClassLoader());
626 
627             String tableName = _getTableName(companyId);
628 
629             directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
630 
631             if (directory != null) {
632                 return directory;
633             }
634 
635             try {
636                 DataSource ds = InfrastructureUtil.getDataSource();
637 
638                 directory = new JdbcDirectory(ds, _dialect, tableName);
639 
640                 _jdbcDirectories.put(tableName, directory);
641 
642                 if (!directory.tableExists()) {
643                     directory.create();
644                 }
645             }
646             catch (IOException ioe) {
647                 throw new RuntimeException(ioe);
648             }
649             catch (UnsupportedOperationException uoe) {
650                 if (_log.isWarnEnabled()) {
651                     _log.warn(
652                         "Database doesn't support the ability to check " +
653                             "whether a table exists");
654                 }
655 
656                 _manuallyCreateJdbcDirectory(directory, tableName);
657             }
658         }
659         finally {
660             currentThread.setContextClassLoader(contextClassLoader);
661         }
662 
663         return directory;
664     }
665 
666     private Directory _getLuceneDirRam(long companyId) {
667         String path = _getPath(companyId);
668 
669         Directory directory = _ramDirectories.get(path);
670 
671         if (directory == null) {
672             directory = new RAMDirectory();
673 
674             _ramDirectories.put(path, directory);
675         }
676 
677         return directory;
678     }
679 
680     private String _getPath(long companyId) {
681         StringBuilder sb = new StringBuilder();
682 
683         sb.append(PropsValues.LUCENE_DIR);
684         sb.append(companyId);
685         sb.append(StringPool.SLASH);
686 
687         return sb.toString();
688     }
689 
690     private String _getTableName(long companyId) {
691         return _LUCENE_TABLE_PREFIX + companyId;
692     }
693 
694     private void _manuallyCreateJdbcDirectory(
695         JdbcDirectory directory, String tableName) {
696 
697         // LEP-2181
698 
699         Connection con = null;
700         ResultSet rs = null;
701 
702         try {
703             con = DataAccess.getConnection();
704 
705             // Check if table exists
706 
707             DatabaseMetaData metaData = con.getMetaData();
708 
709             rs = metaData.getTables(null, null, tableName, null);
710 
711             if (!rs.next()) {
712                 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
713 
714                 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
715 
716                 Class<?> lockClass = directory.getSettings().getLockClass();
717 
718                 JdbcLock jdbcLock = null;
719 
720                 try {
721                     jdbcLock = (JdbcLock)lockClass.newInstance();
722                 }
723                 catch (Exception e) {
724                     throw new JdbcStoreException(
725                         "Failed to create lock class " + lockClass);
726                 }
727 
728                 jdbcLock.initializeDatabase(directory);
729             }
730         }
731         catch (Exception e) {
732             if (_log.isWarnEnabled()) {
733                 _log.warn("Could not create " + tableName);
734             }
735         }
736         finally {
737             DataAccess.cleanUp(con, null, rs);
738         }
739     }
740 
741     private static final String _LUCENE_STORE_TYPE_FILE = "file";
742 
743     private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
744 
745     private static final String _LUCENE_STORE_TYPE_RAM = "ram";
746 
747     private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
748 
749     private static Log _log = LogFactoryUtil.getLog(LuceneUtil.class);
750 
751     private static LuceneUtil _instance = new LuceneUtil();
752 
753     private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
754     private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
755     private Dialect _dialect;
756     private Map<String, Directory> _jdbcDirectories =
757         new ConcurrentHashMap<String, Directory>();
758     private Map<String, Directory> _ramDirectories =
759         new ConcurrentHashMap<String, Directory>();
760 
761 }