1
22
23 package com.liferay.portal.search.lucene;
24
25 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26 import com.liferay.portal.kernel.log.Log;
27 import com.liferay.portal.kernel.log.LogFactoryUtil;
28 import com.liferay.portal.kernel.search.SearchEngineUtil;
29 import com.liferay.portal.kernel.util.FileUtil;
30 import com.liferay.portal.kernel.util.InfrastructureUtil;
31 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
32 import com.liferay.portal.kernel.util.StringPool;
33 import com.liferay.portal.kernel.util.StringUtil;
34 import com.liferay.portal.kernel.util.Validator;
35 import com.liferay.portal.util.PropsKeys;
36 import com.liferay.portal.util.PropsUtil;
37 import com.liferay.portal.util.PropsValues;
38 import com.liferay.util.lucene.KeywordsUtil;
39
40 import java.io.IOException;
41
42 import java.sql.Connection;
43 import java.sql.DatabaseMetaData;
44 import java.sql.ResultSet;
45 import java.sql.Statement;
46
47 import java.util.Date;
48 import java.util.Map;
49 import java.util.concurrent.ConcurrentHashMap;
50
51 import javax.sql.DataSource;
52
53 import org.apache.lucene.analysis.Analyzer;
54 import org.apache.lucene.analysis.WhitespaceAnalyzer;
55 import org.apache.lucene.document.Document;
56 import org.apache.lucene.index.IndexReader;
57 import org.apache.lucene.index.IndexWriter;
58 import org.apache.lucene.index.Term;
59 import org.apache.lucene.queryParser.ParseException;
60 import org.apache.lucene.queryParser.QueryParser;
61 import org.apache.lucene.search.BooleanClause;
62 import org.apache.lucene.search.BooleanQuery;
63 import org.apache.lucene.search.IndexSearcher;
64 import org.apache.lucene.search.Query;
65 import org.apache.lucene.search.TermQuery;
66 import org.apache.lucene.search.WildcardQuery;
67 import org.apache.lucene.store.Directory;
68 import org.apache.lucene.store.FSDirectory;
69 import org.apache.lucene.store.RAMDirectory;
70 import org.apache.lucene.store.jdbc.JdbcDirectory;
71 import org.apache.lucene.store.jdbc.JdbcStoreException;
72 import org.apache.lucene.store.jdbc.dialect.Dialect;
73 import org.apache.lucene.store.jdbc.lock.JdbcLock;
74 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
75
76
83 public class LuceneUtil {
84
85 public static void acquireLock(long companyId) {
86 try {
87 _instance._sharedWriter.acquireLock(companyId, true);
88 }
89 catch (InterruptedException ie) {
90 _log.error(ie);
91 }
92 }
93
94 public static void addDate(Document doc, String field, Date value) {
95 doc.add(LuceneFields.getDate(field, value));
96 }
97
98 public static void addExactTerm(
99 BooleanQuery booleanQuery, String field, boolean value) {
100
101 addExactTerm(booleanQuery, field, String.valueOf(value));
102 }
103
104 public static void addExactTerm(
105 BooleanQuery booleanQuery, String field, Boolean value) {
106
107 addExactTerm(booleanQuery, field, String.valueOf(value));
108 }
109
110 public static void addExactTerm(
111 BooleanQuery booleanQuery, String field, double value) {
112
113 addExactTerm(booleanQuery, field, String.valueOf(value));
114 }
115
116 public static void addExactTerm(
117 BooleanQuery booleanQuery, String field, Double value) {
118
119 addExactTerm(booleanQuery, field, String.valueOf(value));
120 }
121
122 public static void addExactTerm(
123 BooleanQuery booleanQuery, String field, int value) {
124
125 addExactTerm(booleanQuery, field, String.valueOf(value));
126 }
127
128 public static void addExactTerm(
129 BooleanQuery booleanQuery, String field, Integer value) {
130
131 addExactTerm(booleanQuery, field, String.valueOf(value));
132 }
133
134 public static void addExactTerm(
135 BooleanQuery booleanQuery, String field, long value) {
136
137 addExactTerm(booleanQuery, field, String.valueOf(value));
138 }
139
140 public static void addExactTerm(
141 BooleanQuery booleanQuery, String field, Long value) {
142
143 addExactTerm(booleanQuery, field, String.valueOf(value));
144 }
145
146 public static void addExactTerm(
147 BooleanQuery booleanQuery, String field, short value) {
148
149 addExactTerm(booleanQuery, field, String.valueOf(value));
150 }
151
152 public static void addExactTerm(
153 BooleanQuery booleanQuery, String field, Short value) {
154
155 addExactTerm(booleanQuery, field, String.valueOf(value));
156 }
157
158 public static void addExactTerm(
159 BooleanQuery booleanQuery, String field, String value) {
160
161
163 Query query = new TermQuery(new Term(field, value));
164
165 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
166 }
167
168 public static void addRequiredTerm(
169 BooleanQuery booleanQuery, String field, boolean value) {
170
171 addRequiredTerm(booleanQuery, field, String.valueOf(value));
172 }
173
174 public static void addRequiredTerm(
175 BooleanQuery booleanQuery, String field, Boolean value) {
176
177 addRequiredTerm(booleanQuery, field, String.valueOf(value));
178 }
179
180 public static void addRequiredTerm(
181 BooleanQuery booleanQuery, String field, double value) {
182
183 addRequiredTerm(booleanQuery, field, String.valueOf(value));
184 }
185
186 public static void addRequiredTerm(
187 BooleanQuery booleanQuery, String field, Double value) {
188
189 addRequiredTerm(booleanQuery, field, String.valueOf(value));
190 }
191
192 public static void addRequiredTerm(
193 BooleanQuery booleanQuery, String field, int value) {
194
195 addRequiredTerm(booleanQuery, field, String.valueOf(value));
196 }
197
198 public static void addRequiredTerm(
199 BooleanQuery booleanQuery, String field, Integer value) {
200
201 addRequiredTerm(booleanQuery, field, String.valueOf(value));
202 }
203
204 public static void addRequiredTerm(
205 BooleanQuery booleanQuery, String field, long value) {
206
207 addRequiredTerm(booleanQuery, field, String.valueOf(value));
208 }
209
210 public static void addRequiredTerm(
211 BooleanQuery booleanQuery, String field, Long value) {
212
213 addRequiredTerm(booleanQuery, field, String.valueOf(value));
214 }
215
216 public static void addRequiredTerm(
217 BooleanQuery booleanQuery, String field, short value) {
218
219 addRequiredTerm(booleanQuery, field, String.valueOf(value));
220 }
221
222 public static void addRequiredTerm(
223 BooleanQuery booleanQuery, String field, Short value) {
224
225 addRequiredTerm(booleanQuery, field, String.valueOf(value));
226 }
227
228 public static void addRequiredTerm(
229 BooleanQuery booleanQuery, String field, String value) {
230
231 addRequiredTerm(booleanQuery, field, value, false);
232 }
233
234 public static void addRequiredTerm(
235 BooleanQuery booleanQuery, String field, String value, boolean like) {
236
237 if (like) {
238 value = StringUtil.replace(
239 value, StringPool.PERCENT, StringPool.STAR);
240
241 value = value.toLowerCase();
242
243 WildcardQuery wildcardQuery = new WildcardQuery(
244 new Term(field, value));
245
246 booleanQuery.add(wildcardQuery, BooleanClause.Occur.MUST);
247 }
248 else {
249
251 Term term = new Term(field, value);
252 TermQuery termQuery = new TermQuery(term);
253
254 booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
255 }
256 }
257
258 public static void addTerm(
259 BooleanQuery booleanQuery, String field, long value)
260 throws ParseException {
261
262 addTerm(booleanQuery, field, String.valueOf(value));
263 }
264
265 public static void addTerm(
266 BooleanQuery booleanQuery, String field, String value)
267 throws ParseException {
268
269 addTerm(booleanQuery, field, value, false);
270 }
271
272 public static void addTerm(
273 BooleanQuery booleanQuery, String field, String value,
274 boolean like)
275 throws ParseException {
276
277 if (Validator.isNull(value)) {
278 return;
279 }
280
281 if (like) {
282 value = value.toLowerCase();
283
284 StringBuilder sb = new StringBuilder();
285
286 sb.append(value);
287 sb.append(StringPool.STAR);
288
289 WildcardQuery wildcardQuery = new WildcardQuery(
290 new Term(field, sb.toString()));
291
292 booleanQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD);
293 }
294 else {
295 QueryParser queryParser = new QueryParser(
296 field, LuceneUtil.getAnalyzer());
297
298 try {
299 Query query = queryParser.parse(value);
300
301 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
302 }
303 catch (ParseException pe) {
304 if (_log.isDebugEnabled()) {
305 _log.debug(
306 "ParseException thrown, reverting to literal search",
307 pe);
308 }
309
310 value = KeywordsUtil.escape(value);
311
312 Query query = queryParser.parse(value);
313
314 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
315 }
316 }
317 }
318
319 public static void checkLuceneDir(long companyId) {
320 if (SearchEngineUtil.isIndexReadOnly()) {
321 return;
322 }
323
324 Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
325
326 try {
327
328
330 if (luceneDir.fileExists("write.lock")) {
331 luceneDir.deleteFile("write.lock");
332 }
333 }
334 catch (IOException ioe) {
335 _log.error("Unable to clear write lock", ioe);
336 }
337
338 IndexWriter writer = null;
339
340
343 try {
344 if (luceneDir.fileExists("segments.gen")) {
345 writer = new IndexWriter(
346 luceneDir, LuceneUtil.getAnalyzer(), false);
347 }
348 else {
349 writer = new IndexWriter(
350 luceneDir, LuceneUtil.getAnalyzer(), true);
351 }
352 }
353 catch (IOException ioe) {
354 _log.error("Check Lucene directory failed for " + companyId, ioe);
355 }
356 finally {
357 if (writer != null) {
358 try {
359 writer.close();
360 }
361 catch (IOException ioe) {
362 _log.error(ioe);
363 }
364 }
365 }
366 }
367
368 public static void delete(long companyId) {
369 _instance._delete(companyId);
370 }
371
372 public static void deleteDocuments(long companyId, Term term)
373 throws IOException {
374
375 try {
376 _instance._sharedWriter.deleteDocuments(companyId, term);
377 }
378 catch (InterruptedException ie) {
379 _log.error(ie);
380 }
381 }
382
383 public static Analyzer getAnalyzer() {
384 return _instance._getAnalyzer();
385 }
386
387 public static FSDirectory getDirectory(String path, boolean create)
388 throws IOException {
389
390 return FSDirectory.getDirectory(path, false);
391 }
392
393 public static Directory getLuceneDir(long companyId) {
394 return _instance._getLuceneDir(companyId);
395 }
396
397 public static IndexReader getReader(long companyId) throws IOException {
398 return IndexReader.open(getLuceneDir(companyId));
399 }
400
401 public static IndexSearcher getSearcher(long companyId)
402 throws IOException {
403
404 return new IndexSearcher(getLuceneDir(companyId));
405 }
406
407 public static IndexWriter getWriter(long companyId) throws IOException {
408 return getWriter(companyId, false);
409 }
410
411 public static IndexWriter getWriter(long companyId, boolean create)
412 throws IOException {
413
414 return _instance._sharedWriter.getWriter(companyId, create);
415 }
416
417 public static void releaseLock(long companyId) {
418 _instance._sharedWriter.releaseLock(companyId);
419 }
420
421 public static void write(long companyId) {
422 _instance._sharedWriter.write(companyId);
423 }
424
425 public static void write(IndexWriter writer) throws IOException {
426 _instance._sharedWriter.write(writer);
427 }
428
429 private LuceneUtil() {
430 String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
431
432 if (Validator.isNotNull(analyzerName)) {
433 try {
434 _analyzerClass = Class.forName(analyzerName);
435 }
436 catch (Exception e) {
437 _log.error(e);
438 }
439 }
440
441
443 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
444 Connection con = null;
445
446 try {
447 con = DataAccess.getConnection();
448
449 String url = con.getMetaData().getURL();
450
451 int x = url.indexOf(":");
452 int y = url.indexOf(":", x + 1);
453
454 String urlPrefix = url.substring(x + 1, y);
455
456 String dialectClass = PropsUtil.get(
457 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
458
459 if (dialectClass != null) {
460 if (_log.isDebugEnabled()) {
461 _log.debug("JDBC class implementation " + dialectClass);
462 }
463 }
464 else {
465 if (_log.isDebugEnabled()) {
466 _log.debug("JDBC class implementation is null");
467 }
468 }
469
470 if (dialectClass != null) {
471 _dialect =
472 (Dialect)Class.forName(dialectClass).newInstance();
473 }
474 }
475 catch (Exception e) {
476 _log.error(e);
477 }
478 finally{
479 DataAccess.cleanUp(con);
480 }
481
482 if (_dialect == null) {
483 _log.error("No JDBC dialect found");
484 }
485 }
486 }
487
488 public void _delete(long companyId) {
489 if (SearchEngineUtil.isIndexReadOnly()) {
490 return;
491 }
492
493 if (_log.isDebugEnabled()) {
494 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
495 }
496
497 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
498 _deleteFile(companyId);
499 }
500 else if (PropsValues.LUCENE_STORE_TYPE.equals(
501 _LUCENE_STORE_TYPE_JDBC)) {
502
503 _deleteJdbc(companyId);
504 }
505 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
506 _deleteRam(companyId);
507 }
508 else {
509 throw new RuntimeException(
510 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
511 }
512 }
513
514 private void _deleteFile(long companyId) {
515 String path = _getPath(companyId);
516
517 try {
518 Directory directory = getDirectory(path, false);
519
520 directory.close();
521 }
522 catch (Exception e) {
523 if (_log.isWarnEnabled()) {
524 _log.warn("Could not close directory " + path);
525 }
526 }
527
528 FileUtil.deltree(path);
529 }
530
531 private void _deleteJdbc(long companyId) {
532 String tableName = _getTableName(companyId);
533
534 try {
535 Directory directory = _jdbcDirectories.remove(tableName);
536
537 if (directory != null) {
538 directory.close();
539 }
540 }
541 catch (Exception e) {
542 if (_log.isWarnEnabled()) {
543 _log.warn("Could not close directory " + tableName);
544 }
545 }
546
547 Connection con = null;
548 Statement s = null;
549
550 try {
551 con = DataAccess.getConnection();
552
553 s = con.createStatement();
554
555 s.executeUpdate("DELETE FROM " + tableName);
556 }
557 catch (Exception e) {
558 if (_log.isWarnEnabled()) {
559 _log.warn("Could not truncate " + tableName);
560 }
561 }
562 finally {
563 DataAccess.cleanUp(con, s);
564 }
565 }
566
567 private void _deleteRam(long companyId) {
568 }
569
570 private Analyzer _getAnalyzer() {
571 try {
572 return (Analyzer)_analyzerClass.newInstance();
573 }
574 catch (Exception e) {
575 throw new RuntimeException(e);
576 }
577 }
578
579 private Directory _getLuceneDir(long companyId) {
580 if (_log.isDebugEnabled()) {
581 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
582 }
583
584 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
585 return _getLuceneDirFile(companyId);
586 }
587 else if (PropsValues.LUCENE_STORE_TYPE.equals(
588 _LUCENE_STORE_TYPE_JDBC)) {
589
590 return _getLuceneDirJdbc(companyId);
591 }
592 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
593 return _getLuceneDirRam(companyId);
594 }
595 else {
596 throw new RuntimeException(
597 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
598 }
599 }
600
601 private Directory _getLuceneDirFile(long companyId) {
602 Directory directory = null;
603
604 String path = _getPath(companyId);
605
606 try {
607 directory = getDirectory(path, false);
608 }
609 catch (IOException ioe1) {
610 try {
611 if (directory != null) {
612 directory.close();
613 }
614
615 directory = getDirectory(path, true);
616 }
617 catch (IOException ioe2) {
618 throw new RuntimeException(ioe2);
619 }
620 }
621
622 return directory;
623 }
624
625 private Directory _getLuceneDirJdbc(long companyId) {
626 JdbcDirectory directory = null;
627
628 Thread currentThread = Thread.currentThread();
629
630 ClassLoader contextClassLoader = currentThread.getContextClassLoader();
631
632 try {
633 currentThread.setContextClassLoader(
634 PortalClassLoaderUtil.getClassLoader());
635
636 String tableName = _getTableName(companyId);
637
638 directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
639
640 if (directory != null) {
641 return directory;
642 }
643
644 try {
645 DataSource ds = InfrastructureUtil.getDataSource();
646
647 directory = new JdbcDirectory(ds, _dialect, tableName);
648
649 _jdbcDirectories.put(tableName, directory);
650
651 if (!directory.tableExists()) {
652 directory.create();
653 }
654 }
655 catch (IOException ioe) {
656 throw new RuntimeException(ioe);
657 }
658 catch (UnsupportedOperationException uoe) {
659 if (_log.isWarnEnabled()) {
660 _log.warn(
661 "Database doesn't support the ability to check " +
662 "whether a table exists");
663 }
664
665 _manuallyCreateJdbcDirectory(directory, tableName);
666 }
667 }
668 finally {
669 currentThread.setContextClassLoader(contextClassLoader);
670 }
671
672 return directory;
673 }
674
675 private Directory _getLuceneDirRam(long companyId) {
676 String path = _getPath(companyId);
677
678 Directory directory = _ramDirectories.get(path);
679
680 if (directory == null) {
681 directory = new RAMDirectory();
682
683 _ramDirectories.put(path, directory);
684 }
685
686 return directory;
687 }
688
689 private String _getPath(long companyId) {
690 StringBuilder sb = new StringBuilder();
691
692 sb.append(PropsValues.LUCENE_DIR);
693 sb.append(companyId);
694 sb.append(StringPool.SLASH);
695
696 return sb.toString();
697 }
698
699 private String _getTableName(long companyId) {
700 return _LUCENE_TABLE_PREFIX + companyId;
701 }
702
703 private void _manuallyCreateJdbcDirectory(
704 JdbcDirectory directory, String tableName) {
705
706
708 Connection con = null;
709 ResultSet rs = null;
710
711 try {
712 con = DataAccess.getConnection();
713
714
716 DatabaseMetaData metaData = con.getMetaData();
717
718 rs = metaData.getTables(null, null, tableName, null);
719
720 if (!rs.next()) {
721 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
722
723 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
724
725 Class<?> lockClass = directory.getSettings().getLockClass();
726
727 JdbcLock jdbcLock = null;
728
729 try {
730 jdbcLock = (JdbcLock)lockClass.newInstance();
731 }
732 catch (Exception e) {
733 throw new JdbcStoreException(
734 "Failed to create lock class " + lockClass);
735 }
736
737 jdbcLock.initializeDatabase(directory);
738 }
739 }
740 catch (Exception e) {
741 if (_log.isWarnEnabled()) {
742 _log.warn("Could not create " + tableName);
743 }
744 }
745 finally {
746 DataAccess.cleanUp(con, null, rs);
747 }
748 }
749
750 private static final String _LUCENE_STORE_TYPE_FILE = "file";
751
752 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
753
754 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
755
756 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
757
758 private static Log _log = LogFactoryUtil.getLog(LuceneUtil.class);
759
760 private static LuceneUtil _instance = new LuceneUtil();
761
762 private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
763 private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
764 private Dialect _dialect;
765 private Map<String, Directory> _jdbcDirectories =
766 new ConcurrentHashMap<String, Directory>();
767 private Map<String, Directory> _ramDirectories =
768 new ConcurrentHashMap<String, Directory>();
769
770 }