1
22
23 package com.liferay.portal.search.lucene;
24
25 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26 import com.liferay.portal.kernel.search.Field;
27 import com.liferay.portal.kernel.util.FileUtil;
28 import com.liferay.portal.kernel.util.InfrastructureUtil;
29 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
30 import com.liferay.portal.kernel.util.StringPool;
31 import com.liferay.portal.kernel.util.Validator;
32 import com.liferay.portal.util.PropsKeys;
33 import com.liferay.portal.util.PropsUtil;
34 import com.liferay.portal.util.PropsValues;
35 import com.liferay.util.lucene.KeywordsUtil;
36
37 import java.io.IOException;
38
39 import java.sql.Connection;
40 import java.sql.DatabaseMetaData;
41 import java.sql.ResultSet;
42 import java.sql.Statement;
43
44 import java.util.Date;
45 import java.util.Map;
46 import java.util.concurrent.ConcurrentHashMap;
47
48 import javax.sql.DataSource;
49
50 import org.apache.commons.logging.Log;
51 import org.apache.commons.logging.LogFactory;
52 import org.apache.lucene.analysis.Analyzer;
53 import org.apache.lucene.analysis.WhitespaceAnalyzer;
54 import org.apache.lucene.document.Document;
55 import org.apache.lucene.index.IndexReader;
56 import org.apache.lucene.index.IndexWriter;
57 import org.apache.lucene.index.Term;
58 import org.apache.lucene.queryParser.ParseException;
59 import org.apache.lucene.queryParser.QueryParser;
60 import org.apache.lucene.search.BooleanClause;
61 import org.apache.lucene.search.BooleanQuery;
62 import org.apache.lucene.search.IndexSearcher;
63 import org.apache.lucene.search.Query;
64 import org.apache.lucene.search.TermQuery;
65 import org.apache.lucene.store.Directory;
66 import org.apache.lucene.store.FSDirectory;
67 import org.apache.lucene.store.RAMDirectory;
68 import org.apache.lucene.store.jdbc.JdbcDirectory;
69 import org.apache.lucene.store.jdbc.JdbcStoreException;
70 import org.apache.lucene.store.jdbc.dialect.Dialect;
71 import org.apache.lucene.store.jdbc.lock.JdbcLock;
72 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
73
74
81 public class LuceneUtil {
82
83 public static void acquireLock(long companyId) {
84 try {
85 _instance._sharedWriter.acquireLock(companyId, true);
86 }
87 catch (InterruptedException ie) {
88 _log.error(ie);
89 }
90 }
91
92 public static void addDate(Document doc, String field, Date value) {
93 doc.add(LuceneFields.getDate(field, value));
94 }
95
96 public static void addExactTerm(
97 BooleanQuery booleanQuery, String field, long value) {
98
99 addExactTerm(booleanQuery, field, String.valueOf(value));
100 }
101
102 public static void addExactTerm(
103 BooleanQuery booleanQuery, String field, String value) {
104
105
107 Query query = new TermQuery(new Term(field, value));
108
109 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
110 }
111
112 public static void addKeyword(Document doc, String field, double value) {
113 addKeyword(doc, field, String.valueOf(value));
114 }
115
116 public static void addKeyword(Document doc, String field, long value) {
117 addKeyword(doc, field, String.valueOf(value));
118 }
119
120 public static void addKeyword(Document doc, String field, String value) {
121 if (Validator.isNotNull(value)) {
122 doc.add(LuceneFields.getKeyword(field, value));
123 }
124 }
125
126 public static void addKeyword(Document doc, String field, String[] values) {
127 if (values == null) {
128 return;
129 }
130
131 for (int i = 0; i < values.length; i++) {
132 addKeyword(doc, field, values[i]);
133 }
134 }
135
136 public static void addRequiredTerm(
137 BooleanQuery booleanQuery, String field, long value) {
138
139 addRequiredTerm(booleanQuery, field, String.valueOf(value));
140 }
141
142 public static void addRequiredTerm(
143 BooleanQuery booleanQuery, String field, String value) {
144
145
147 Term term = new Term(field, value);
148 TermQuery termQuery = new TermQuery(term);
149
150 booleanQuery.add(termQuery, BooleanClause.Occur.MUST);
151 }
152
153 public static void addModifiedDate(Document doc) {
154 doc.add(LuceneFields.getDate(Field.MODIFIED));
155 }
156
157 public static void addTerm(
158 BooleanQuery booleanQuery, String field, long value)
159 throws ParseException {
160
161 addTerm(booleanQuery, field, String.valueOf(value));
162 }
163
164 public static void addTerm(
165 BooleanQuery booleanQuery, String field, String value)
166 throws ParseException {
167
168 if (Validator.isNotNull(value)) {
169 QueryParser queryParser = new QueryParser(
170 field, LuceneUtil.getAnalyzer());
171
172 try {
173 Query query = queryParser.parse(value);
174
175 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
176 }
177 catch (ParseException pe) {
178 if (_log.isDebugEnabled()) {
179 _log.debug(
180 "ParseException thrown, reverting to literal search",
181 pe);
182 }
183
184 value = KeywordsUtil.escape(value);
185
186 Query query = queryParser.parse(value);
187
188 booleanQuery.add(query, BooleanClause.Occur.SHOULD);
189 }
190 }
191 }
192
193 public static void addText(Document doc, String field, long value) {
194 addText(doc, field, String.valueOf(value));
195 }
196
197 public static void addText(Document doc, String field, String value) {
198 if (Validator.isNotNull(value)) {
199 doc.add(LuceneFields.getText(field, value));
200 }
201 }
202
203 public static void checkLuceneDir(long companyId) {
204 if (PropsValues.INDEX_READ_ONLY) {
205 return;
206 }
207
208 Directory luceneDir = LuceneUtil.getLuceneDir(companyId);
209
210 try {
211
212
214 if (luceneDir.fileExists("write.lock")) {
215 luceneDir.deleteFile("write.lock");
216 }
217 }
218 catch (IOException ioe) {
219 _log.error("Unable to clear write lock", ioe);
220 }
221
222 IndexWriter writer = null;
223
224
227 try {
228 if (luceneDir.fileExists("segments.gen")) {
229 writer = new IndexWriter(
230 luceneDir, LuceneUtil.getAnalyzer(), false);
231 }
232 else {
233 writer = new IndexWriter(
234 luceneDir, LuceneUtil.getAnalyzer(), true);
235 }
236 }
237 catch (IOException ioe) {
238 _log.error("Check Lucene directory failed for " + companyId, ioe);
239 }
240 finally {
241 if (writer != null) {
242 try {
243 writer.close();
244 }
245 catch (IOException ioe) {
246 _log.error(ioe);
247 }
248 }
249 }
250 }
251
252 public static void delete(long companyId) {
253 _instance._delete(companyId);
254 }
255
256 public static void deleteDocuments(long companyId, Term term)
257 throws IOException {
258
259 try {
260 _instance._sharedWriter.deleteDocuments(companyId, term);
261 }
262 catch (InterruptedException ie) {
263 _log.error(ie);
264 }
265 }
266
267 public static Analyzer getAnalyzer() {
268 return _instance._getAnalyzer();
269 }
270
271 public static Directory getLuceneDir(long companyId) {
272 return _instance._getLuceneDir(companyId);
273 }
274
275 public static IndexReader getReader(long companyId) throws IOException {
276 return IndexReader.open(getLuceneDir(companyId));
277 }
278
279 public static IndexSearcher getSearcher(long companyId)
280 throws IOException {
281
282 return new IndexSearcher(getLuceneDir(companyId));
283 }
284
285 public static IndexWriter getWriter(long companyId) throws IOException {
286 return getWriter(companyId, false);
287 }
288
289 public static IndexWriter getWriter(long companyId, boolean create)
290 throws IOException {
291
292 return _instance._sharedWriter.getWriter(companyId, create);
293 }
294
295 public static void releaseLock(long companyId) {
296 _instance._sharedWriter.releaseLock(companyId);
297 }
298
299 public static void write(long companyId) {
300 _instance._sharedWriter.write(companyId);
301 }
302
303 public static void write(IndexWriter writer) throws IOException {
304 _instance._sharedWriter.write(writer);
305 }
306
307 private LuceneUtil() {
308 String analyzerName = PropsUtil.get(PropsKeys.LUCENE_ANALYZER);
309
310 if (Validator.isNotNull(analyzerName)) {
311 try {
312 _analyzerClass = Class.forName(analyzerName);
313 }
314 catch (Exception e) {
315 _log.error(e);
316 }
317 }
318
319
321 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
322 Connection con = null;
323
324 try {
325 con = DataAccess.getConnection();
326
327 String url = con.getMetaData().getURL();
328
329 int x = url.indexOf(":");
330 int y = url.indexOf(":", x + 1);
331
332 String urlPrefix = url.substring(x + 1, y);
333
334 String dialectClass = PropsUtil.get(
335 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
336
337 if (dialectClass != null) {
338 if (_log.isDebugEnabled()) {
339 _log.debug("JDBC class implementation " + dialectClass);
340 }
341 }
342 else {
343 if (_log.isDebugEnabled()) {
344 _log.debug("JDBC class implementation is null");
345 }
346 }
347
348 if (dialectClass != null) {
349 _dialect =
350 (Dialect)Class.forName(dialectClass).newInstance();
351 }
352 }
353 catch (Exception e) {
354 _log.error(e);
355 }
356 finally{
357 DataAccess.cleanUp(con);
358 }
359
360 if (_dialect == null) {
361 _log.error("No JDBC dialect found");
362 }
363 }
364 }
365
366 public void _delete(long companyId) {
367 if (PropsValues.INDEX_READ_ONLY) {
368 return;
369 }
370
371 if (_log.isDebugEnabled()) {
372 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
373 }
374
375 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
376 _deleteFile(companyId);
377 }
378 else if (PropsValues.LUCENE_STORE_TYPE.equals(
379 _LUCENE_STORE_TYPE_JDBC)) {
380
381 _deleteJdbc(companyId);
382 }
383 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
384 _deleteRam(companyId);
385 }
386 else {
387 throw new RuntimeException(
388 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
389 }
390 }
391
392 private void _deleteFile(long companyId) {
393 String path = _getPath(companyId);
394
395 try {
396 Directory directory = FSDirectory.getDirectory(path, false);
397
398 directory.close();
399 }
400 catch (Exception e) {
401 if (_log.isWarnEnabled()) {
402 _log.warn("Could not close directory " + path);
403 }
404 }
405
406 FileUtil.deltree(path);
407 }
408
409 private void _deleteJdbc(long companyId) {
410 String tableName = _getTableName(companyId);
411
412 try {
413 Directory directory = _jdbcDirectories.remove(tableName);
414
415 if (directory != null) {
416 directory.close();
417 }
418 }
419 catch (Exception e) {
420 if (_log.isWarnEnabled()) {
421 _log.warn("Could not close directory " + tableName);
422 }
423 }
424
425 Connection con = null;
426 Statement s = null;
427
428 try {
429 con = DataAccess.getConnection();
430
431 s = con.createStatement();
432
433 s.executeUpdate("DELETE FROM " + tableName);
434 }
435 catch (Exception e) {
436 if (_log.isWarnEnabled()) {
437 _log.warn("Could not truncate " + tableName);
438 }
439 }
440 finally {
441 DataAccess.cleanUp(con, s);
442 }
443 }
444
445 private void _deleteRam(long companyId) {
446 }
447
448 private Analyzer _getAnalyzer() {
449 try {
450 return (Analyzer)_analyzerClass.newInstance();
451 }
452 catch (Exception e) {
453 throw new RuntimeException(e);
454 }
455 }
456
457 private Directory _getLuceneDir(long companyId) {
458 if (_log.isDebugEnabled()) {
459 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
460 }
461
462 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
463 return _getLuceneDirFile(companyId);
464 }
465 else if (PropsValues.LUCENE_STORE_TYPE.equals(
466 _LUCENE_STORE_TYPE_JDBC)) {
467
468 return _getLuceneDirJdbc(companyId);
469 }
470 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
471 return _getLuceneDirRam(companyId);
472 }
473 else {
474 throw new RuntimeException(
475 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
476 }
477 }
478
479 private Directory _getLuceneDirFile(long companyId) {
480 Directory directory = null;
481
482 String path = _getPath(companyId);
483
484 try {
485 directory = FSDirectory.getDirectory(path, false);
486 }
487 catch (IOException ioe1) {
488 try {
489 if (directory != null) {
490 directory.close();
491 }
492
493 directory = FSDirectory.getDirectory(path, true);
494 }
495 catch (IOException ioe2) {
496 throw new RuntimeException(ioe2);
497 }
498 }
499
500 return directory;
501 }
502
503 private Directory _getLuceneDirJdbc(long companyId) {
504 JdbcDirectory directory = null;
505
506 ClassLoader contextClassLoader =
507 Thread.currentThread().getContextClassLoader();
508
509 try {
510 Thread.currentThread().setContextClassLoader(
511 PortalClassLoaderUtil.getClassLoader());
512
513 String tableName = _getTableName(companyId);
514
515 directory = (JdbcDirectory)_jdbcDirectories.get(tableName);
516
517 if (directory != null) {
518 return directory;
519 }
520
521 try {
522 DataSource ds = InfrastructureUtil.getDataSource();
523
524 directory = new JdbcDirectory(ds, _dialect, tableName);
525
526 _jdbcDirectories.put(tableName, directory);
527
528 if (!directory.tableExists()) {
529 directory.create();
530 }
531 }
532 catch (IOException ioe) {
533 throw new RuntimeException(ioe);
534 }
535 catch (UnsupportedOperationException uoe) {
536 if (_log.isWarnEnabled()) {
537 _log.warn(
538 "Database doesn't support the ability to check " +
539 "whether a table exists");
540 }
541
542 _manuallyCreateJdbcDirectory(directory, tableName);
543 }
544 }
545 finally {
546 Thread.currentThread().setContextClassLoader(contextClassLoader);
547 }
548
549 return directory;
550 }
551
552 private Directory _getLuceneDirRam(long companyId) {
553 String path = _getPath(companyId);
554
555 Directory directory = _ramDirectories.get(path);
556
557 if (directory == null) {
558 directory = new RAMDirectory();
559
560 _ramDirectories.put(path, directory);
561 }
562
563 return directory;
564 }
565
566 private String _getPath(long companyId) {
567 StringBuilder sb = new StringBuilder();
568
569 sb.append(PropsValues.LUCENE_DIR);
570 sb.append(companyId);
571 sb.append(StringPool.SLASH);
572
573 return sb.toString();
574 }
575
576 private String _getTableName(long companyId) {
577 return _LUCENE_TABLE_PREFIX + companyId;
578 }
579
580 private void _manuallyCreateJdbcDirectory(
581 JdbcDirectory directory, String tableName) {
582
583
585 Connection con = null;
586 ResultSet rs = null;
587
588 try {
589 con = DataAccess.getConnection();
590
591
593 DatabaseMetaData metaData = con.getMetaData();
594
595 rs = metaData.getTables(null, null, tableName, null);
596
597 if (!rs.next()) {
598 JdbcTemplate jdbcTemplate = directory.getJdbcTemplate();
599
600 jdbcTemplate.executeUpdate(directory.getTable().sqlCreate());
601
602 Class<?> lockClass = directory.getSettings().getLockClass();
603
604 JdbcLock jdbcLock = null;
605
606 try {
607 jdbcLock = (JdbcLock)lockClass.newInstance();
608 }
609 catch (Exception e) {
610 throw new JdbcStoreException(
611 "Failed to create lock class " + lockClass);
612 }
613
614 jdbcLock.initializeDatabase(directory);
615 }
616 }
617 catch (Exception e) {
618 if (_log.isWarnEnabled()) {
619 _log.warn("Could not create " + tableName);
620 }
621 }
622 finally {
623 DataAccess.cleanUp(con, null, rs);
624 }
625 }
626
627 private static final String _LUCENE_STORE_TYPE_FILE = "file";
628
629 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
630
631 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
632
633 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
634
635 private static Log _log = LogFactory.getLog(LuceneUtil.class);
636
637 private static LuceneUtil _instance = new LuceneUtil();
638
639 private IndexWriterFactory _sharedWriter = new IndexWriterFactory();
640 private Class<?> _analyzerClass = WhitespaceAnalyzer.class;
641 private Dialect _dialect;
642 private Map<String, Directory> _jdbcDirectories =
643 new ConcurrentHashMap<String, Directory>();
644 private Map<String, Directory> _ramDirectories =
645 new ConcurrentHashMap<String, Directory>();
646
647 }