1
22
23 package com.liferay.portal.search.lucene;
24
25 import com.liferay.portal.kernel.dao.jdbc.DataAccess;
26 import com.liferay.portal.kernel.log.Log;
27 import com.liferay.portal.kernel.log.LogFactoryUtil;
28 import com.liferay.portal.kernel.search.SearchEngineUtil;
29 import com.liferay.portal.kernel.util.FileUtil;
30 import com.liferay.portal.kernel.util.InfrastructureUtil;
31 import com.liferay.portal.kernel.util.PortalClassLoaderUtil;
32 import com.liferay.portal.kernel.util.PropsKeys;
33 import com.liferay.portal.kernel.util.StringPool;
34 import com.liferay.portal.util.PropsUtil;
35 import com.liferay.portal.util.PropsValues;
36
37 import java.io.File;
38 import java.io.IOException;
39
40 import java.sql.Connection;
41 import java.sql.DatabaseMetaData;
42 import java.sql.ResultSet;
43 import java.sql.Statement;
44
45 import java.util.Map;
46 import java.util.concurrent.ConcurrentHashMap;
47 import java.util.concurrent.Executors;
48 import java.util.concurrent.ScheduledExecutorService;
49 import java.util.concurrent.TimeUnit;
50
51 import javax.sql.DataSource;
52
53 import org.apache.lucene.document.Document;
54 import org.apache.lucene.index.IndexWriter;
55 import org.apache.lucene.index.Term;
56 import org.apache.lucene.store.Directory;
57 import org.apache.lucene.store.FSDirectory;
58 import org.apache.lucene.store.RAMDirectory;
59 import org.apache.lucene.store.jdbc.JdbcDirectory;
60 import org.apache.lucene.store.jdbc.JdbcStoreException;
61 import org.apache.lucene.store.jdbc.dialect.Dialect;
62 import org.apache.lucene.store.jdbc.lock.JdbcLock;
63 import org.apache.lucene.store.jdbc.support.JdbcTemplate;
64
65
72 public class IndexAccessorImpl implements IndexAccessor {
73
74 public IndexAccessorImpl(long companyId) {
75 _companyId = companyId;
76
77 _checkLuceneDir();
78 _initIndexWriter();
79 _initCommitScheduler();
80 _initDialect();
81 }
82
83 public void addDocument(Document document) throws IOException {
84 if (SearchEngineUtil.isIndexReadOnly()) {
85 return;
86 }
87
88 _write(null, document);
89 }
90
91 public void close() {
92 try {
93 _indexWriter.close();
94 }
95 catch(Exception e) {
96 _log.error(
97 "Closing Lucene writer failed for " + _companyId, e);
98 }
99 }
100
101 public void delete() {
102 if (SearchEngineUtil.isIndexReadOnly()) {
103 return;
104 }
105
106 close();
107
108 if (_log.isDebugEnabled()) {
109 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
110 }
111
112 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
113 _deleteFile();
114 }
115 else if (PropsValues.LUCENE_STORE_TYPE.equals(
116 _LUCENE_STORE_TYPE_JDBC)) {
117
118 _deleteJdbc();
119 }
120 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
121 _deleteRam();
122 }
123 else {
124 throw new RuntimeException(
125 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
126 }
127
128 _initIndexWriter();
129 }
130
131 public void deleteDocuments(Term term) throws IOException {
132 if (SearchEngineUtil.isIndexReadOnly()) {
133 return;
134 }
135
136 try {
137 _indexWriter.deleteDocuments(term);
138
139 _batchCount++;
140 }
141 finally {
142 _commit();
143 }
144 }
145
146 public long getCompanyId() {
147 return _companyId;
148 }
149
150 public Directory getLuceneDir() {
151 if (_log.isDebugEnabled()) {
152 _log.debug("Lucene store type " + PropsValues.LUCENE_STORE_TYPE);
153 }
154
155 if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_FILE)) {
156 return _getLuceneDirFile();
157 }
158 else if (PropsValues.LUCENE_STORE_TYPE.equals(
159 _LUCENE_STORE_TYPE_JDBC)) {
160
161 return _getLuceneDirJdbc();
162 }
163 else if (PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_RAM)) {
164 return _getLuceneDirRam();
165 }
166 else {
167 throw new RuntimeException(
168 "Invalid store type " + PropsValues.LUCENE_STORE_TYPE);
169 }
170 }
171
172 public void updateDocument(Term term, Document document)
173 throws IOException {
174
175 if (SearchEngineUtil.isIndexReadOnly()) {
176 return;
177 }
178
179 _write(term, document);
180 }
181
182 private void _checkLuceneDir() {
183 if (SearchEngineUtil.isIndexReadOnly()) {
184 return;
185 }
186
187 try {
188 Directory directory = getLuceneDir();
189
190 if (IndexWriter.isLocked(directory)) {
191 IndexWriter.unlock(directory);
192 }
193 }
194 catch (Exception e) {
195 _log.error("Check Lucene directory failed for " + _companyId, e);
196 }
197 }
198
199 private void _commit() throws IOException {
200 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE == 0) ||
201 (PropsValues.LUCENE_COMMIT_BATCH_SIZE <= _batchCount)) {
202
203 _doCommit();
204 }
205 }
206
207 private void _deleteFile() {
208 String path = _getPath();
209
210 try {
211 Directory directory = _getDirectory(path);
212
213 directory.close();
214 }
215 catch (Exception e) {
216 if (_log.isWarnEnabled()) {
217 _log.warn("Could not close directory " + path);
218 }
219 }
220
221 FileUtil.deltree(path);
222 }
223
224 private void _deleteJdbc() {
225 String tableName = _getTableName();
226
227 try {
228 Directory directory = _jdbcDirectories.remove(tableName);
229
230 if (directory != null) {
231 directory.close();
232 }
233 }
234 catch (Exception e) {
235 if (_log.isWarnEnabled()) {
236 _log.warn("Could not close directory " + tableName);
237 }
238 }
239
240 Connection con = null;
241 Statement s = null;
242
243 try {
244 con = DataAccess.getConnection();
245
246 s = con.createStatement();
247
248 s.executeUpdate("DELETE FROM " + tableName);
249 }
250 catch (Exception e) {
251 if (_log.isWarnEnabled()) {
252 _log.warn("Could not truncate " + tableName);
253 }
254 }
255 finally {
256 DataAccess.cleanUp(con, s);
257 }
258 }
259
260 private void _deleteRam() {
261 }
262
263 private void _doCommit() throws IOException {
264 if (_indexWriter != null) {
265 _indexWriter.commit();
266 }
267
268 _batchCount = 0;
269 }
270
271 private FSDirectory _getDirectory(String path) throws IOException {
272 return FSDirectory.open(new File(path));
273 }
274
275 private Directory _getLuceneDirFile() {
276 Directory directory = null;
277
278 String path = _getPath();
279
280 try {
281 directory = _getDirectory(path);
282 }
283 catch (IOException ioe1) {
284 if (directory != null) {
285 try {
286 directory.close();
287 }
288 catch (Exception e) {
289 }
290 }
291 }
292
293 return directory;
294 }
295
296 private Directory _getLuceneDirJdbc() {
297 JdbcDirectory jdbcDirectory = null;
298
299 Thread currentThread = Thread.currentThread();
300
301 ClassLoader contextClassLoader = currentThread.getContextClassLoader();
302
303 try {
304 currentThread.setContextClassLoader(
305 PortalClassLoaderUtil.getClassLoader());
306
307 String tableName = _getTableName();
308
309 jdbcDirectory = (JdbcDirectory)_jdbcDirectories.get(tableName);
310
311 if (jdbcDirectory != null) {
312 return jdbcDirectory;
313 }
314
315 try {
316 DataSource dataSource = InfrastructureUtil.getDataSource();
317
318 jdbcDirectory = new JdbcDirectory(
319 dataSource, _dialect, tableName);
320
321 _jdbcDirectories.put(tableName, jdbcDirectory);
322
323 if (!jdbcDirectory.tableExists()) {
324 jdbcDirectory.create();
325 }
326 }
327 catch (IOException ioe) {
328 throw new RuntimeException(ioe);
329 }
330 catch (UnsupportedOperationException uoe) {
331 if (_log.isWarnEnabled()) {
332 _log.warn(
333 "Database doesn't support the ability to check " +
334 "whether a table exists");
335 }
336
337 _manuallyCreateJdbcDirectory(jdbcDirectory, tableName);
338 }
339 }
340 finally {
341 currentThread.setContextClassLoader(contextClassLoader);
342 }
343
344 return jdbcDirectory;
345 }
346
347 private Directory _getLuceneDirRam() {
348 String path = _getPath();
349
350 Directory directory = _ramDirectories.get(path);
351
352 if (directory == null) {
353 directory = new RAMDirectory();
354
355 _ramDirectories.put(path, directory);
356 }
357
358 return directory;
359 }
360
361 private String _getPath() {
362 StringBuilder sb = new StringBuilder();
363
364 sb.append(PropsValues.LUCENE_DIR);
365 sb.append(_companyId);
366 sb.append(StringPool.SLASH);
367
368 return sb.toString();
369 }
370
371 private String _getTableName() {
372 return _LUCENE_TABLE_PREFIX + _companyId;
373 }
374
375 private void _initCommitScheduler() {
376 if ((PropsValues.LUCENE_COMMIT_BATCH_SIZE <= 0) ||
377 (PropsValues.LUCENE_COMMIT_TIME_INTERVAL <= 0)) {
378
379 return;
380 }
381
382 ScheduledExecutorService scheduledExecutorService =
383 Executors.newSingleThreadScheduledExecutor();
384
385 Runnable runnable = new Runnable() {
386
387 public void run() {
388 try {
389 _doCommit();
390 }
391 catch (IOException ioe) {
392 _log.error("Could not run scheduled commit", ioe);
393 }
394 }
395
396 };
397
398 scheduledExecutorService.scheduleWithFixedDelay(
399 runnable, 0, PropsValues.LUCENE_COMMIT_TIME_INTERVAL,
400 TimeUnit.MILLISECONDS);
401 }
402
403 private void _initDialect() {
404 if (!PropsValues.LUCENE_STORE_TYPE.equals(_LUCENE_STORE_TYPE_JDBC)) {
405 return;
406 }
407
408 Connection con = null;
409
410 try {
411 con = DataAccess.getConnection();
412
413 String url = con.getMetaData().getURL();
414
415 int x = url.indexOf(StringPool.COLON);
416 int y = url.indexOf(StringPool.COLON, x + 1);
417
418 String urlPrefix = url.substring(x + 1, y);
419
420 String dialectClass = PropsUtil.get(
421 PropsKeys.LUCENE_STORE_JDBC_DIALECT + urlPrefix);
422
423 if (dialectClass != null) {
424 if (_log.isDebugEnabled()) {
425 _log.debug("JDBC class implementation " + dialectClass);
426 }
427 }
428 else {
429 if (_log.isDebugEnabled()) {
430 _log.debug("JDBC class implementation is null");
431 }
432 }
433
434 if (dialectClass != null) {
435 _dialect = (Dialect)Class.forName(dialectClass).newInstance();
436 }
437 }
438 catch (Exception e) {
439 _log.error(e);
440 }
441 finally{
442 DataAccess.cleanUp(con);
443 }
444
445 if (_dialect == null) {
446 _log.error("No JDBC dialect found");
447 }
448 }
449
450 private void _initIndexWriter() {
451 try {
452 _indexWriter = new IndexWriter(
453 getLuceneDir(), LuceneHelperUtil.getAnalyzer(),
454 IndexWriter.MaxFieldLength.LIMITED);
455
456 _indexWriter.setMergeFactor(PropsValues.LUCENE_MERGE_FACTOR);
457 _indexWriter.setRAMBufferSizeMB(PropsValues.LUCENE_BUFFER_SIZE);
458 }
459 catch (Exception e) {
460 _log.error(
461 "Initializing Lucene writer failed for " + _companyId, e);
462 }
463 }
464
465 private void _manuallyCreateJdbcDirectory(
466 JdbcDirectory jdbcDirectory, String tableName) {
467
468
470 Connection con = null;
471 ResultSet rs = null;
472
473 try {
474 con = DataAccess.getConnection();
475
476
478 DatabaseMetaData metaData = con.getMetaData();
479
480 rs = metaData.getTables(null, null, tableName, null);
481
482 if (!rs.next()) {
483 JdbcTemplate jdbcTemplate = jdbcDirectory.getJdbcTemplate();
484
485 jdbcTemplate.executeUpdate(
486 jdbcDirectory.getTable().sqlCreate());
487
488 Class<?> lockClass = jdbcDirectory.getSettings().getLockClass();
489
490 JdbcLock jdbcLock = null;
491
492 try {
493 jdbcLock = (JdbcLock)lockClass.newInstance();
494 }
495 catch (Exception e) {
496 throw new JdbcStoreException(
497 "Could not create lock class " + lockClass);
498 }
499
500 jdbcLock.initializeDatabase(jdbcDirectory);
501 }
502 }
503 catch (Exception e) {
504 if (_log.isWarnEnabled()) {
505 _log.warn("Could not create " + tableName);
506 }
507 }
508 finally {
509 DataAccess.cleanUp(con, null, rs);
510 }
511 }
512
513 private void _write(Term term, Document document) throws IOException {
514 try {
515 if (term != null) {
516 _indexWriter.updateDocument(term, document);
517 }
518 else {
519 _indexWriter.addDocument(document);
520 }
521
522 _optimizeCount++;
523
524 if ((PropsValues.LUCENE_OPTIMIZE_INTERVAL == 0) ||
525 (_optimizeCount >= PropsValues.LUCENE_OPTIMIZE_INTERVAL)) {
526
527 _indexWriter.optimize();
528
529 _optimizeCount = 0;
530 }
531
532 _batchCount++;
533 }
534 finally {
535 _commit();
536 }
537 }
538
539 private static final String _LUCENE_STORE_TYPE_FILE = "file";
540
541 private static final String _LUCENE_STORE_TYPE_JDBC = "jdbc";
542
543 private static final String _LUCENE_STORE_TYPE_RAM = "ram";
544
545 private static final String _LUCENE_TABLE_PREFIX = "LUCENE_";
546
547 private static Log _log = LogFactoryUtil.getLog(IndexAccessorImpl.class);
548
549 private int _batchCount;
550 private long _companyId;
551 private Dialect _dialect;
552 private IndexWriter _indexWriter;
553 private Map<String, Directory> _jdbcDirectories =
554 new ConcurrentHashMap<String, Directory>();
555 private int _optimizeCount;
556 private Map<String, Directory> _ramDirectories =
557 new ConcurrentHashMap<String, Directory>();
558
559 }