基於Lucene多索引進行索引和搜索

Lucene支持創建多個索引目錄，同時存儲多個索引。我們可能擔心的問題是，在索引的過程中，分散地存儲到多個索引目錄中，是否在搜索時能夠得到全局的相關度計算得分，其實Lucene的ParallelMultiSearcher和MultiSearcher支持全局得分的計算，也就是說，雖然索引分布在多個索引目錄中，在搜索的時候還會將全部的索引數據聚合在一起進行查詢匹配和得分計算。

索引目錄處理

下面我們通過將索引隨機地分布到以a~z的26個目錄中，並實現一個索引和搜索的程序，來驗證一下Lucene得分的計算。

首先，實現一個用來構建索引目錄以及處理搜索的工具類，代碼如下所示：

1. package org.shirdrn.lucene;
3. import java.io.File;
4. import java.io.IOException;
5. import java.util.ArrayList;
6. import java.util.HashMap;
7. import java.util.Iterator;
8. import java.util.List;
9. import java.util.Map;
10. import java.util.Random;
11. import java.util.concurrent.locks.Lock;
12. import java.util.concurrent.locks.ReentrantLock;
14. import org.apache.lucene.index.CorruptIndexException;
15. import org.apache.lucene.index.IndexWriter;
16. import org.apache.lucene.index.IndexWriterConfig;
17. import org.apache.lucene.index.IndexWriter.MaxFieldLength;
18. import org.apache.lucene.search.DefaultSimilarity;
19. import org.apache.lucene.search.IndexSearcher;
20. import org.apache.lucene.search.Searchable;
21. import org.apache.lucene.search.Similarity;
22. import org.apache.lucene.store.FSDirectory;
23. import org.apache.lucene.store.LockObtainFailedException;
24. import org.shirdrn.lucene.MultipleIndexing.IndexWriterObj;
26. /**
27. * Indexing accross multiple Lucene indexes.
28. *
29. * @author shirdrn
30. * @date 2011-12-12
31. */
32. public class IndexHelper {
34. private static WriterHelper writerHelper = null;
35. private static SearcherHelper searcherHelper = null;
37. public static WriterHelper newWriterHelper(String root, IndexWriterConfig indexConfig) {
38. return WriterHelper.newInstance(root, indexConfig);
39. }
41. public static SearcherHelper newSearcherHelper(String root, IndexWriterConfig indexConfig) {
42. return SearcherHelper.newInstance(root, indexConfig);
43. }
45. protected static class WriterHelper {
46. private String alphabet = "abcdefghijklmnopqrstuvwxyz";
47. private Lock locker = new ReentrantLock();
48. private String indexRootDir = null;
49. private IndexWriterConfig indexConfig;
50. private Map<Character, IndexWriterObj> indexWriters = new HashMap<Character, IndexWriterObj>();
51. private static Random random = new Random();
52. private WriterHelper() {
54. }
55. private synchronized static WriterHelper newInstance(String root, IndexWriterConfig indexConfig) {
56. if(writerHelper==null) {
57. writerHelper = new WriterHelper();
58. writerHelper.indexRootDir = root;
59. writerHelper.indexConfig = indexConfig;
60. }
61. return writerHelper;
62. }
63. public IndexWriterObj selectIndexWriter() {
64. int pos = random.nextInt(alphabet.length());
65. char ch = alphabet.charAt(pos);
66. String dir = new String(new char[] {ch});
67. locker.lock();
68. try {
69. File path = new File(indexRootDir, dir);
70. if(!path.exists()) {
71. path.mkdir();
72. }
73. if(!indexWriters.containsKey(ch)) {
74. IndexWriter indexWriter = new IndexWriter(FSDirectory.open(path), indexConfig.getAnalyzer(), MaxFieldLength.UNLIMITED);
75. indexWriters.put(ch, new IndexWriterObj(indexWriter, dir));
76. }
77. } catch (CorruptIndexException e) {
78. e.printStackTrace();
79. } catch (LockObtainFailedException e) {
80. e.printStackTrace();
81. } catch (IOException e) {
82. e.printStackTrace();
83. } finally {
84. locker.unlock();
85. }
86. return indexWriters.get(ch);
87. }
88. @SuppressWarnings("deprecation")
89. public void closeAll(boolean autoOptimize) {
90. Iterator<Map.Entry<Character, IndexWriterObj>> iter = indexWriters.entrySet().iterator();
91. while(iter.hasNext()) {
92. Map.Entry<Character, IndexWriterObj> entry = iter.next();
93. try {
94. if(autoOptimize) {
95. entry.getValue().indexWriter.optimize();
96. }
97. entry.getValue().indexWriter.close();
98. } catch (CorruptIndexException e) {
99. e.printStackTrace();
100. } catch (IOException e) {
101. e.printStackTrace();
102. }
103. }
104. }
105. }
107. protected static class SearcherHelper {
108. private List<IndexSearcher> searchers = new ArrayList<IndexSearcher>();
109. private Similarity similarity = new DefaultSimilarity();
110. private SearcherHelper() {
112. }
113. private synchronized static SearcherHelper newInstance(String root, IndexWriterConfig indexConfig) {
114. if(searcherHelper==null) {
115. searcherHelper = new SearcherHelper();
116. if(indexConfig.getSimilarity()!=null) {
117. searcherHelper.similarity = indexConfig.getSimilarity();
118. }
119. File indexRoot = new File(root);
120. File[] files = indexRoot.listFiles();
121. for(File f : files) {
122. IndexSearcher searcher = null;
123. try {
124. searcher = new IndexSearcher(FSDirectory.open(f));
125. } catch (CorruptIndexException e) {
126. e.printStackTrace();
127. } catch (IOException e) {
128. e.printStackTrace();
129. }
130. if(searcher!=null) {
131. searcher.setSimilarity(searcherHelper.similarity);
132. searcherHelper.searchers.add(searcher);
133. }
134. }
135. }
136. return searcherHelper;
137. }
138. public void closeAll() {
139. Iterator<IndexSearcher> iter = searchers.iterator();
140. while(iter.hasNext()) {
141. try {
142. iter.next().close();
143. } catch (IOException e) {
144. e.printStackTrace();
145. }
146. }
147. }
148. public Searchable[] getSearchers() {
149. Searchable[] a = new Searchable[searchers.size()];
150. return searchers.toArray(a);
151. }
152. }
153. }

由於在索引的時候，同時打開了多個Directory實例，而每個Directory對應一個IndexWriter，我們通過記錄a~z這26個字母為每個IndexWriter的名字，將IndexWriter和目錄名稱包裹在IndexWriterObj類的對象中，便於通過日志看到實際數據的分布。在進行Lucene Document構建的時候，將這個索引目錄的名字（a~z字符中之一）做成一個Field。在索引的時候，值需要調用IndexHelper.WriterHelper的selectIndexWriter()方法，即可以自動選擇對應的IndexWriter實例去進行索引。

在搜索的時候，通過IndexHelper.SearcherHelper工具來獲取多個Searchable實例的數組，調用getSearchers()即可以獲取到，提供給MultiSearcher構建搜索。