diff --git a/build.xml b/build.xml index 1f98321..bd22200 100644 --- a/build.xml +++ b/build.xml @@ -38,7 +38,7 @@ Copy to ICS6 - + Copy to ICS6Extend Copy to AradonExtend diff --git a/publish/isearcher_4.10.jar b/publish/isearcher_4.10.jar index 639de75..0e50377 100644 Binary files a/publish/isearcher_4.10.jar and b/publish/isearcher_4.10.jar differ diff --git a/src/net/ion/nsearcher/common/WriteDocument.java b/src/net/ion/nsearcher/common/WriteDocument.java index cced0e6..e5e1f77 100644 --- a/src/net/ion/nsearcher/common/WriteDocument.java +++ b/src/net/ion/nsearcher/common/WriteDocument.java @@ -43,19 +43,27 @@ public class WriteDocument extends AbDocument { private IndexSession isession; private boolean newDoc = false; private float boost = 1.0f ; + private Document doc; public WriteDocument(IndexSession indexSession, String docId) { this.isession = indexSession ; this.docId = docId; + this.doc = new Document(); + } + + public WriteDocument(IndexSession indexSession, String docId, Document doc) { + this.isession = indexSession ; + this.docId = docId; + this.doc = (doc == null) ? new Document() : doc; } public WriteDocument(IndexSession indexSession) { this.isession = indexSession ; this.docId = new ObjectId().toString(); this.newDoc = true ; + this.doc = new Document(); } - public String idValue() { return docId; } @@ -72,7 +80,6 @@ public WriteDocument boost(float boost){ public Document toLuceneDoc() { FieldIndexingStrategy strategy = isession.fieldIndexingStrategy(); - Document doc = new Document(); StringBuilder bodyBuilder = new StringBuilder(512); bodyBuilder.append(docId + " ") ; @@ -194,6 +201,12 @@ public WriteDocument text(String fieldName, String value) { add(MyField.text(fieldName, value)); return this; } + + public WriteDocument stext(String fieldName, String value) { + add(MyField.manual(fieldName, value, Store.YES, true, MyFieldType.Text)); + return this; + } + public WriteDocument number(String fieldName, long value) { add(MyField.number(fieldName, value)); @@ -256,4 +269,5 @@ public Void insertVoid() throws IOException { return null ; } + } diff --git a/src/net/ion/nsearcher/index/IndexSession.java b/src/net/ion/nsearcher/index/IndexSession.java index 47eb03b..7c2b781 100644 --- a/src/net/ion/nsearcher/index/IndexSession.java +++ b/src/net/ion/nsearcher/index/IndexSession.java @@ -5,6 +5,7 @@ import net.ion.framework.util.MapUtil; import net.ion.nsearcher.common.AbDocument.Action; import net.ion.nsearcher.common.FieldIndexingStrategy; +import net.ion.nsearcher.common.MyField; import net.ion.nsearcher.common.SearchConstant; import net.ion.nsearcher.common.WriteDocument; import net.ion.nsearcher.search.SingleSearcher; @@ -16,6 +17,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; @@ -72,6 +74,14 @@ public WriteDocument newDocument(){ return new WriteDocument(this) ; } + + public WriteDocument loadDocument(String docId) throws IOException, ParseException { + Document findDoc = searcher.central().newSearcher().createRequestByKey(docId).findOne().toLuceneDoc() ; + WriteDocument result = new WriteDocument(this, docId, findDoc); + return result; + } + + public FieldIndexingStrategy fieldIndexingStrategy() { return fieldIndexingStrategy; } diff --git a/src/net/ion/nsearcher/search/AbstractDocCollector.java b/src/net/ion/nsearcher/search/AbstractDocCollector.java new file mode 100644 index 0000000..3c26614 --- /dev/null +++ b/src/net/ion/nsearcher/search/AbstractDocCollector.java @@ -0,0 +1,26 @@ +package net.ion.nsearcher.search; + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.DirectoryReader; + +import net.ion.nsearcher.common.ReadDocument; + +public abstract class AbstractDocCollector implements DocCollector { + public abstract ColResult accept(ReadDocument doc) ; + + public ColResult accept(DirectoryReader dreader, SearchRequest sreq, int docId) throws IOException{ + return accept(toDoc(dreader, sreq, docId)) ; + } + + private ReadDocument toDoc(DirectoryReader dreader, SearchRequest sreq, int docId) throws IOException { + Set fields = sreq.selectorField(); + if (fields == null || fields.size() == 0) { + return ReadDocument.loadDocument(dreader.document(docId)); + } + return ReadDocument.loadDocument(dreader.document(docId, sreq.selectorField())); + } + + +} diff --git a/src/net/ion/nsearcher/search/DocCollector.java b/src/net/ion/nsearcher/search/DocCollector.java new file mode 100644 index 0000000..5395c9b --- /dev/null +++ b/src/net/ion/nsearcher/search/DocCollector.java @@ -0,0 +1,20 @@ +package net.ion.nsearcher.search; + +import java.io.IOException; + +import org.apache.lucene.index.DirectoryReader; + +public interface DocCollector { + + public enum ColResult { + ACCEPT, REVOKE, BREAK + } + + public final static DocCollector BLANK = new DocCollector() { + public ColResult accept(DirectoryReader dreader, SearchRequest sreq, int docId) { + return ColResult.ACCEPT; + } + }; + + public ColResult accept(DirectoryReader dreader, SearchRequest sreq, int docId) throws IOException ; +} diff --git a/src/net/ion/nsearcher/search/PageCollector.java b/src/net/ion/nsearcher/search/PageCollector.java new file mode 100644 index 0000000..d12b964 --- /dev/null +++ b/src/net/ion/nsearcher/search/PageCollector.java @@ -0,0 +1,32 @@ +package net.ion.nsearcher.search; + +import java.io.IOException; + +import net.ion.framework.db.Page; + +import org.apache.lucene.index.DirectoryReader; + +public class PageCollector implements DocCollector{ + + private Page page ; + private DocCollector pre; + private int count = -1 ; + + public PageCollector(Page page, DocCollector pre){ + this.page = page ; + this.pre = pre ; + } + + @Override + public ColResult accept(DirectoryReader dreader, SearchRequest sreq, int docId) throws IOException { + if (pre.accept(dreader, sreq, docId) == ColResult.ACCEPT){ + count++ ; + if (count >= page.getStartLoc() && count < page.getEndLoc()){ + return ColResult.ACCEPT ; + } else if (count == page.getEndLoc()){ + return ColResult.BREAK ; + } + } + return ColResult.REVOKE ; + } +} diff --git a/src/net/ion/nsearcher/search/SearchRequest.java b/src/net/ion/nsearcher/search/SearchRequest.java index a9c99d8..bcea516 100644 --- a/src/net/ion/nsearcher/search/SearchRequest.java +++ b/src/net/ion/nsearcher/search/SearchRequest.java @@ -38,6 +38,7 @@ public class SearchRequest { private Set columns = SetUtil.newSet() ; private Set lazyColumns = SetUtil.newSet() ; private String userDefine = ""; + private DocCollector collector = DocCollector.BLANK; SearchRequest(Searcher searcher, Query query){ this.searcher = searcher ; @@ -229,5 +230,14 @@ public StoredFieldVisitor selector(){ return columns.size() == 0 ? null : new DocumentStoredFieldVisitor(columns) ; } + public SearchRequest collect(DocCollector collector) { + this.collector = collector ; + return this; + } + + public DocCollector collector(){ + return collector ; + } + } diff --git a/src/net/ion/nsearcher/search/SearchResponse.java b/src/net/ion/nsearcher/search/SearchResponse.java index 0116e92..543cc02 100644 --- a/src/net/ion/nsearcher/search/SearchResponse.java +++ b/src/net/ion/nsearcher/search/SearchResponse.java @@ -24,22 +24,26 @@ public class SearchResponse { private final long endTime; private Future postFuture ; private List docs ; - private TopDocs tdocs; - private SearchResponse(ISearchable searcher, SearchRequest sreq, List docs, TopDocs tdocs, long startTime) { + private int totalCount ; + private SearchResponse(ISearchable searcher, SearchRequest sreq, List docs, int totalCount, long startTime) { this.searcher = searcher ; this.sreq = sreq ; this.startTime = startTime; this.endTime = System.currentTimeMillis(); this.docs = docs ; - this.tdocs = tdocs ; + this.totalCount = totalCount ; } + public static SearchResponse create(ISearchable searcher, SearchRequest sreq, List docs, int totalCount, long startTime) throws IOException { + return new SearchResponse(searcher, sreq, makeDocument(sreq, docs), totalCount, startTime) ; + } + public static SearchResponse create(ISearchable searcher, SearchRequest sreq, TopDocs docs, long startTime) throws IOException { - return new SearchResponse(searcher, sreq, makeDocument(searcher, sreq, docs), docs, startTime); + return new SearchResponse(searcher, sreq, makeDocument(sreq, docs), docs.totalHits, startTime); } public int totalCount() { - return tdocs.totalHits ; + return totalCount ; // 전체 total은 searcherImpl이 구함. // return searcher.totalCount(sreq, sreq.getFilter()) ; } @@ -87,7 +91,16 @@ public List getDocument() throws IOException{ return eachDoc(EachDocHandler.TOLIST) ; } - private static List makeDocument(ISearchable searcher, SearchRequest sreq, TopDocs docs) { + private static List makeDocument(SearchRequest sreq, List docs) { + List result = ListUtil.newList() ; + + for (int i = sreq.skip(); i < Math.min(sreq.limit(), docs.size()); i++) { + result.add(docs.get(i)); + } + return result; + } + + private static List makeDocument(SearchRequest sreq, TopDocs docs) { ScoreDoc[] sdocs = docs.scoreDocs; List result = ListUtil.newList() ; diff --git a/src/net/ion/nsearcher/search/SingleSearcher.java b/src/net/ion/nsearcher/search/SingleSearcher.java index f6b6cc6..9432d8a 100644 --- a/src/net/ion/nsearcher/search/SingleSearcher.java +++ b/src/net/ion/nsearcher/search/SingleSearcher.java @@ -2,28 +2,36 @@ import java.io.Closeable; import java.io.IOException; +import java.util.List; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import net.ion.framework.util.Debug; +import net.ion.framework.util.ListUtil; import net.ion.nsearcher.common.IKeywordField; import net.ion.nsearcher.common.ReadDocument; import net.ion.nsearcher.config.Central; import net.ion.nsearcher.config.SearchConfig; import net.ion.nsearcher.reader.InfoReader; +import net.ion.nsearcher.search.DocCollector.ColResult; import org.apache.lucene.document.Document; +import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.search.Collector; import org.apache.lucene.search.Filter; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortField.Type; @@ -50,7 +58,7 @@ public static SingleSearcher create(SearchConfig sconfig, Central central) throw return new SingleSearcher(central, DirectoryReader.open(central.dir())); } - public SearchResponse search(SearchRequest sreq, Filter filters) throws IOException { + public SearchResponse search(final SearchRequest sreq, Filter filters) throws IOException { reloadReader(); Lock rlock = central.readLock() ; @@ -59,6 +67,33 @@ public SearchResponse search(SearchRequest sreq, Filter filters) throws IOExcept locked = rlock.tryLock(); long startTime = System.currentTimeMillis(); + if (sreq.collector() != AbstractDocCollector.BLANK){ + final AtomicInteger total = new AtomicInteger() ; + final List docs = ListUtil.newList() ; + try { + isearcher.search(sreq.query(), filters, new Collector() { + public void setScorer(Scorer scorer) throws IOException { + } + public void setNextReader(AtomicReaderContext atomicreadercontext) throws IOException { + } + public void collect(int docId) throws IOException { + ColResult cresult = sreq.collector().accept(dreader, sreq, docId); + if (cresult == ColResult.ACCEPT) { + docs.add(docId) ; + total.incrementAndGet() ; + } + if (cresult == ColResult.BREAK) throw new IllegalStateException("break") ; + } + public boolean acceptsDocsOutOfOrder() { + return false; + } + }); + } catch(IllegalStateException ignore){ + } + return SearchResponse.create(this, sreq, docs, total.intValue(), startTime) ; + } + + TopDocs docs = isearcher.search(sreq.query(), filters, sreq.limit(), sreq.sort()); return SearchResponse.create(this, sreq, docs, startTime); } finally { @@ -79,6 +114,8 @@ public Document findById(String id) throws IOException{ public int totalCount(SearchRequest sreq, Filter filters) { try { // reloadReader() ; + if (sreq.collector() == AbstractDocCollector.BLANK) throw new IllegalArgumentException("with collector condition, this method meanless") ; + TopDocs docs = isearcher.search(sreq.query(), filters, Integer.MAX_VALUE); return docs.totalHits; } catch (IOException e) { diff --git a/src/org/apache/lucene/analysis/ko/MyKoreanAnalyzer.java b/src/org/apache/lucene/analysis/ko/MyKoreanAnalyzer.java index d1c9229..74f8617 100644 --- a/src/org/apache/lucene/analysis/ko/MyKoreanAnalyzer.java +++ b/src/org/apache/lucene/analysis/ko/MyKoreanAnalyzer.java @@ -5,20 +5,12 @@ import java.io.Reader; import java.io.StringReader; import java.nio.CharBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.Set; import net.ion.framework.util.IOUtil; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.LowerCaseFilter; import org.apache.lucene.analysis.core.StopFilter; -import org.apache.lucene.analysis.ko.HanjaMappingFilter; -import org.apache.lucene.analysis.ko.KoreanFilter; -import org.apache.lucene.analysis.ko.KoreanTokenizer; -import org.apache.lucene.analysis.ko.PunctuationDelimitFilter; -import org.apache.lucene.analysis.ko.WordSegmentFilter; import org.apache.lucene.analysis.standard.ClassicFilter; import org.apache.lucene.analysis.util.CharArraySet; import org.apache.lucene.analysis.util.StopwordAnalyzerBase; diff --git a/test/net/ion/nsearcher/index/TestDocument.java b/test/net/ion/nsearcher/index/TestDocument.java index b9050a5..a09263f 100644 --- a/test/net/ion/nsearcher/index/TestDocument.java +++ b/test/net/ion/nsearcher/index/TestDocument.java @@ -143,4 +143,29 @@ public Document handle(IndexSession isession) throws Exception { request.find().debugPrint(); } + + public void testLoadDocument() throws Exception { + Document doc = indexer.index(new IndexJob() { + public Document handle(IndexSession isession) throws Exception { + final WriteDocument writeDoc = isession.newDocument("bleujin").keyword("@path", "_emp").keyword("@path", "/ion").stext("explain", "hello bleujin"); + isession.insertDocument(writeDoc); + return writeDoc.toLuceneDoc(); + } + }); + +// cen.newSearcher().createRequestByKey("bleujin").find().debugPrint(); + + indexer.index(new IndexJob() { + @Override + public Void handle(IndexSession isession) throws Exception { + isession.loadDocument("bleujin").number("age", 20).update() ; + return null; + } + }) ; + + ReadDocument rdoc = cen.newSearcher().createRequest("").findOne() ; + assertEquals("hello bleujin", rdoc.asString("explain")); + assertEquals(true, cen.newSearcher().createRequestByTerm("explain", "bleujin").findOne() != null) ; + } + } diff --git a/test/net/ion/nsearcher/search/TestSearchRequestCollect.java b/test/net/ion/nsearcher/search/TestSearchRequestCollect.java new file mode 100644 index 0000000..4d74bfb --- /dev/null +++ b/test/net/ion/nsearcher/search/TestSearchRequestCollect.java @@ -0,0 +1,67 @@ +package net.ion.nsearcher.search; + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; + +import net.ion.framework.db.Page; +import net.ion.framework.util.Debug; +import net.ion.nsearcher.common.ReadDocument; +import net.ion.nsearcher.config.Central; +import net.ion.nsearcher.config.CentralConfig; +import net.ion.nsearcher.index.IndexJob; +import net.ion.nsearcher.index.IndexSession; +import junit.framework.TestCase; + +public class TestSearchRequestCollect extends TestCase { + + + private Central central; + private Searcher searcher; + + @Override + protected void setUp() throws Exception { + super.setUp(); + this.central = CentralConfig.newRam().build() ; + this.searcher = central.newSearcher() ; + + central.newIndexer().index(new IndexJob() { + @Override + public Void handle(IndexSession isession) throws Exception { + for (int i = 0; i < 105; i++) { + isession.newDocument("" + i).number("i", i).update() ; + } + return null; + } + }) ; + } + + public void testCollect() throws Exception { + SearchResponse response = central.newSearcher().createRequest("").selections("IS-Key", "i").collect(new AbstractDocCollector(){ + @Override + public ColResult accept(ReadDocument doc) { + return doc.asLong("i", 0) >= 25 && doc.asLong("i", 0) <= 35 ? ColResult.ACCEPT : ColResult.REVOKE ; + } + }).page(Page.create(2, 1)).find() ; + + response.debugPrint(); + Debug.line(response.totalCount(), response.size(), response.getDocument().size()); + } + + + public void testPageCollector() throws Exception { + final AbstractDocCollector bullhock = new AbstractDocCollector(){ + @Override + public ColResult accept(ReadDocument doc) { + return doc.asLong("i", 0) >= 40 && doc.asLong("i", 0) <= 50 ? ColResult.ACCEPT : ColResult.REVOKE ; + } + }; + + SearchRequest request = central.newSearcher().createRequest("").selections("IS-Key", "i").collect(new PageCollector(Page.create(3, 3), bullhock)); + SearchResponse response = request.find() ; + + response.debugPrint(); + Debug.line(response.totalCount(), response.size(), response.getDocument().size()); + } +} diff --git a/test/net/ion/nsearcher/search/TestSort.java b/test/net/ion/nsearcher/search/TestSort.java index ea1ee7d..b266acd 100644 --- a/test/net/ion/nsearcher/search/TestSort.java +++ b/test/net/ion/nsearcher/search/TestSort.java @@ -26,12 +26,34 @@ public Void handle(IndexSession isession) throws Exception { return null; } }) ; - - + Sort sort = central.newSearcher().createRequest("").sort("val=desc").sort() ; SortField sfield = sort.getSort()[0]; assertEquals(true, sfield.getType() == Type.STRING) ; assertEquals(true, sfield.getReverse()) ; assertEquals("val", sfield.getField()) ; } + + public void testDescending() throws Exception { + Central central = CentralConfig.newRam().build() ; + + Sort sort = central.newSearcher().createRequest("").descending("val").sort() ; + SortField sfield = sort.getSort()[0]; + assertEquals(true, sfield.getType() == Type.STRING) ; + assertEquals(true, sfield.getReverse()) ; + assertEquals("val", sfield.getField()) ; + + } + + public void testAscending() throws Exception { + Central central = CentralConfig.newRam().build() ; + + Sort sort = central.newSearcher().createRequest("").ascending("val").sort() ; + SortField sfield = sort.getSort()[0]; + assertEquals(true, sfield.getType() == Type.STRING) ; + assertEquals(false, sfield.getReverse()) ; + assertEquals("val", sfield.getField()) ; + + } + }