19 package org.sleuthkit.autopsy.keywordsearch;
21 import java.util.ArrayList;
22 import java.util.Collection;
23 import java.util.List;
25 import java.util.logging.Level;
26 import org.apache.commons.lang3.StringUtils;
27 import org.apache.commons.lang3.math.NumberUtils;
28 import org.apache.solr.client.solrj.SolrQuery;
29 import org.apache.solr.client.solrj.SolrRequest;
30 import org.apache.solr.client.solrj.SolrRequest.METHOD;
31 import org.apache.solr.client.solrj.response.QueryResponse;
32 import org.apache.solr.common.SolrDocument;
33 import org.apache.solr.common.SolrDocumentList;
34 import org.apache.solr.common.params.CursorMarkParams;
39 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
41 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
50 class LuceneQuery
implements KeywordSearchQuery {
52 private static final Logger logger = Logger.getLogger(LuceneQuery.class.getName());
53 private String keywordStringEscaped;
54 private boolean isEscaped;
55 private final Keyword originalKeyword ;
56 private final KeywordList keywordList ;
57 private final List<KeywordQueryFilter> filters =
new ArrayList<>();
58 private String field = null;
59 private static final int MAX_RESULTS_PER_CURSOR_MARK = 512;
60 static final int SNIPPET_LENGTH = 50;
61 static final String HIGHLIGHT_FIELD = Server.Schema.TEXT.toString();
63 private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);
70 LuceneQuery(KeywordList keywordList, Keyword keyword) {
71 this.keywordList = keywordList;
72 this.originalKeyword = keyword;
73 this.keywordStringEscaped = this.originalKeyword.getSearchTerm();
77 public void addFilter(KeywordQueryFilter filter) {
78 this.filters.add(filter);
82 public void setField(String field) {
87 public void setSubstringQuery() {
90 keywordStringEscaped +=
"*";
94 public void escape() {
95 keywordStringEscaped = KeywordSearchUtil.escapeLuceneQuery(originalKeyword.getSearchTerm());
100 public boolean isEscaped() {
105 public boolean isLiteral() {
106 return originalKeyword.searchTermIsLiteral();
110 public String getEscapedQueryString() {
111 return this.keywordStringEscaped;
115 public String getQueryString() {
116 return this.originalKeyword.getSearchTerm();
120 public KeywordList getKeywordList() {
125 public QueryResults performQuery() throws KeywordSearchModuleException, NoOpenCoreException {
127 final Server solrServer = KeywordSearch.getServer();
128 double indexSchemaVersion = NumberUtils.toDouble(solrServer.getIndexInfo().getSchemaVersion());
130 SolrQuery solrQuery = createAndConfigureSolrQuery(KeywordSearchSettings.getShowSnippets());
132 final String strippedQueryString = StringUtils.strip(getQueryString(),
"\"");
134 String cursorMark = CursorMarkParams.CURSOR_MARK_START;
135 boolean allResultsProcessed =
false;
136 List<KeywordHit> matches =
new ArrayList<>();
137 while (!allResultsProcessed) {
138 solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
139 QueryResponse response = solrServer.query(solrQuery, SolrRequest.METHOD.POST);
140 SolrDocumentList resultList = response.getResults();
142 Map<String, Map<String, List<String>>> highlightResponse = response.getHighlighting();
144 for (SolrDocument resultDoc : resultList) {
152 final String docId = resultDoc.getFieldValue(Server.Schema.ID.toString()).toString();
153 final Integer chunkSize = (Integer) resultDoc.getFieldValue(Server.Schema.CHUNK_SIZE.toString());
154 final Collection<Object> content = resultDoc.getFieldValues(Server.Schema.CONTENT_STR.toString());
156 if (indexSchemaVersion < 2.0) {
158 matches.add(createKeywordtHit(highlightResponse, docId));
161 for (Object content_obj : content) {
162 String content_str = (String) content_obj;
164 int firstOccurence = StringUtils.indexOfIgnoreCase(content_str, strippedQueryString);
166 if (chunkSize == null || chunkSize == 0 || (firstOccurence > -1 && firstOccurence < chunkSize)) {
167 matches.add(createKeywordtHit(highlightResponse, docId));
171 }
catch (TskException ex) {
172 throw new KeywordSearchModuleException(ex);
175 String nextCursorMark = response.getNextCursorMark();
176 if (cursorMark.equals(nextCursorMark)) {
177 allResultsProcessed =
true;
179 cursorMark = nextCursorMark;
182 QueryResults results =
new QueryResults(
this);
184 results.addResult(
new Keyword(originalKeyword.getSearchTerm(),
true,
true, originalKeyword.getListName(), originalKeyword.getOriginalTerm()), matches);
190 public boolean validate() {
191 return StringUtils.isNotBlank(originalKeyword.getSearchTerm());
195 public BlackboardArtifact writeSingleFileHitsToBlackBoard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) {
196 final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName();
198 Collection<BlackboardAttribute> attributes =
new ArrayList<>();
199 BlackboardArtifact bba;
201 bba = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
202 }
catch (TskCoreException e) {
203 logger.log(Level.WARNING,
"Error adding bb artifact for keyword hit", e);
207 if (snippet != null) {
208 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
210 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm()));
211 if (StringUtils.isNotBlank(listName)) {
212 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
215 if (originalKeyword != null) {
216 BlackboardAttribute.ATTRIBUTE_TYPE selType = originalKeyword.getArtifactAttributeType();
217 if (selType != null) {
218 attributes.add(
new BlackboardAttribute(selType, MODULE_NAME, foundKeyword.getSearchTerm()));
221 if (originalKeyword.searchTermIsWholeWord()) {
222 attributes.add(
new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.LITERAL.ordinal()));
224 attributes.add(
new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.SUBSTRING.ordinal()));
229 hit.getArtifactID().ifPresent(artifactID
230 -> attributes.add(
new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID))
235 bba.addAttributes(attributes);
237 }
catch (TskCoreException e) {
238 logger.log(Level.WARNING,
"Error adding bb attributes to artifact", e);
251 private SolrQuery createAndConfigureSolrQuery(
boolean snippets) {
252 SolrQuery q =
new SolrQuery();
253 q.setShowDebugInfo(DEBUG);
255 String queryStr = originalKeyword.searchTermIsLiteral()
256 ? KeywordSearchUtil.quoteQuery(keywordStringEscaped) : keywordStringEscaped;
261 queryStr = field +
":" + queryStr;
263 q.setQuery(queryStr);
264 q.setRows(MAX_RESULTS_PER_CURSOR_MARK);
266 q.setSort(SolrQuery.SortClause.asc(Server.Schema.ID.toString()));
268 q.setFields(Server.Schema.ID.toString(),
269 Server.Schema.CHUNK_SIZE.toString(),
270 Server.Schema.CONTENT_STR.toString());
272 for (KeywordQueryFilter filter : filters) {
273 q.addFilterQuery(filter.toString());
277 configurwQueryForHighlighting(q);
289 private static void configurwQueryForHighlighting(SolrQuery q) {
290 q.addHighlightField(HIGHLIGHT_FIELD);
291 q.setHighlightSnippets(1);
292 q.setHighlightFragsize(SNIPPET_LENGTH);
295 q.setParam(
"hl.useFastVectorHighlighter",
"on");
296 q.setParam(
"hl.tag.pre",
"«");
297 q.setParam(
"hl.tag.post",
"«");
298 q.setParam(
"hl.fragListBuilder",
"simple");
301 q.setParam(
"hl.fragCharSize", Integer.toString(q.getQuery().length()));
305 q.setParam(
"hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED);
308 private KeywordHit createKeywordtHit(Map<String, Map<String, List<String>>> highlightResponse, String docId)
throws TskException {
314 if (KeywordSearchSettings.getShowSnippets()) {
315 List<String> snippetList = highlightResponse.get(docId).get(Server.Schema.TEXT.toString());
317 if (snippetList != null) {
318 snippet = EscapeUtil.unEscapeHtml(snippetList.get(0)).trim();
322 return new KeywordHit(docId, snippet, originalKeyword.getSearchTerm());
339 static String querySnippet(String query,
long solrObjectId,
boolean isRegex,
boolean group)
throws NoOpenCoreException {
340 return querySnippet(query, solrObjectId, 0, isRegex, group);
358 static String querySnippet(String query,
long solrObjectId,
int chunkID,
boolean isRegex,
boolean group)
throws NoOpenCoreException {
359 SolrQuery q =
new SolrQuery();
360 q.setShowDebugInfo(DEBUG);
364 queryStr = HIGHLIGHT_FIELD +
":"
365 + (group ? KeywordSearchUtil.quoteQuery(query)
372 queryStr = KeywordSearchUtil.quoteQuery(query);
374 q.setQuery(queryStr);
376 String contentIDStr = (chunkID == 0)
377 ? Long.toString(solrObjectId)
378 : Server.getChunkIdString(solrObjectId, chunkID);
379 String idQuery = Server.Schema.ID.toString() +
":" + KeywordSearchUtil.escapeLuceneQuery(contentIDStr);
380 q.addFilterQuery(idQuery);
382 configurwQueryForHighlighting(q);
384 Server solrServer = KeywordSearch.getServer();
387 QueryResponse response = solrServer.query(q, METHOD.POST);
388 Map<String, Map<String, List<String>>> responseHighlight = response.getHighlighting();
389 Map<String, List<String>> responseHighlightID = responseHighlight.get(contentIDStr);
390 if (responseHighlightID == null) {
393 List<String> contentHighlights = responseHighlightID.get(LuceneQuery.HIGHLIGHT_FIELD);
394 if (contentHighlights == null) {
398 return EscapeUtil.unEscapeHtml(contentHighlights.get(0)).trim();
400 }
catch (NoOpenCoreException ex) {
401 logger.log(Level.SEVERE,
"Error executing Lucene Solr Query: " + query +
". Solr doc id " + solrObjectId +
", chunkID " + chunkID , ex);
403 }
catch (KeywordSearchModuleException ex) {
404 logger.log(Level.SEVERE,
"Error executing Lucene Solr Query: " + query +
". Solr doc id " + solrObjectId +
", chunkID " + chunkID , ex);