19 package org.sleuthkit.autopsy.keywordsearch;
21 import com.google.common.base.CharMatcher;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.List;
27 import java.util.logging.Level;
28 import java.util.regex.Matcher;
29 import java.util.regex.Pattern;
30 import org.apache.commons.lang3.StringUtils;
31 import org.apache.commons.validator.routines.DomainValidator;
32 import org.apache.solr.client.solrj.SolrQuery;
33 import org.apache.solr.client.solrj.SolrQuery.SortClause;
34 import org.apache.solr.client.solrj.SolrRequest;
35 import org.apache.solr.client.solrj.response.QueryResponse;
36 import org.apache.solr.common.SolrDocument;
37 import org.apache.solr.common.SolrDocumentList;
38 import org.apache.solr.common.params.CursorMarkParams;
39 import org.openide.util.Exceptions;
40 import org.openide.util.NbBundle;
53 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
55 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
74 final class RegexQuery
implements KeywordSearchQuery {
76 public static final Logger LOGGER = Logger.getLogger(RegexQuery.class.getName());
88 private static final CharSequence[] UNSUPPORTED_CHARS = {
"\\d",
"\\D",
"\\w",
"\\W",
"\\s",
"\\S",
"\\n",
89 "\\t",
"\\r",
"\\f",
"\\a",
"\\e",
"\\v",
"\\V",
"\\h",
"\\H",
"\\p"};
91 private static final int MAX_RESULTS_PER_CURSOR_MARK = 512;
92 private static final int MIN_EMAIL_ADDR_LENGTH = 8;
93 private static final String SNIPPET_DELIMITER = String.valueOf(Character.toChars(171));
95 private final List<KeywordQueryFilter> filters =
new ArrayList<>();
96 private final KeywordList keywordList;
97 private final Keyword originalKeyword;
98 private final String keywordString;
99 private final boolean queryStringContainsWildcardPrefix;
100 private final boolean queryStringContainsWildcardSuffix;
102 private boolean escaped;
103 private String escapedQuery;
104 private String field = Server.Schema.CONTENT_STR.toString();
112 RegexQuery(KeywordList keywordList, Keyword keyword) {
113 this.keywordList = keywordList;
114 this.originalKeyword = keyword;
115 this.keywordString = keyword.getSearchTerm();
117 this.queryStringContainsWildcardPrefix = this.keywordString.startsWith(
".*");
118 this.queryStringContainsWildcardSuffix = this.keywordString.endsWith(
".*");
122 public KeywordList getKeywordList() {
127 public boolean validate() {
128 if (keywordString.isEmpty()) {
133 Pattern.compile(keywordString, Pattern.UNICODE_CHARACTER_CLASS);
139 for (CharSequence c : UNSUPPORTED_CHARS) {
140 if (keywordString.contains(c)) {
145 }
catch (IllegalArgumentException ex) {
151 public QueryResults performQuery() throws NoOpenCoreException {
153 final Server solrServer = KeywordSearch.getServer();
154 SolrQuery solrQuery =
new SolrQuery();
173 solrQuery.setQuery((field == null ? Server.Schema.CONTENT_STR.toString() : field) +
":/"
174 + (queryStringContainsWildcardPrefix ?
"" :
".*") + getQueryString()
175 + (queryStringContainsWildcardSuffix ?
"" :
".*") +
"/");
178 solrQuery.setFields(Server.Schema.CONTENT_STR.toString(), Server.Schema.ID.toString(), Server.Schema.CHUNK_SIZE.toString());
181 .map(KeywordQueryFilter::toString)
182 .forEach(solrQuery::addFilterQuery);
184 solrQuery.setRows(MAX_RESULTS_PER_CURSOR_MARK);
186 solrQuery.setSort(SortClause.asc(Server.Schema.ID.toString()));
188 String cursorMark = CursorMarkParams.CURSOR_MARK_START;
189 SolrDocumentList resultList;
190 boolean allResultsProcessed =
false;
191 QueryResults results =
new QueryResults(
this);
193 while (!allResultsProcessed) {
195 solrQuery.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
196 QueryResponse response = solrServer.query(solrQuery, SolrRequest.METHOD.POST);
197 resultList = response.getResults();
199 for (SolrDocument resultDoc : resultList) {
201 List<KeywordHit> keywordHits = createKeywordHits(resultDoc);
202 for (KeywordHit hit : keywordHits) {
203 Keyword keywordInstance =
new Keyword(hit.getHit(),
true,
true, originalKeyword.getListName(), originalKeyword.getOriginalTerm());
204 List<KeywordHit> hitsForKeyword = results.getResults(keywordInstance);
205 if (hitsForKeyword == null) {
206 hitsForKeyword =
new ArrayList<>();
207 results.addResult(keywordInstance, hitsForKeyword);
209 hitsForKeyword.add(hit);
211 }
catch (TskCoreException ex) {
212 LOGGER.log(Level.SEVERE,
"Error creating keyword hits", ex);
216 String nextCursorMark = response.getNextCursorMark();
217 if (cursorMark.equals(nextCursorMark)) {
218 allResultsProcessed =
true;
220 cursorMark = nextCursorMark;
221 }
catch (KeywordSearchModuleException ex) {
222 LOGGER.log(Level.SEVERE,
"Error executing Regex Solr Query: " + keywordString, ex);
223 MessageNotifyUtil.Notify.error(NbBundle.getMessage(Server.class,
"Server.query.exception.msg", keywordString), ex.getCause().getMessage());
230 private List<KeywordHit> createKeywordHits(SolrDocument solrDoc)
throws TskCoreException {
232 final HashMap<String, String> keywordsFoundInThisDocument =
new HashMap<>();
234 List<KeywordHit> hits =
new ArrayList<>();
235 final String docId = solrDoc.getFieldValue(Server.Schema.ID.toString()).toString();
236 final Integer chunkSize = (Integer) solrDoc.getFieldValue(Server.Schema.CHUNK_SIZE.toString());
238 final Collection<Object> content_str = solrDoc.getFieldValues(Server.Schema.CONTENT_STR.toString());
240 final Pattern pattern = Pattern.compile(keywordString);
242 for (Object content_obj : content_str) {
243 String content = (String) content_obj;
244 Matcher hitMatcher = pattern.matcher(content);
247 while (hitMatcher.find(offset)) {
252 if (chunkSize != null && hitMatcher.start() >= chunkSize) {
256 String hit = hitMatcher.group();
258 offset = hitMatcher.end();
259 final ATTRIBUTE_TYPE artifactAttributeType = originalKeyword.getArtifactAttributeType();
267 if (!queryStringContainsWildcardSuffix
268 && (artifactAttributeType == ATTRIBUTE_TYPE.TSK_PHONE_NUMBER
269 || artifactAttributeType == ATTRIBUTE_TYPE.TSK_IP_ADDRESS)) {
270 if (artifactAttributeType == ATTRIBUTE_TYPE.TSK_PHONE_NUMBER) {
272 hit = hit.replaceAll(
"^[^0-9\\(]",
"");
275 hit = hit.replaceAll(
"^[^0-9]",
"");
278 hit = hit.replaceAll(
"[^0-9]$",
"");
293 if (keywordsFoundInThisDocument.containsKey(hit)) {
296 keywordsFoundInThisDocument.put(hit, hit);
298 if (artifactAttributeType == null) {
299 hits.add(
new KeywordHit(docId, makeSnippet(content, hitMatcher, hit), hit));
301 switch (artifactAttributeType) {
308 if (hit.length() >= MIN_EMAIL_ADDR_LENGTH
309 && DomainValidator.getInstance(
true).isValidTld(hit.substring(hit.lastIndexOf(
'.')))) {
310 hits.add(
new KeywordHit(docId, makeSnippet(content, hitMatcher, hit), hit));
314 case TSK_CARD_NUMBER:
320 Matcher ccnMatcher = CREDIT_CARD_NUM_PATTERN.matcher(hit);
322 for (
int rLength = hit.length(); rLength >= 12; rLength--) {
323 ccnMatcher.region(0, rLength);
324 if (ccnMatcher.find()) {
325 final String group = ccnMatcher.group(
"ccn");
326 if (CreditCardValidator.isValidCCN(group)) {
327 hits.add(
new KeywordHit(docId, makeSnippet(content, hitMatcher, hit), hit));
334 hits.add(
new KeywordHit(docId, makeSnippet(content, hitMatcher, hit), hit));
341 }
catch (Throwable error) {
350 throw new TskCoreException(
"Failed to create keyword hits for Solr document id " + docId +
" due to " + error.getMessage());
368 private String makeSnippet(String content, Matcher hitMatcher, String hit) {
370 int maxIndex = content.length() - 1;
371 final int end = hitMatcher.end();
372 final int start = hitMatcher.start();
374 return content.substring(Integer.max(0, start - 20), Integer.max(0, start))
375 + SNIPPET_DELIMITER + hit + SNIPPET_DELIMITER
376 + content.substring(Integer.min(maxIndex, end), Integer.min(maxIndex, end + 20));
380 public void addFilter(KeywordQueryFilter filter) {
381 this.filters.add(filter);
385 public void setField(String field) {
390 public void setSubstringQuery() {
394 synchronized public void escape() {
395 if (isEscaped() ==
false) {
396 escapedQuery = KeywordSearchUtil.escapeLuceneQuery(keywordString);
402 synchronized public boolean isEscaped() {
407 public boolean isLiteral() {
412 public String getQueryString() {
413 return originalKeyword.getSearchTerm();
417 synchronized public String getEscapedQueryString() {
418 if (
false == isEscaped()) {
441 public BlackboardArtifact postKeywordHitToBlackboard(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) {
442 final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName();
444 if (content == null) {
445 LOGGER.log(Level.WARNING,
"Error adding artifact for keyword hit to blackboard");
452 if (originalKeyword.getArtifactAttributeType() == ATTRIBUTE_TYPE.TSK_CARD_NUMBER) {
453 createCCNAccount(content, foundKeyword, hit, snippet, listName);
461 BlackboardArtifact newArtifact;
462 Collection<BlackboardAttribute> attributes =
new ArrayList<>();
464 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, foundKeyword.getSearchTerm()));
465 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP, MODULE_NAME, getQueryString()));
468 newArtifact = content.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
469 }
catch (TskCoreException ex) {
470 LOGGER.log(Level.SEVERE,
"Error adding artifact for keyword hit to blackboard", ex);
474 if (StringUtils.isNotBlank(listName)) {
475 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
477 if (snippet != null) {
478 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
481 hit.getArtifactID().ifPresent(artifactID
482 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID))
485 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal()));
488 newArtifact.addAttributes(attributes);
490 }
catch (TskCoreException e) {
491 LOGGER.log(Level.SEVERE,
"Error adding bb attributes for terms search artifact", e);
496 private void createCCNAccount(Content content, Keyword foundKeyword, KeywordHit hit, String snippet, String listName) {
498 final String MODULE_NAME = KeywordSearchModuleFactory.getModuleName();
500 if (originalKeyword.getArtifactAttributeType() != ATTRIBUTE_TYPE.TSK_CARD_NUMBER) {
501 LOGGER.log(Level.SEVERE,
"Keyword hit is not a credit card number");
509 Collection<BlackboardAttribute> attributes =
new ArrayList<>();
511 Map<BlackboardAttribute.Type, BlackboardAttribute> parsedTrackAttributeMap =
new HashMap<>();
512 Matcher matcher = TermsComponentQuery.CREDIT_CARD_TRACK1_PATTERN.matcher(hit.getSnippet());
513 if (matcher.find()) {
514 parseTrack1Data(parsedTrackAttributeMap, matcher);
516 matcher = CREDIT_CARD_TRACK2_PATTERN.matcher(hit.getSnippet());
517 if (matcher.find()) {
518 parseTrack2Data(parsedTrackAttributeMap, matcher);
520 final BlackboardAttribute ccnAttribute = parsedTrackAttributeMap.get(
new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_CARD_NUMBER));
521 if (ccnAttribute == null || StringUtils.isBlank(ccnAttribute.getValueString())) {
523 if (hit.isArtifactHit()) {
524 LOGGER.log(Level.SEVERE, String.format(
"Failed to parse credit card account number for artifact keyword hit: term = %s, snippet = '%s', artifact id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getArtifactID().get()));
527 LOGGER.log(Level.SEVERE, String.format(
"Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s', object id = %d", foundKeyword.getSearchTerm(), hit.getSnippet(), hit.getContentID()));
528 }
catch (TskCoreException ex) {
529 LOGGER.log(Level.SEVERE, String.format(
"Failed to parse credit card account number for content keyword hit: term = %s, snippet = '%s' ", foundKeyword.getSearchTerm(), hit.getSnippet()));
530 LOGGER.log(Level.SEVERE,
"There was a error getting contentID for keyword hit.", ex);
535 attributes.addAll(parsedTrackAttributeMap.values());
541 final int bin = Integer.parseInt(ccnAttribute.getValueString().substring(0, 8));
542 CreditCards.BankIdentificationNumber binInfo = CreditCards.getBINInfo(bin);
543 if (binInfo != null) {
544 binInfo.getScheme().ifPresent(scheme
545 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_SCHEME, MODULE_NAME, scheme)));
546 binInfo.getCardType().ifPresent(cardType
547 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CARD_TYPE, MODULE_NAME, cardType)));
548 binInfo.getBrand().ifPresent(brand
549 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BRAND_NAME, MODULE_NAME, brand)));
550 binInfo.getBankName().ifPresent(bankName
551 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_BANK_NAME, MODULE_NAME, bankName)));
552 binInfo.getBankPhoneNumber().ifPresent(phoneNumber
553 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PHONE_NUMBER, MODULE_NAME, phoneNumber)));
554 binInfo.getBankURL().ifPresent(url
555 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL, MODULE_NAME, url)));
556 binInfo.getCountry().ifPresent(country
557 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNTRY, MODULE_NAME, country)));
558 binInfo.getBankCity().ifPresent(city
559 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_CITY, MODULE_NAME, city)));
567 if (content instanceof AbstractFile) {
568 AbstractFile file = (AbstractFile) content;
569 if (file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNUSED_BLOCKS
570 || file.getType() == TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS) {
571 attributes.add(
new BlackboardAttribute(KEYWORD_SEARCH_DOCUMENT_ID, MODULE_NAME, hit.getSolrDocumentId()));
575 if (StringUtils.isNotBlank(listName)) {
576 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_SET_NAME, MODULE_NAME, listName));
578 if (snippet != null) {
579 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW, MODULE_NAME, snippet));
582 hit.getArtifactID().ifPresent(artifactID
583 -> attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT, MODULE_NAME, artifactID))
586 attributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SEARCH_TYPE, MODULE_NAME, KeywordSearch.QueryType.REGEX.ordinal()));
593 AccountFileInstance ccAccountInstance = Case.getCurrentCase().getSleuthkitCase().getCommunicationsManager().createAccountFileInstance(Account.Type.CREDIT_CARD, ccnAttribute.getValueString() , MODULE_NAME, content);
595 ccAccountInstance.addAttributes(attributes);
597 }
catch (TskCoreException ex) {
598 LOGGER.log(Level.SEVERE,
"Error creating CCN account instance", ex);
611 static private void parseTrack2Data(Map<BlackboardAttribute.Type, BlackboardAttribute> attributesMap, Matcher matcher) {
612 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_NUMBER,
"accountNumber", matcher);
613 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_EXPIRATION,
"expiration", matcher);
614 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_SERVICE_CODE,
"serviceCode", matcher);
615 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_DISCRETIONARY,
"discretionary", matcher);
616 addAttributeIfNotAlreadyCaptured(attributesMap, ATTRIBUTE_TYPE.TSK_CARD_LRC,
"LRC", matcher);
628 static private void parseTrack1Data(Map<BlackboardAttribute.Type, BlackboardAttribute> attributeMap, Matcher matcher) {
629 parseTrack2Data(attributeMap, matcher);
630 addAttributeIfNotAlreadyCaptured(attributeMap, ATTRIBUTE_TYPE.TSK_NAME_PERSON,
"name", matcher);
645 static private void addAttributeIfNotAlreadyCaptured(Map<BlackboardAttribute.Type, BlackboardAttribute> attributeMap, ATTRIBUTE_TYPE attrType, String groupName, Matcher matcher) {
646 BlackboardAttribute.Type type =
new BlackboardAttribute.Type(attrType);
647 attributeMap.computeIfAbsent(type, t -> {
648 String value = matcher.group(groupName);
649 if (attrType.equals(ATTRIBUTE_TYPE.TSK_CARD_NUMBER)) {
650 attributeMap.put(
new BlackboardAttribute.Type(ATTRIBUTE_TYPE.TSK_KEYWORD),
651 new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD, MODULE_NAME, value));
652 value = CharMatcher.anyOf(
" -").removeFrom(value);
654 if (StringUtils.isNotBlank(value)) {
655 return new BlackboardAttribute(attrType, MODULE_NAME, value);