19 package org.sleuthkit.autopsy.filequery;
33 import java.util.ArrayList;
34 import java.util.List;
35 import java.util.stream.Collectors;
36 import org.openide.util.NbBundle;
45 class FileSearchFiltering {
57 static List<ResultFile> runQueries(List<FileFilter> filters, SleuthkitCase caseDb, CentralRepository centralRepoDb)
throws FileSearchException {
59 throw new FileSearchException(
"Case DB parameter is null");
62 String combinedQuery =
"";
63 for (FileFilter filter : filters) {
64 if (!filter.getWhereClause().isEmpty()) {
65 if (!combinedQuery.isEmpty()) {
66 combinedQuery +=
" AND ";
68 combinedQuery +=
"(" + filter.getWhereClause() +
")";
72 if (combinedQuery.isEmpty()) {
74 throw new FileSearchException(
"Selected filters do not include a case database query");
77 return getResultList(filters, combinedQuery, caseDb, centralRepoDb);
78 }
catch (TskCoreException ex) {
79 throw new FileSearchException(
"Error querying case database", ex);
97 private static List<ResultFile> getResultList(List<FileFilter> filters, String combinedQuery, SleuthkitCase caseDb, CentralRepository centralRepoDb)
throws TskCoreException, FileSearchException {
99 List<ResultFile> resultList =
new ArrayList<>();
100 List<AbstractFile> sqlResults = caseDb.findAllFilesWhere(combinedQuery);
103 if (sqlResults.isEmpty()) {
108 for (AbstractFile abstractFile : sqlResults) {
109 resultList.add(
new ResultFile(abstractFile));
113 for (FileFilter filter : filters) {
114 if (filter.useAlternateFilter()) {
115 resultList = filter.applyAlternateFilter(resultList, caseDb, centralRepoDb);
118 if (resultList.isEmpty()) {
128 static abstract class FileFilter {
137 abstract String getWhereClause();
145 boolean useAlternateFilter() {
163 List<ResultFile> applyAlternateFilter(List<ResultFile> currentResults, SleuthkitCase caseDb,
164 CentralRepository centralRepoDb)
throws FileSearchException {
165 return new ArrayList<>();
173 abstract String getDesc();
179 static class SizeFilter
extends FileFilter {
181 private final List<FileSize> fileSizes;
188 SizeFilter(List<FileSize> fileSizes) {
189 this.fileSizes = fileSizes;
193 String getWhereClause() {
194 String queryStr =
"";
195 for (FileSize size : fileSizes) {
196 if (!queryStr.isEmpty()) {
199 if (size.getMaxBytes() != FileSize.NO_MAXIMUM) {
200 queryStr +=
"(size > \'" + size.getMinBytes() +
"\' AND size <= \'" + size.getMaxBytes() +
"\')";
202 queryStr +=
"(size >= \'" + size.getMinBytes() +
"\')";
210 "FileSearchFiltering.SizeFilter.desc=Files with size in range(s): {0}",
211 "FileSearchFiltering.SizeFilter.or= or ",
212 "# {0} - Minimum bytes",
213 "# {1} - Maximum bytes",
214 "FileSearchFiltering.SizeFilter.range=({0} to {1})",})
218 for (FileSize size : fileSizes) {
219 if (!desc.isEmpty()) {
220 desc += Bundle.FileSearchFiltering_SizeFilter_or();
222 desc += Bundle.FileSearchFiltering_SizeFilter_range(size.getMinBytes(), size.getMaxBytes());
224 desc = Bundle.FileSearchFiltering_SizeFilter_desc(desc);
233 static class ParentSearchTerm {
235 private final String searchStr;
236 private final boolean fullPath;
237 private final boolean included;
248 ParentSearchTerm(String searchStr,
boolean isFullPath,
boolean isIncluded) {
249 this.searchStr = searchStr;
250 this.fullPath = isFullPath;
251 this.included = isIncluded;
259 String getSQLForTerm() {
263 return "parent_path=\'" + getSearchStr() +
"\'";
265 return "parent_path LIKE \'%" + getSearchStr() +
"%\'";
269 return "parent_path!=\'" + getSearchStr() +
"\'";
271 return "parent_path NOT LIKE \'%" + getSearchStr() +
"%\'";
277 "FileSearchFiltering.ParentSearchTerm.fullString= (exact)",
278 "FileSearchFiltering.ParentSearchTerm.subString= (substring)",
279 "FileSearchFiltering.ParentSearchTerm.includeString= (include)",
280 "FileSearchFiltering.ParentSearchTerm.excludeString= (exclude)",})
282 public String toString() {
283 String returnString = getSearchStr();
285 returnString += Bundle.FileSearchFiltering_ParentSearchTerm_fullString();
287 returnString += Bundle.FileSearchFiltering_ParentSearchTerm_subString();
290 returnString += Bundle.FileSearchFiltering_ParentSearchTerm_includeString();
292 returnString += Bundle.FileSearchFiltering_ParentSearchTerm_excludeString();
300 boolean isFullPath() {
307 boolean isIncluded() {
314 String getSearchStr() {
322 static class ParentFilter
extends FileFilter {
324 private final List<ParentSearchTerm> parentSearchTerms;
331 ParentFilter(List<ParentSearchTerm> parentSearchTerms) {
332 this.parentSearchTerms = parentSearchTerms;
336 String getWhereClause() {
337 String includeQueryStr =
"";
338 String excludeQueryStr =
"";
339 for (ParentSearchTerm searchTerm : parentSearchTerms) {
340 if (searchTerm.isIncluded()) {
341 if (!includeQueryStr.isEmpty()) {
342 includeQueryStr +=
" OR ";
344 includeQueryStr += searchTerm.getSQLForTerm();
346 if (!excludeQueryStr.isEmpty()) {
347 excludeQueryStr +=
" AND ";
349 excludeQueryStr += searchTerm.getSQLForTerm();
352 if (!includeQueryStr.isEmpty()) {
353 includeQueryStr =
"(" + includeQueryStr +
")";
355 if (!excludeQueryStr.isEmpty()) {
356 excludeQueryStr =
"(" + excludeQueryStr +
")";
358 if (includeQueryStr.isEmpty() || excludeQueryStr.isEmpty()) {
359 return includeQueryStr + excludeQueryStr;
361 return includeQueryStr +
" AND " + excludeQueryStr;
367 "FileSearchFiltering.ParentFilter.desc=Files with paths matching: {0}",
368 "FileSearchFiltering.ParentFilter.or= or ",
369 "FileSearchFiltering.ParentFilter.exact=(exact match)",
370 "FileSearchFiltering.ParentFilter.substring=(substring)",})
374 for (ParentSearchTerm searchTerm : parentSearchTerms) {
375 if (!desc.isEmpty()) {
376 desc += Bundle.FileSearchFiltering_ParentFilter_or();
378 if (searchTerm.isFullPath()) {
379 desc += searchTerm.getSearchStr() + Bundle.FileSearchFiltering_ParentFilter_exact();
381 desc += searchTerm.getSearchStr() + Bundle.FileSearchFiltering_ParentFilter_substring();
384 desc = Bundle.FileSearchFiltering_ParentFilter_desc(desc);
392 static class DataSourceFilter
extends FileFilter {
394 private final List<DataSource> dataSources;
401 DataSourceFilter(List<DataSource> dataSources) {
402 this.dataSources = dataSources;
406 String getWhereClause() {
407 String queryStr =
"";
408 for (DataSource ds : dataSources) {
409 if (!queryStr.isEmpty()) {
412 queryStr +=
"\'" + ds.getId() +
"\'";
414 queryStr =
"data_source_obj_id IN (" + queryStr +
")";
420 "FileSearchFiltering.DataSourceFilter.desc=Files in data source(s): {0}",
421 "FileSearchFiltering.DataSourceFilter.or= or ",
422 "# {0} - Data source name",
423 "# {1} - Data source ID",
424 "FileSearchFiltering.DataSourceFilter.datasource={0}({1})",})
428 for (DataSource ds : dataSources) {
429 if (!desc.isEmpty()) {
430 desc += Bundle.FileSearchFiltering_DataSourceFilter_or();
432 desc += Bundle.FileSearchFiltering_DataSourceFilter_datasource(ds.getName(), ds.getId());
434 desc = Bundle.FileSearchFiltering_DataSourceFilter_desc(desc);
443 static class KeywordListFilter
extends FileFilter {
445 private final List<String> listNames;
452 KeywordListFilter(List<String> listNames) {
453 this.listNames = listNames;
457 String getWhereClause() {
458 String keywordListPart = concatenateNamesForSQL(listNames);
460 String queryStr =
"(obj_id IN (SELECT obj_id from blackboard_artifacts WHERE artifact_id IN "
461 +
"(SELECT artifact_id FROM blackboard_attributes WHERE artifact_type_id = 9 AND attribute_type_ID = 37 "
462 +
"AND (" + keywordListPart +
"))))";
469 "FileSearchFiltering.KeywordListFilter.desc=Files with keywords in list(s): {0}",})
472 return Bundle.FileSearchFiltering_KeywordListFilter_desc(concatenateSetNamesForDisplay(listNames));
479 static class FileTypeFilter
extends FileFilter {
481 private final List<FileType> categories;
488 FileTypeFilter(List<FileType> categories) {
489 this.categories = categories;
497 FileTypeFilter(FileType category) {
498 this.categories =
new ArrayList<>();
499 this.categories.add(category);
503 String getWhereClause() {
504 String queryStr =
"";
505 for (FileType cat : categories) {
506 for (String type : cat.getMediaTypes()) {
507 if (!queryStr.isEmpty()) {
510 queryStr +=
"\'" + type +
"\'";
513 queryStr =
"mime_type IN (" + queryStr +
")";
519 "FileSearchFiltering.FileTypeFilter.desc=Files with type: {0}",
520 "FileSearchFiltering.FileTypeFilter.or= or ",})
524 for (FileType cat : categories) {
525 if (!desc.isEmpty()) {
526 desc += Bundle.FileSearchFiltering_FileTypeFilter_or();
528 desc += cat.toString();
530 desc = Bundle.FileSearchFiltering_FileTypeFilter_desc(desc);
538 static class FrequencyFilter
extends FileFilter {
540 private final List<Frequency> frequencies;
547 FrequencyFilter(List<Frequency> frequencies) {
548 this.frequencies = frequencies;
552 String getWhereClause() {
559 boolean useAlternateFilter() {
564 List<ResultFile> applyAlternateFilter(List<ResultFile> currentResults, SleuthkitCase caseDb,
565 CentralRepository centralRepoDb)
throws FileSearchException {
569 if (currentResults.isEmpty()) {
570 throw new FileSearchException(
"Can not run on empty list");
574 FileSearch.FrequencyAttribute freqAttr =
new FileSearch.FrequencyAttribute();
575 freqAttr.addAttributeToResultFiles(currentResults, caseDb, centralRepoDb);
578 List<ResultFile> frequencyResults =
new ArrayList<>();
579 for (ResultFile file : currentResults) {
580 if (frequencies.contains(file.getFrequency())) {
581 frequencyResults.add(file);
584 return frequencyResults;
589 "FileSearchFiltering.FrequencyFilter.desc=Files with frequency: {0}",
590 "FileSearchFiltering.FrequencyFilter.or= or ",})
594 for (Frequency freq : frequencies) {
595 if (!desc.isEmpty()) {
596 desc += Bundle.FileSearchFiltering_FrequencyFilter_or();
600 return Bundle.FileSearchFiltering_FrequencyFilter_desc(desc);
608 static class HashSetFilter
extends FileFilter {
610 private final List<String> setNames;
617 HashSetFilter(List<String> setNames) {
618 this.setNames = setNames;
622 String getWhereClause() {
623 String hashSetPart = concatenateNamesForSQL(setNames);
625 String queryStr =
"(obj_id IN (SELECT obj_id from blackboard_artifacts WHERE artifact_id IN "
626 +
"(SELECT artifact_id FROM blackboard_attributes WHERE artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID()
627 +
" AND attribute_type_ID = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() +
" "
628 +
"AND (" + hashSetPart +
"))))";
635 "FileSearchFiltering.HashSetFilter.desc=Files with hash set hits in set(s): {0}",})
638 return Bundle.FileSearchFiltering_HashSetFilter_desc(concatenateSetNamesForDisplay(setNames));
646 static class InterestingFileSetFilter
extends FileFilter {
648 private final List<String> setNames;
655 InterestingFileSetFilter(List<String> setNames) {
656 this.setNames = setNames;
660 String getWhereClause() {
661 String intItemSetPart = concatenateNamesForSQL(setNames);
663 String queryStr =
"(obj_id IN (SELECT obj_id from blackboard_artifacts WHERE artifact_id IN "
664 +
"(SELECT artifact_id FROM blackboard_attributes WHERE artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT.getTypeID()
665 +
" AND attribute_type_ID = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_SET_NAME.getTypeID() +
" "
666 +
"AND (" + intItemSetPart +
"))))";
673 "FileSearchFiltering.InterestingItemSetFilter.desc=Files with interesting item hits in set(s): {0}",})
676 return Bundle.FileSearchFiltering_InterestingItemSetFilter_desc(concatenateSetNamesForDisplay(setNames));
684 static class ObjectDetectionFilter
extends FileFilter {
686 private final List<String> typeNames;
693 ObjectDetectionFilter(List<String> typeNames) {
694 this.typeNames = typeNames;
698 String getWhereClause() {
699 String objTypePart = concatenateNamesForSQL(typeNames);
701 String queryStr =
"(obj_id IN (SELECT obj_id from blackboard_artifacts WHERE artifact_id IN "
702 +
"(SELECT artifact_id FROM blackboard_attributes WHERE artifact_type_id = " + BlackboardArtifact.ARTIFACT_TYPE.TSK_OBJECT_DETECTED.getTypeID()
703 +
" AND attribute_type_ID = " + BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DESCRIPTION.getTypeID() +
" "
704 +
"AND (" + objTypePart +
"))))";
711 "FileSearchFiltering.ObjectDetectionFilter.desc=Files with objects detected in set(s): {0}",})
714 return Bundle.FileSearchFiltering_ObjectDetectionFilter_desc(concatenateSetNamesForDisplay(typeNames));
722 static class ScoreFilter
extends FileFilter {
724 private final List<Score> scores;
731 ScoreFilter(List<Score> scores) {
732 this.scores = scores;
736 String getWhereClause() {
741 String hashsetQueryPart =
"";
742 String tagQueryPart =
"";
743 String intItemQueryPart =
"";
745 if (scores.contains(Score.NOTABLE)) {
747 hashsetQueryPart =
" (known = " + TskData.FileKnown.BAD.getFileKnownValue() +
") ";
750 if (scores.contains(Score.INTERESTING)) {
752 intItemQueryPart =
" (obj_id IN (SELECT obj_id from blackboard_artifacts WHERE artifact_type_id = "
753 + BlackboardArtifact.ARTIFACT_TYPE.TSK_INTERESTING_FILE_HIT.getTypeID() +
")) ";
756 if (scores.contains(Score.NOTABLE) && scores.contains(Score.INTERESTING)) {
758 tagQueryPart =
"(obj_id IN (SELECT obj_id FROM content_tags))";
759 }
else if (scores.contains(Score.NOTABLE)) {
761 tagQueryPart =
"(obj_id IN (SELECT obj_id FROM content_tags WHERE tag_name_id IN (SELECT tag_name_id FROM tag_names WHERE knownStatus = "
762 + TskData.FileKnown.BAD.getFileKnownValue() +
")))";
763 }
else if (scores.contains(Score.INTERESTING)) {
765 tagQueryPart =
"(obj_id IN (SELECT obj_id FROM content_tags WHERE tag_name_id IN (SELECT tag_name_id FROM tag_names WHERE knownStatus != "
766 + TskData.FileKnown.BAD.getFileKnownValue() +
")))";
769 String queryStr = hashsetQueryPart;
770 if (!intItemQueryPart.isEmpty()) {
771 if (!queryStr.isEmpty()) {
774 queryStr += intItemQueryPart;
776 if (!tagQueryPart.isEmpty()) {
777 if (!queryStr.isEmpty()) {
780 queryStr += tagQueryPart;
787 "FileSearchFiltering.ScoreFilter.desc=Files with score(s) of : {0}",})
790 return Bundle.FileSearchFiltering_ScoreFilter_desc(
791 concatenateSetNamesForDisplay(scores.stream().map(p -> p.toString()).collect(Collectors.toList())));
799 static class TagsFilter
extends FileFilter {
801 private final List<TagName> tagNames;
808 TagsFilter(List<TagName> tagNames) {
809 this.tagNames = tagNames;
813 String getWhereClause() {
815 for (TagName tagName : tagNames) {
816 if (!tagIDs.isEmpty()) {
819 tagIDs += tagName.getId();
822 String queryStr =
"(obj_id IN (SELECT obj_id FROM content_tags WHERE tag_name_id IN (" + tagIDs +
")))";
829 "FileSearchFiltering.TagsFilter.desc=Files that have been tagged {0}",
830 "FileSearchFiltering.TagsFilter.or= or ",})
834 for (TagName name : tagNames) {
835 if (!desc.isEmpty()) {
836 desc += Bundle.FileSearchFiltering_TagsFilter_or();
838 desc += name.getDisplayName();
840 return Bundle.FileSearchFiltering_TagsFilter_desc(desc);
847 static class UserCreatedFilter
extends FileFilter {
852 UserCreatedFilter() {
857 String getWhereClause() {
858 return "(obj_id IN (SELECT obj_id from blackboard_artifacts WHERE artifact_id IN "
859 +
"(SELECT artifact_id FROM blackboard_attributes WHERE artifact_type_id = "
860 + BlackboardArtifact.ARTIFACT_TYPE.TSK_USER_CONTENT_SUSPECTED.getTypeID() +
")))";
864 "FileSearchFiltering.UserCreatedFilter.desc=Files that contain EXIF data",})
867 return Bundle.FileSearchFiltering_UserCreatedFilter_desc();
875 static class NotableFilter
extends FileFilter {
885 String getWhereClause() {
892 boolean useAlternateFilter() {
897 List<ResultFile> applyAlternateFilter(List<ResultFile> currentResults, SleuthkitCase caseDb,
898 CentralRepository centralRepoDb)
throws FileSearchException {
900 if (centralRepoDb == null) {
901 throw new FileSearchException(
"Can not run Previously Notable filter with null Central Repository DB");
906 if (currentResults.isEmpty()) {
907 throw new FileSearchException(
"Can not run on empty list");
911 List<ResultFile> notableResults =
new ArrayList<>();
914 CorrelationAttributeInstance.Type type = CorrelationAttributeInstance.getDefaultCorrelationTypes().get(CorrelationAttributeInstance.FILES_TYPE_ID);
916 for (ResultFile file : currentResults) {
917 if (file.getFirstInstance().getMd5Hash() != null && !file.getFirstInstance().getMd5Hash().isEmpty()) {
920 String value = file.getFirstInstance().getMd5Hash();
921 if (centralRepoDb.getCountArtifactInstancesKnownBad(type, value) > 0) {
922 notableResults.add(file);
926 return notableResults;
927 }
catch (CentralRepoException | CorrelationAttributeNormalizationException ex) {
928 throw new FileSearchException(
"Error querying central repository", ex);
933 "FileSearchFiltering.PreviouslyNotableFilter.desc=Files that were previously marked as notable",})
936 return Bundle.FileSearchFiltering_PreviouslyNotableFilter_desc();
943 static class KnownFilter
extends FileFilter {
946 String getWhereClause() {
947 return "known!=" + TskData.FileKnown.KNOWN.getFileKnownValue();
951 "FileSearchFiltering.KnownFilter.desc=Files which are not known"})
954 return Bundle.FileSearchFiltering_KnownFilter_desc();
959 "FileSearchFiltering.concatenateSetNamesForDisplay.comma=, ",})
960 private static String concatenateSetNamesForDisplay(List<String> setNames) {
962 for (String setName : setNames) {
963 if (!desc.isEmpty()) {
964 desc += Bundle.FileSearchFiltering_concatenateSetNamesForDisplay_comma();
979 private static String concatenateNamesForSQL(List<String> setNames) {
981 for (String setName : setNames) {
982 if (!result.isEmpty()) {
985 result +=
"value_text = \'" + setName +
"\'";
990 private FileSearchFiltering() {