19 package org.sleuthkit.autopsy.keywordsearch;
21 import com.google.common.io.CharSource;
22 import java.io.IOException;
23 import java.io.PushbackReader;
24 import java.io.Reader;
25 import java.util.List;
26 import java.util.concurrent.ExecutorService;
27 import java.util.concurrent.Executors;
28 import java.util.concurrent.Future;
29 import java.util.concurrent.TimeUnit;
30 import java.util.concurrent.TimeoutException;
31 import java.util.logging.Level;
32 import java.util.stream.Collectors;
33 import java.util.stream.Stream;
34 import org.apache.tika.Tika;
35 import org.apache.tika.metadata.Metadata;
36 import org.apache.tika.parser.ParseContext;
37 import org.openide.util.NbBundle;
46 class TikaTextExtractor
extends FileTextExtractor {
48 static final private Logger logger = Logger.getLogger(TikaTextExtractor.class.getName());
49 private final ExecutorService tikaParseExecutor = Executors.newSingleThreadExecutor();
51 private static final List<String> TIKA_SUPPORTED_TYPES
52 =
new Tika().getParser().getSupportedTypes(
new ParseContext())
54 .map(mt -> mt.getType() +
"/" + mt.getSubtype())
55 .collect(Collectors.toList());
58 public void logWarning(
final String msg, Exception ex) {
59 KeywordSearch.getTikaLogger().log(Level.WARNING, msg, ex);
63 public Reader getReader(AbstractFile sourceFile)
throws TextExtractorException {
64 ReadContentInputStream stream =
new ReadContentInputStream(sourceFile);
66 Metadata metadata =
new Metadata();
68 final Future<Reader> future = tikaParseExecutor.submit(() ->
new Tika().parse(stream, metadata));
70 final Reader tikaReader = future.get(getTimeout(sourceFile.getSize()), TimeUnit.SECONDS);
73 PushbackReader pushbackReader =
new PushbackReader(tikaReader);
74 int read = pushbackReader.read();
76 throw new TextExtractorException(
"Unable to extract text: Tika returned empty reader for " + sourceFile);
78 pushbackReader.unread(read);
81 CharSource metaDataCharSource = getMetaDataCharSource(metadata);
82 return CharSource.concat(
new ReaderCharSource(pushbackReader), metaDataCharSource).openStream();
83 }
catch (TimeoutException te) {
84 final String msg = NbBundle.getMessage(this.getClass(),
"AbstractFileTikaTextExtract.index.tikaParseTimeout.text", sourceFile.getId(), sourceFile.getName());
86 throw new TextExtractorException(msg, te);
87 }
catch (TextExtractorException ex) {
89 }
catch (Exception ex) {
90 KeywordSearch.getTikaLogger().log(Level.WARNING,
"Exception: Unable to Tika parse the content" + sourceFile.getId() +
": " + sourceFile.getName(), ex.getCause());
91 final String msg = NbBundle.getMessage(this.getClass(),
"AbstractFileTikaTextExtract.index.exception.tikaParse.msg", sourceFile.getId(), sourceFile.getName());
93 throw new TextExtractorException(msg, ex);
107 static private CharSource getMetaDataCharSource(Metadata metadata) {
108 return CharSource.wrap(
109 new StringBuilder(
"\n\n------------------------------METADATA------------------------------\n\n")
110 .append(Stream.of(metadata.names()).sorted()
111 .map(key -> key +
": " + metadata.get(key))
112 .collect(Collectors.joining(
"\n"))
117 public boolean isContentTypeSpecific() {
122 public boolean isSupported(AbstractFile file, String detectedFormat) {
123 if (detectedFormat == null
124 || FileTextExtractor.BLOB_MIME_TYPES.contains(detectedFormat)
125 || FileTextExtractor.ARCHIVE_MIME_TYPES.contains(detectedFormat)
126 || (detectedFormat.startsWith(
"video/") && !detectedFormat.equals(
"video/x-flv"))
130 return TIKA_SUPPORTED_TYPES.contains(detectedFormat);
134 public boolean isDisabled() {
145 private static int getTimeout(
long size) {
146 if (size < 1024 * 1024L)
149 }
else if (size < 10 * 1024 * 1024L)
152 }
else if (size < 100 * 1024 * 1024L)