19 package org.sleuthkit.autopsy.keywordsearch;
21 import com.google.common.io.CharSource;
22 import java.io.IOException;
23 import java.io.PushbackReader;
24 import java.io.Reader;
25 import java.util.List;
26 import java.util.concurrent.ExecutorService;
27 import java.util.concurrent.Executors;
28 import java.util.concurrent.Future;
29 import java.util.concurrent.TimeUnit;
30 import java.util.concurrent.TimeoutException;
31 import java.util.logging.Level;
32 import java.util.stream.Collectors;
33 import java.util.stream.Stream;
34 import org.apache.tika.Tika;
35 import org.apache.tika.metadata.Metadata;
36 import org.apache.tika.parser.AutoDetectParser;
37 import org.apache.tika.parser.ParseContext;
38 import org.apache.tika.parser.Parser;
39 import org.apache.tika.parser.ParsingReader;
40 import org.apache.tika.parser.microsoft.OfficeParserConfig;
41 import org.openide.util.NbBundle;
50 class TikaTextExtractor
extends ContentTextExtractor {
52 static final private Logger logger = Logger.getLogger(TikaTextExtractor.class.getName());
53 private final ExecutorService tikaParseExecutor = Executors.newSingleThreadExecutor();
55 private final AutoDetectParser parser =
new AutoDetectParser();
57 private static final List<String> TIKA_SUPPORTED_TYPES
58 =
new Tika().getParser().getSupportedTypes(
new ParseContext())
60 .map(mt -> mt.getType() +
"/" + mt.getSubtype())
61 .collect(Collectors.toList());
64 public void logWarning(
final String msg, Exception ex) {
65 KeywordSearch.getTikaLogger().log(Level.WARNING, msg, ex);
69 public Reader getReader(Content content)
throws TextExtractorException {
70 ReadContentInputStream stream =
new ReadContentInputStream(content);
72 Metadata metadata =
new Metadata();
73 ParseContext parseContext =
new ParseContext();
74 parseContext.set(Parser.class, parser);
78 OfficeParserConfig officeParserConfig =
new OfficeParserConfig();
79 officeParserConfig.setUseSAXPptxExtractor(
true);
80 officeParserConfig.setUseSAXDocxExtractor(
true);
81 parseContext.set(OfficeParserConfig.class, officeParserConfig);
84 final Future<Reader> future = tikaParseExecutor.submit(() ->
new ParsingReader(parser, stream, metadata, parseContext));
86 final Reader tikaReader = future.get(getTimeout(content.getSize()), TimeUnit.SECONDS);
89 PushbackReader pushbackReader =
new PushbackReader(tikaReader);
90 int read = pushbackReader.read();
92 throw new TextExtractorException(
"Unable to extract text: Tika returned empty reader for " + content);
94 pushbackReader.unread(read);
97 CharSource metaDataCharSource = getMetaDataCharSource(metadata);
98 return CharSource.concat(
new ReaderCharSource(pushbackReader), metaDataCharSource).openStream();
99 }
catch (TimeoutException te) {
100 final String msg = NbBundle.getMessage(this.getClass(),
"AbstractFileTikaTextExtract.index.tikaParseTimeout.text", content.getId(), content.getName());
102 throw new TextExtractorException(msg, te);
103 }
catch (TextExtractorException ex) {
105 }
catch (Exception ex) {
106 KeywordSearch.getTikaLogger().log(Level.WARNING,
"Exception: Unable to Tika parse the content" + content.getId() +
": " + content.getName(), ex.getCause());
107 final String msg = NbBundle.getMessage(this.getClass(),
"AbstractFileTikaTextExtract.index.exception.tikaParse.msg", content.getId(), content.getName());
109 throw new TextExtractorException(msg, ex);
123 static private CharSource getMetaDataCharSource(Metadata metadata) {
124 return CharSource.wrap(
125 new StringBuilder(
"\n\n------------------------------METADATA------------------------------\n\n")
126 .append(Stream.of(metadata.names()).sorted()
127 .map(key -> key +
": " + metadata.get(key))
128 .collect(Collectors.joining(
"\n"))
133 public boolean isContentTypeSpecific() {
138 public boolean isSupported(Content content, String detectedFormat) {
139 if (detectedFormat == null
140 || ContentTextExtractor.BLOB_MIME_TYPES.contains(detectedFormat)
141 || ContentTextExtractor.ARCHIVE_MIME_TYPES.contains(detectedFormat)
142 || (detectedFormat.startsWith(
"video/") && !detectedFormat.equals(
"video/x-flv"))
146 return TIKA_SUPPORTED_TYPES.contains(detectedFormat);
150 public boolean isDisabled() {
161 private static int getTimeout(
long size) {
162 if (size < 1024 * 1024L)
165 }
else if (size < 10 * 1024 * 1024L)
168 }
else if (size < 100 * 1024 * 1024L)