23 package org.sleuthkit.autopsy.recentactivity;
25 import java.io.BufferedReader;
26 import org.openide.util.NbBundle;
30 import java.io.FileInputStream;
31 import java.io.FileNotFoundException;
32 import java.io.IOException;
33 import java.io.InputStreamReader;
34 import java.text.ParseException;
35 import java.text.SimpleDateFormat;
36 import java.util.ArrayList;
37 import java.util.List;
38 import java.util.logging.Level;
40 import java.util.Collection;
41 import java.util.Scanner;
42 import java.util.stream.Collectors;
43 import org.openide.modules.InstalledFileLocator;
44 import org.openide.util.NbBundle.Messages;
51 import org.
sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
53 import org.
sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
65 class ExtractIE
extends Extract {
67 private static final Logger logger = Logger.getLogger(ExtractIE.class.getName());
68 private final IngestServices services = IngestServices.getInstance();
69 private final String moduleTempResultsDir;
70 private String PASCO_LIB_PATH;
71 private final String JAVA_PATH;
72 private static final String RESOURCE_URL_PREFIX =
"res://";
73 private static final SimpleDateFormat dateFormatter =
new SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
74 private Content dataSource;
75 private IngestJobContext context;
78 "Progress_Message_IE_History=IE History",
79 "Progress_Message_IE_Bookmarks=IE Bookmarks",
80 "Progress_Message_IE_Cookies=IE Cookies",
81 "Progress_Message_IE_Downloads=IE Downloads",
82 "Progress_Message_IE_FormHistory=IE Form History",
83 "Progress_Message_IE_AutoFill=IE Auto Fill",
84 "Progress_Message_IE_Logins=IE Logins",
87 ExtractIE() throws NoCurrentCaseException {
88 moduleName = NbBundle.getMessage(ExtractIE.class,
"ExtractIE.moduleName.text");
89 moduleTempResultsDir = RAImageIngestModule.getRATempPath(Case.getCurrentCaseThrows(),
"IE") + File.separator +
"results";
90 JAVA_PATH = PlatformUtil.getJavaPath();
94 public void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
95 this.dataSource = dataSource;
96 this.context = context;
99 progressBar.progress(Bundle.Progress_Message_IE_Bookmarks());
102 progressBar.progress(Bundle.Progress_Message_IE_Cookies());
105 progressBar.progress(Bundle.Progress_Message_IE_History());
112 private void getBookmark() {
114 List<AbstractFile> favoritesFiles;
116 favoritesFiles = fileManager.
findFiles(dataSource,
"%.url",
"Favorites");
117 }
catch (TskCoreException ex) {
118 logger.log(Level.WARNING,
"Error fetching 'url' files for Internet Explorer bookmarks.", ex);
119 this.addErrorMessage(
120 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getBookmark.errMsg.errGettingBookmarks",
125 if (favoritesFiles.isEmpty()) {
126 logger.log(Level.INFO,
"Didn't find any IE bookmark files.");
131 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
132 for (AbstractFile fav : favoritesFiles) {
133 if (fav.getSize() == 0) {
137 if (context.dataSourceIngestIsCancelled()) {
141 String url = getURLFromIEBookmarkFile(fav);
143 String name = fav.getName();
144 Long datetime = fav.getCrtime();
145 String Tempdate = datetime.toString();
146 datetime = Long.valueOf(Tempdate);
147 String domain = extractDomain(url);
149 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
150 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
151 RecentActivityExtracterModuleFactory.getModuleName(), url));
152 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
153 RecentActivityExtracterModuleFactory.getModuleName(), name));
154 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
155 RecentActivityExtracterModuleFactory.getModuleName(), datetime));
156 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
157 RecentActivityExtracterModuleFactory.getModuleName(),
158 NbBundle.getMessage(this.getClass(),
"ExtractIE.moduleName.text")));
159 if (domain != null && domain.isEmpty() ==
false) {
160 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
161 RecentActivityExtracterModuleFactory.getModuleName(), domain));
164 BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_BOOKMARK, fav, bbattributes);
166 bbartifacts.add(bbart);
169 services.fireModuleDataEvent(
new ModuleDataEvent(
170 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName"),
171 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK, bbartifacts));
174 private String getURLFromIEBookmarkFile(AbstractFile fav) {
175 BufferedReader reader =
new BufferedReader(
new InputStreamReader(
new ReadContentInputStream(fav)));
176 String line, url =
"";
178 line = reader.readLine();
179 while (null != line) {
182 if (line.startsWith(
"URL")) {
183 url = line.substring(line.indexOf(
"=") + 1);
186 line = reader.readLine();
188 }
catch (IOException ex) {
189 logger.log(Level.WARNING,
"Failed to read from content: " + fav.getName(), ex);
190 this.addErrorMessage(
191 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getURLFromIEBmkFile.errMsg", this.getName(),
193 }
catch (IndexOutOfBoundsException ex) {
194 logger.log(Level.WARNING,
"Failed while getting URL of IE bookmark. Unexpected format of the bookmark file: " + fav.getName(), ex);
195 this.addErrorMessage(
196 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getURLFromIEBmkFile.errMsg2", this.getName(),
201 }
catch (IOException ex) {
202 logger.log(Level.WARNING,
"Failed to close reader.", ex);
212 private void getCookie() {
214 List<AbstractFile> cookiesFiles;
216 cookiesFiles = fileManager.
findFiles(dataSource,
"%.txt",
"Cookies");
217 }
catch (TskCoreException ex) {
218 logger.log(Level.WARNING,
"Error getting cookie files for IE");
219 this.addErrorMessage(
220 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getCookie.errMsg.errGettingFile", this.getName()));
224 if (cookiesFiles.isEmpty()) {
225 logger.log(Level.INFO,
"Didn't find any IE cookies files.");
230 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
231 for (AbstractFile cookiesFile : cookiesFiles) {
232 if (context.dataSourceIngestIsCancelled()) {
235 if (cookiesFile.getSize() == 0) {
239 byte[] t =
new byte[(int) cookiesFile.getSize()];
241 final int bytesRead = cookiesFile.read(t, 0, cookiesFile.getSize());
242 }
catch (TskCoreException ex) {
243 logger.log(Level.WARNING,
"Error reading bytes of Internet Explorer cookie.", ex);
244 this.addErrorMessage(
245 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getCookie.errMsg.errReadingIECookie",
246 this.getName(), cookiesFile.getName()));
249 String cookieString =
new String(t);
250 String[] values = cookieString.split(
"\n");
251 String url = values.length > 2 ? values[2] :
"";
252 String value = values.length > 1 ? values[1] :
"";
253 String name = values.length > 0 ? values[0] :
"";
254 Long datetime = cookiesFile.getCrtime();
255 String tempDate = datetime.toString();
256 datetime = Long.valueOf(tempDate);
257 String domain = extractDomain(url);
259 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
260 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
261 RecentActivityExtracterModuleFactory.getModuleName(), url));
262 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME,
263 RecentActivityExtracterModuleFactory.getModuleName(), datetime));
264 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
265 RecentActivityExtracterModuleFactory.getModuleName(), (name != null) ? name :
""));
266 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
267 RecentActivityExtracterModuleFactory.getModuleName(), value));
268 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
269 RecentActivityExtracterModuleFactory.getModuleName(),
270 NbBundle.getMessage(this.getClass(),
"ExtractIE.moduleName.text")));
271 if (domain != null && domain.isEmpty() ==
false) {
272 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
273 RecentActivityExtracterModuleFactory.getModuleName(), domain));
275 BlackboardArtifact bbart = this.addArtifact(ARTIFACT_TYPE.TSK_WEB_COOKIE, cookiesFile, bbattributes);
277 bbartifacts.add(bbart);
280 services.fireModuleDataEvent(
new ModuleDataEvent(
281 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName"),
282 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE, bbartifacts));
288 private void getHistory() {
289 logger.log(Level.INFO,
"Pasco results path: {0}", moduleTempResultsDir);
290 boolean foundHistory =
false;
292 final File pascoRoot = InstalledFileLocator.getDefault().locate(
"pasco2", ExtractIE.class.getPackage().getName(),
false);
293 if (pascoRoot == null) {
294 this.addErrorMessage(
295 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.unableToGetHist", this.getName()));
296 logger.log(Level.SEVERE,
"Error finding pasco program ");
300 final String pascoHome = pascoRoot.getAbsolutePath();
301 logger.log(Level.INFO,
"Pasco2 home: {0}", pascoHome);
303 PASCO_LIB_PATH = pascoHome + File.separator +
"pasco2.jar" + File.pathSeparator
304 + pascoHome + File.separator +
"*";
306 File resultsDir =
new File(moduleTempResultsDir);
311 List<AbstractFile> indexFiles;
313 indexFiles = fileManager.
findFiles(dataSource,
"index.dat");
314 }
catch (TskCoreException ex) {
315 this.addErrorMessage(NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errGettingHistFiles",
317 logger.log(Level.WARNING,
"Error fetching 'index.data' files for Internet Explorer history.");
321 if (indexFiles.isEmpty()) {
322 String msg = NbBundle.getMessage(this.getClass(),
"ExtractIE.getHistory.errMsg.noHistFiles");
323 logger.log(Level.INFO, msg);
328 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
330 String indexFileName;
331 for (AbstractFile indexFile : indexFiles) {
337 indexFileName =
"index" + Integer.toString((
int) indexFile.getId()) +
".dat";
339 temps = RAImageIngestModule.getRATempPath(currentCase,
"IE") + File.separator + indexFileName;
340 File datFile =
new File(temps);
341 if (context.dataSourceIngestIsCancelled()) {
345 ContentUtils.writeToFile(indexFile, datFile, context::dataSourceIngestIsCancelled);
346 }
catch (IOException e) {
347 logger.log(Level.WARNING,
"Error while trying to write index.dat file " + datFile.getAbsolutePath(), e);
348 this.addErrorMessage(
349 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errWriteFile", this.getName(),
350 datFile.getAbsolutePath()));
354 String filename =
"pasco2Result." + indexFile.getId() +
".txt";
355 boolean bPascProcSuccess = executePasco(temps, filename);
356 if (context.dataSourceIngestIsCancelled()) {
362 if (bPascProcSuccess) {
364 bbartifacts.addAll(parsePascoOutput(indexFile, filename).stream()
365 .filter(bbart -> bbart.getArtifactTypeID() == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID())
366 .collect(Collectors.toList()));
367 if (context.dataSourceIngestIsCancelled()) {
375 logger.log(Level.WARNING,
"pasco execution failed on: {0}",
this.getName());
376 this.addErrorMessage(
377 NbBundle.getMessage(
this.getClass(),
"ExtractIE.getHistory.errMsg.errProcHist", this.getName()));
382 services.fireModuleDataEvent(
new ModuleDataEvent(
383 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parentModuleName"),
384 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
396 private boolean executePasco(String indexFilePath, String outputFileName) {
397 boolean success =
true;
399 final String outputFileFullPath = moduleTempResultsDir + File.separator + outputFileName;
400 final String errFileFullPath = moduleTempResultsDir + File.separator + outputFileName +
".err";
401 logger.log(Level.INFO,
"Writing pasco results to: {0}", outputFileFullPath);
402 List<String> commandLine =
new ArrayList<>();
403 commandLine.add(JAVA_PATH);
404 commandLine.add(
"-cp");
405 commandLine.add(PASCO_LIB_PATH);
406 commandLine.add(
"isi.pasco2.Main");
407 commandLine.add(
"-T");
408 commandLine.add(
"history");
409 commandLine.add(indexFilePath);
410 ProcessBuilder processBuilder =
new ProcessBuilder(commandLine);
411 processBuilder.redirectOutput(
new File(outputFileFullPath));
412 processBuilder.redirectError(
new File(errFileFullPath));
422 ExecUtil.execute(processBuilder,
new DataSourceIngestModuleProcessTerminator(context));
424 }
catch (IOException ex) {
426 logger.log(Level.SEVERE,
"Unable to execute Pasco to process Internet Explorer web history.", ex);
440 private Collection<BlackboardArtifact> parsePascoOutput(AbstractFile origFile, String pascoOutputFileName) {
442 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
443 String fnAbs = moduleTempResultsDir + File.separator + pascoOutputFileName;
445 File file =
new File(fnAbs);
446 if (file.exists() ==
false) {
447 this.addErrorMessage(
448 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.notFound", this.getName(),
450 logger.log(Level.WARNING,
"Pasco Output not found: {0}", file.getPath());
456 if (file.length() == 0) {
462 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
463 }
catch (FileNotFoundException ex) {
464 this.addErrorMessage(
465 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.errParsing", this.getName(),
467 logger.log(Level.WARNING,
"Unable to find the Pasco file at " + file.getPath(), ex);
470 while (fileScanner.hasNext()) {
472 if (context.dataSourceIngestIsCancelled()) {
476 String line = fileScanner.nextLine();
477 if (!line.startsWith(
"URL")) {
481 String[] lineBuff = line.split(
"\\t");
483 if (lineBuff.length < 4) {
484 logger.log(Level.INFO,
"Found unrecognized IE history format.");
488 String actime = lineBuff[3];
489 Long ftime = (long) 0;
491 String realurl = null;
498 if (lineBuff[1].contains(
"@")) {
499 String url[] = lineBuff[1].split(
"@", 2);
504 domain = extractDomain(url[0]);
506 if (domain != null && domain.isEmpty() ==
false) {
510 realurl = lineBuff[1].trim();
517 user = user.replace(
"Visited:",
"");
518 user = user.replace(
":Host:",
"");
519 user = user.replaceAll(
"(:)(.*?)(:)",
"");
522 realurl = realurl.replace(
"Visited:",
"");
523 realurl = realurl.replaceAll(
":(.*?):",
"");
524 realurl = realurl.replace(
":Host:",
"");
525 realurl = realurl.trim();
526 domain = extractDomain(realurl);
532 realurl = lineBuff[1].trim();
533 domain = extractDomain(realurl);
536 if (!actime.isEmpty()) {
538 Long epochtime = dateFormatter.parse(actime).getTime();
539 ftime = epochtime / 1000;
540 }
catch (ParseException e) {
541 this.addErrorMessage(
542 NbBundle.getMessage(
this.getClass(),
"ExtractIE.parsePascoOutput.errMsg.errParsingEntry",
544 logger.log(Level.WARNING, String.format(
"Error parsing Pasco results, may have partial processing of corrupt file (id=%d)", origFile.getId()), e);
549 BlackboardArtifact bbart = origFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY);
550 Collection<BlackboardAttribute> bbattributes =
new ArrayList<>();
551 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
552 RecentActivityExtracterModuleFactory.getModuleName(), realurl));
555 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
556 RecentActivityExtracterModuleFactory.getModuleName(), ftime));
557 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_REFERRER,
558 RecentActivityExtracterModuleFactory.getModuleName(),
""));
560 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
561 RecentActivityExtracterModuleFactory.getModuleName(),
562 NbBundle.getMessage(this.getClass(),
563 "ExtractIE.moduleName.text")));
564 if (domain != null && domain.isEmpty() ==
false) {
565 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
566 RecentActivityExtracterModuleFactory.getModuleName(), domain));
568 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_USER_NAME,
569 RecentActivityExtracterModuleFactory.getModuleName(), user));
570 bbart.addAttributes(bbattributes);
573 this.indexArtifact(bbart);
574 bbartifacts.add(bbart);
575 }
catch (TskCoreException ex) {
576 logger.log(Level.SEVERE,
"Error writing Internet Explorer web history artifact to the blackboard. Pasco results will be incomplete", ex);
591 private String extractDomain(String url) {
592 if (url == null || url.isEmpty()) {
596 if (url.toLowerCase().startsWith(RESOURCE_URL_PREFIX)) {
603 return NetworkUtils.extractDomain(url);
synchronized List< AbstractFile > findFiles(String fileName)