19 package org.sleuthkit.autopsy.recentactivity;
22 import java.io.FileInputStream;
23 import java.io.FileNotFoundException;
24 import java.io.IOException;
25 import java.nio.file.Path;
26 import java.nio.file.Paths;
27 import java.text.ParseException;
28 import java.text.SimpleDateFormat;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.Collection;
32 import java.util.HashMap;
33 import java.util.List;
34 import java.util.Scanner;
35 import java.util.logging.Level;
36 import org.openide.modules.InstalledFileLocator;
37 import org.openide.util.NbBundle.Messages;
59 final class ExtractEdge
extends Extract {
61 private static final Logger LOG = Logger.getLogger(ExtractEdge.class.getName());
62 private final IngestServices services = IngestServices.getInstance();
63 private final Path moduleTempResultPath;
64 private Content dataSource;
65 private IngestJobContext context;
66 private HashMap<String, ArrayList<String>> containersTable;
68 private static final String EDGE =
"Edge";
70 private static final String EDGE_KEYWORD_VISIT =
"Visited:";
71 private static final String IGNORE_COMMA_IN_QUOTES_REGEX =
",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)";
73 private static final String EDGE_TABLE_TYPE_DOWNLOAD =
"iedownload";
74 private static final String EDGE_TABLE_TYPE_HISTORY =
"History";
75 private static final String EDGE_TABLE_TYPE_COOKIE =
"cookie";
77 private static final String EDGE_HEAD_URL =
"url";
78 private static final String EDGE_HEAD_ACCESSTIME =
"accessedtime";
79 private static final String EDGE_HEAD_NAME =
"name";
80 private static final String EDGE_HEAD_CONTAINER_ID =
"containerid";
81 private static final String EDGE_HEAD_RESPONSEHEAD =
"responseheaders";
82 private static final String EDGE_HEAD_TITLE =
"title";
83 private static final String EDGE_HEAD_RDOMAIN =
"rdomain";
84 private static final String EDGE_HEAD_VALUE =
"value";
85 private static final String EDGE_HEAD_LASTMOD =
"lastmodified";
87 private static final String EDGE_WEBCACHE_PREFIX =
"WebCacheV01";
88 private static final String EDGE_CONTAINER_FILE_PREFIX =
"Container_";
89 private static final String EDGE_CONTAINER_FILE_EXT =
".csv";
90 private static final String EDGE_WEBCACHE_EXT =
".dat";
92 private static final String ESE_TOOL_NAME =
"ESEDatabaseView.exe";
93 private static final String EDGE_WEBCACHE_NAME =
"WebCacheV01.dat";
94 private static final String EDGE_SPARTAN_NAME =
"Spartan.edb";
95 private static final String EDGE_CONTAINTERS_FILE_NAME =
"Containers.csv";
96 private static final String EDGE_FAVORITE_FILE_NAME =
"Favorites.csv";
97 private static final String EDGE_OUTPUT_FILE_NAME =
"Output.txt";
98 private static final String EDGE_ERROR_FILE_NAME =
"File.txt";
99 private static final String EDGE_WEBCACHE_FOLDER_NAME =
"WebCache";
100 private static final String EDGE_SPARTAN_FOLDER_NAME =
"MicrosoftEdge";
102 private static final String ESE_TOOL_FOLDER =
"ESEDatabaseView";
103 private static final String EDGE_RESULT_FOLDER_NAME =
"results";
105 private static final SimpleDateFormat DATE_FORMATTER =
new SimpleDateFormat(
"MM/dd/yyyy hh:mm:ss a");
108 "ExtractEdge_process_errMsg_unableFindESEViewer=Unable to find ESEDatabaseViewer",
109 "ExtractEdge_process_errMsg_errGettingWebCacheFiles=Error trying to retrieving Edge WebCacheV01 file",
110 "ExtractEdge_process_errMsg_webcacheFail=Failure processing Microsoft Edge WebCacheV01.dat file",
111 "ExtractEdge_process_errMsg_spartanFail=Failure processing Microsoft Edge spartan.edb file",
112 "ExtractEdge_Module_Name=Microsoft Edge",
113 "ExtractEdge_getHistory_containerFileNotFound=Error while trying to analyze Edge history",
114 "Progress_Message_Edge_History=Microsoft Edge History",
115 "Progress_Message_Edge_Bookmarks=Microsoft Edge Bookmarks",
116 "Progress_Message_Edge_Cookies=Microsoft Edge Cookies",
122 ExtractEdge() throws NoCurrentCaseException {
123 moduleTempResultPath = Paths.get(RAImageIngestModule.getRATempPath(Case.getCurrentCaseThrows(), EDGE), EDGE_RESULT_FOLDER_NAME);
127 protected String getName() {
128 return Bundle.ExtractEdge_Module_Name();
132 void process(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar) {
133 this.dataSource = dataSource;
134 this.context = context;
135 this.setFoundData(
false);
137 List<AbstractFile> webCacheFiles = null;
138 List<AbstractFile> spartanFiles = null;
141 webCacheFiles = fetchWebCacheDBFiles();
142 }
catch (TskCoreException ex) {
143 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_errGettingWebCacheFiles());
144 LOG.log(Level.SEVERE,
"Error fetching 'WebCacheV01.dat' files for Microsoft Edge", ex);
148 spartanFiles = fetchSpartanDBFiles();
149 }
catch (TskCoreException ex) {
150 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail());
151 LOG.log(Level.SEVERE,
"Error fetching 'spartan.edb' files for Microsoft Edge", ex);
155 if (webCacheFiles == null && spartanFiles == null) {
159 this.setFoundData(
true);
161 if (!PlatformUtil.isWindowsOS()) {
162 LOG.log(Level.WARNING,
"Microsoft Edge files found, unable to parse on Non-Windows system");
166 final String esedumper = getPathForESEDumper();
167 if (esedumper == null) {
168 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_unableFindESEViewer());
169 LOG.log(Level.SEVERE,
"Error finding ESEDatabaseViewer program");
174 this.processWebCacheDbFile(esedumper, webCacheFiles, progressBar);
175 }
catch (IOException | TskCoreException ex) {
176 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_webcacheFail());
177 LOG.log(Level.SEVERE,
"Error returned from processWebCacheDbFile", ex);
180 progressBar.progress(Bundle.Progress_Message_Edge_Bookmarks());
182 this.processSpartanDbFile(esedumper, spartanFiles);
183 }
catch (IOException | TskCoreException ex) {
184 this.addErrorMessage(Bundle.ExtractEdge_process_errMsg_spartanFail());
185 LOG.log(Level.SEVERE,
"Error returned from processSpartanDbFile", ex);
198 void processWebCacheDbFile(String eseDumperPath, List<AbstractFile> webCacheFiles, DataSourceIngestModuleProgress progressBar)
throws IOException, TskCoreException {
200 for (AbstractFile webCacheFile : webCacheFiles) {
202 if (context.dataSourceIngestIsCancelled()) {
206 clearContainerTable();
209 String tempWebCacheFileName = EDGE_WEBCACHE_PREFIX
210 + Integer.toString((
int) webCacheFile.getId()) + EDGE_WEBCACHE_EXT;
211 File tempWebCacheFile =
new File(RAImageIngestModule.getRATempPath(currentCase, EDGE), tempWebCacheFileName);
214 ContentUtils.writeToFile(webCacheFile, tempWebCacheFile,
215 context::dataSourceIngestIsCancelled);
216 }
catch (IOException ex) {
217 throw new IOException(
"Error writingToFile: " + webCacheFile, ex);
220 File resultsDir =
new File(moduleTempResultPath.toAbsolutePath() + Integer.toString((
int) webCacheFile.getId()));
223 executeDumper(eseDumperPath, tempWebCacheFile.getAbsolutePath(),
224 resultsDir.getAbsolutePath());
226 if (context.dataSourceIngestIsCancelled()) {
230 progressBar.progress(Bundle.Progress_Message_Edge_History());
232 this.getHistory(webCacheFile, resultsDir);
234 if (context.dataSourceIngestIsCancelled()) {
238 progressBar.progress(Bundle.Progress_Message_Edge_Cookies());
240 this.getCookies(webCacheFile, resultsDir);
243 tempWebCacheFile.delete();
244 FileUtil.deleteFileDir(resultsDir);
258 void processSpartanDbFile(String eseDumperPath, List<AbstractFile> spartanFiles)
throws IOException, TskCoreException {
260 for (AbstractFile spartanFile : spartanFiles) {
262 if (context.dataSourceIngestIsCancelled()) {
267 String tempSpartanFileName = EDGE_WEBCACHE_PREFIX
268 + Integer.toString((
int) spartanFile.getId()) + EDGE_WEBCACHE_EXT;
269 File tempSpartanFile =
new File(RAImageIngestModule.getRATempPath(currentCase, EDGE), tempSpartanFileName);
272 ContentUtils.writeToFile(spartanFile, tempSpartanFile,
273 context::dataSourceIngestIsCancelled);
274 }
catch (IOException ex) {
275 throw new IOException(
"Error writingToFile: " + spartanFile, ex);
278 File resultsDir =
new File(moduleTempResultPath.toAbsolutePath() + Integer.toString((
int) spartanFile.getId()));
281 executeDumper(eseDumperPath, tempSpartanFile.getAbsolutePath(),
282 resultsDir.getAbsolutePath());
284 if (context.dataSourceIngestIsCancelled()) {
288 this.getBookmarks(spartanFile, resultsDir);
291 tempSpartanFile.delete();
292 FileUtil.deleteFileDir(resultsDir);
306 private void getHistory(AbstractFile origFile, File resultDir)
throws TskCoreException, FileNotFoundException {
307 ArrayList<File> historyFiles = getHistoryFiles(resultDir);
309 if (historyFiles == null) {
313 for (File file : historyFiles) {
316 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
317 }
catch (FileNotFoundException ex) {
318 LOG.log(Level.WARNING,
"Unable to find the ESEDatabaseView file at " + file.getPath(), ex);
322 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
325 List<String> headers = null;
326 while (fileScanner.hasNext()) {
327 String line = fileScanner.nextLine();
328 if (headers == null) {
329 headers = Arrays.asList(line.toLowerCase().split(
","));
333 if (line.contains(EDGE_KEYWORD_VISIT)) {
334 BlackboardArtifact ba = getHistoryArtifact(origFile, headers, line);
337 this.indexArtifact(ba);
345 if (!bbartifacts.isEmpty()) {
346 services.fireModuleDataEvent(
new ModuleDataEvent(
347 RecentActivityExtracterModuleFactory.getModuleName(),
348 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
361 private void getBookmarks(AbstractFile origFile, File resultDir)
throws TskCoreException {
363 File favoriteFile =
new File(resultDir, EDGE_FAVORITE_FILE_NAME);
366 fileScanner =
new Scanner(
new FileInputStream(favoriteFile));
367 }
catch (FileNotFoundException ex) {
373 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
376 List<String> headers = null;
377 while (fileScanner.hasNext()) {
378 String line = fileScanner.nextLine();
379 if (headers == null) {
380 headers = Arrays.asList(line.toLowerCase().split(
","));
384 BlackboardArtifact ba = getBookmarkArtifact(origFile, headers, line);
387 this.indexArtifact(ba);
394 if (!bbartifacts.isEmpty()) {
395 services.fireModuleDataEvent(
new ModuleDataEvent(
396 RecentActivityExtracterModuleFactory.getModuleName(),
397 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
408 private void getCookies(AbstractFile origFile, File resultDir)
throws TskCoreException {
409 File containerFiles[] = resultDir.listFiles((dir, name) -> name.toLowerCase().contains(EDGE_TABLE_TYPE_COOKIE));
411 if (containerFiles == null) {
415 for (File file : containerFiles) {
418 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
419 }
catch (FileNotFoundException ex) {
420 LOG.log(Level.WARNING,
"Unable to find the ESEDatabaseView file at " + file.getPath(), ex);
424 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
427 List<String> headers = null;
428 while (fileScanner.hasNext()) {
429 String line = fileScanner.nextLine();
430 if (headers == null) {
431 headers = Arrays.asList(line.toLowerCase().split(
","));
435 BlackboardArtifact ba = getCookieArtifact(origFile, headers, line);
438 this.indexArtifact(ba);
445 if (!bbartifacts.isEmpty()) {
446 services.fireModuleDataEvent(
new ModuleDataEvent(
447 RecentActivityExtracterModuleFactory.getModuleName(),
448 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY, bbartifacts));
463 private void getDownloads(AbstractFile origFile, File resultDir)
throws TskCoreException, FileNotFoundException {
464 ArrayList<File> downloadFiles = getDownloadFiles(resultDir);
466 if (downloadFiles == null) {
470 for (File file : downloadFiles) {
473 fileScanner =
new Scanner(
new FileInputStream(file.toString()));
474 }
catch (FileNotFoundException ex) {
475 LOG.log(Level.WARNING,
"Unable to find the ESEDatabaseView file at " + file.getPath(), ex);
478 Collection<BlackboardArtifact> bbartifacts =
new ArrayList<>();
481 List<String> headers = null;
482 while (fileScanner.hasNext()) {
483 String line = fileScanner.nextLine();
484 if (headers == null) {
485 headers = Arrays.asList(line.toLowerCase().split(
","));
489 if (line.contains(EDGE_TABLE_TYPE_DOWNLOAD)) {
491 BlackboardArtifact ba = getDownloadArtifact(origFile, headers, line);
494 this.indexArtifact(ba);
502 if (!bbartifacts.isEmpty()) {
503 services.fireModuleDataEvent(
new ModuleDataEvent(
504 RecentActivityExtracterModuleFactory.getModuleName(),
505 BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_DOWNLOAD, bbartifacts));
515 private String getPathForESEDumper() {
516 Path path = Paths.get(ESE_TOOL_FOLDER, ESE_TOOL_NAME);
517 File eseToolFile = InstalledFileLocator.getDefault().locate(path.toString(),
518 ExtractEdge.class.getPackage().getName(),
false);
519 if (eseToolFile != null) {
520 return eseToolFile.getAbsolutePath();
532 private List<AbstractFile> fetchWebCacheDBFiles() throws TskCoreException {
534 = currentCase.getServices().getFileManager();
535 return fileManager.
findFiles(dataSource, EDGE_WEBCACHE_NAME, EDGE_WEBCACHE_FOLDER_NAME);
544 private List<AbstractFile> fetchSpartanDBFiles() throws TskCoreException {
546 = currentCase.getServices().getFileManager();
547 return fileManager.
findFiles(dataSource, EDGE_SPARTAN_NAME, EDGE_SPARTAN_FOLDER_NAME);
561 private void executeDumper(String dumperPath, String inputFilePath,
562 String outputDir)
throws IOException {
564 final Path outputFilePath = Paths.get(outputDir, EDGE_OUTPUT_FILE_NAME);
565 final Path errFilePath = Paths.get(outputDir, EDGE_ERROR_FILE_NAME);
566 LOG.log(Level.INFO,
"Writing ESEDatabaseViewer results to: {0}", outputDir);
568 List<String> commandLine =
new ArrayList<>();
569 commandLine.add(dumperPath);
570 commandLine.add(
"/table");
571 commandLine.add(inputFilePath);
572 commandLine.add(
"*");
573 commandLine.add(
"/scomma");
574 commandLine.add(outputDir +
"\\" +
"*.csv");
576 ProcessBuilder processBuilder =
new ProcessBuilder(commandLine);
577 processBuilder.redirectOutput(outputFilePath.toFile());
578 processBuilder.redirectError(errFilePath.toFile());
580 ExecUtil.execute(processBuilder,
new DataSourceIngestModuleProcessTerminator(context));
593 private BlackboardArtifact getHistoryArtifact(AbstractFile origFile, List<String> headers, String line)
throws TskCoreException {
594 String[] rowSplit = line.split(
",");
596 int index = headers.indexOf(EDGE_HEAD_URL);
597 String urlUserStr = rowSplit[index];
599 String[] str = urlUserStr.split(
"@");
600 String user = (str[0].replace(EDGE_KEYWORD_VISIT,
"")).trim();
603 index = headers.indexOf(EDGE_HEAD_ACCESSTIME);
604 String accessTime = rowSplit[index].trim();
607 Long epochtime = DATE_FORMATTER.parse(accessTime).getTime();
608 ftime = epochtime / 1000;
609 }
catch (ParseException ex) {
610 LOG.log(Level.WARNING,
"The Accessed Time format in history file seems invalid " + accessTime, ex);
613 BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY);
615 bbart.addAttributes(createHistoryAttribute(url, ftime,
618 NetworkUtils.extractDomain(url), user));
632 private BlackboardArtifact getCookieArtifact(AbstractFile origFile, List<String> headers, String line)
throws TskCoreException {
633 String[] lineSplit = line.split(
",");
635 String accessTime = lineSplit[headers.indexOf(EDGE_HEAD_LASTMOD)].trim();
638 Long epochtime = DATE_FORMATTER.parse(accessTime).getTime();
639 ftime = epochtime / 1000;
640 }
catch (ParseException ex) {
641 LOG.log(Level.WARNING,
"The Accessed Time format in history file seems invalid " + accessTime, ex);
644 String domain = lineSplit[headers.indexOf(EDGE_HEAD_RDOMAIN)].trim();
645 String name = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_NAME)].trim());
646 String value = hexToChar(lineSplit[headers.indexOf(EDGE_HEAD_VALUE)].trim());
647 String url = flipDomain(domain);
649 BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_COOKIE);
650 bbart.addAttributes(createCookieAttributes(url, ftime, name, value, this.getName(), NetworkUtils.extractDomain(url)));
667 private BlackboardArtifact getDownloadArtifact(AbstractFile origFile, List<String> headers, String line)
throws TskCoreException {
668 BlackboardArtifact bbart = null;
670 String[] lineSplit = line.split(
",");
671 String rheader = lineSplit[headers.indexOf(EDGE_HEAD_RESPONSEHEAD)];
688 private BlackboardArtifact getBookmarkArtifact(AbstractFile origFile, List<String> headers, String line)
throws TskCoreException {
690 String[] lineSplit = line.split(IGNORE_COMMA_IN_QUOTES_REGEX, -1);
692 String url = lineSplit[headers.indexOf(EDGE_HEAD_URL)];
693 String title = lineSplit[headers.indexOf(EDGE_HEAD_TITLE)].replace(
"\"",
"");
699 BlackboardArtifact bbart = origFile.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_BOOKMARK);
700 bbart.addAttributes(createBookmarkAttributes(url, title, null,
701 this.getName(), NetworkUtils.extractDomain(url)));
711 private String hexToChar(String hexString) {
712 String[] hexValues = hexString.split(
" ");
713 StringBuilder output =
new StringBuilder();
715 for (String str : hexValues) {
717 int value = Integer.parseInt(str, 16);
719 output.append((
char) value);
721 }
catch (NumberFormatException ex) {
726 return output.toString();
740 private String flipDomain(String domain) {
741 if (domain == null || domain.isEmpty()) {
745 String[] tokens = domain.split(
"\\.");
747 if (tokens.length < 2 || tokens.length > 3) {
751 StringBuilder buf =
new StringBuilder();
752 if (tokens.length > 2) {
753 buf.append(tokens[2]);
756 buf.append(tokens[1]);
758 buf.append(tokens[0]);
760 return buf.toString();
770 private ArrayList<File> getDownloadFiles(File resultDir)
throws FileNotFoundException {
771 return getContainerFiles(resultDir, EDGE_TABLE_TYPE_DOWNLOAD);
781 private ArrayList<File> getHistoryFiles(File resultDir)
throws FileNotFoundException {
782 return getContainerFiles(resultDir, EDGE_TABLE_TYPE_HISTORY);
793 private ArrayList<File> getContainerFiles(File resultDir, String type)
throws FileNotFoundException {
794 HashMap<String, ArrayList<String>> idTable = getContainerIDTable(resultDir);
796 ArrayList<String> idList = idTable.get(type);
797 if (idList == null) {
801 ArrayList<File> fileList =
new ArrayList<>();
802 for (String str : idList) {
803 String fileName = EDGE_CONTAINER_FILE_PREFIX + str + EDGE_CONTAINER_FILE_EXT;
804 fileList.add(
new File(resultDir, fileName));
820 private HashMap<String, ArrayList<String>> getContainerIDTable(File resultDir)
throws FileNotFoundException {
822 if (containersTable == null) {
823 File containerFile =
new File(resultDir, EDGE_CONTAINTERS_FILE_NAME);
825 try (Scanner fileScanner =
new Scanner(
new FileInputStream(containerFile))) {
826 List<String> headers = null;
827 containersTable =
new HashMap<>();
830 while (fileScanner.hasNext()) {
831 String line = fileScanner.nextLine();
832 if (headers == null) {
833 headers = Arrays.asList(line.toLowerCase().split(
","));
834 nameIdx = headers.indexOf(EDGE_HEAD_NAME);
835 idIdx = headers.indexOf(EDGE_HEAD_CONTAINER_ID);
837 String[] row = line.split(
",");
838 String name = row[nameIdx];
839 String
id = row[idIdx];
841 ArrayList<String> idList = containersTable.get(name);
842 if (idList == null) {
843 idList =
new ArrayList<>();
844 containersTable.put(name, idList);
853 return containersTable;
859 private void clearContainerTable(){
860 containersTable = null;
synchronized List< AbstractFile > findFiles(String fileName)