23 package org.sleuthkit.autopsy.recentactivity;
 
   25 import com.google.gson.JsonArray;
 
   26 import com.google.gson.JsonElement;
 
   27 import com.google.gson.JsonIOException;
 
   28 import com.google.gson.JsonObject;
 
   29 import com.google.gson.JsonParser;
 
   30 import com.google.gson.JsonSyntaxException;
 
   32 import java.io.FileNotFoundException;
 
   33 import java.io.FileReader;
 
   34 import java.io.IOException;
 
   35 import java.io.UnsupportedEncodingException;
 
   36 import java.net.URLDecoder;
 
   37 import java.util.ArrayList;
 
   38 import java.util.Arrays;
 
   39 import java.util.Collection;
 
   40 import java.util.HashMap;
 
   41 import java.util.HashSet;
 
   42 import java.util.List;
 
   44 import java.util.logging.Level;
 
   45 import org.apache.commons.io.FilenameUtils;
 
   46 import org.openide.util.NbBundle;
 
   47 import org.openide.util.NbBundle.Messages;
 
   67     "Progress_Message_Firefox_History=Firefox History",
 
   68     "Progress_Message_Firefox_Bookmarks=Firefox Bookmarks",
 
   69     "Progress_Message_Firefox_Cookies=Firefox Cookies",
 
   70     "Progress_Message_Firefox_Downloads=Firefox Downloads",
 
   71     "Progress_Message_Firefox_FormHistory=Firefox Form History",
 
   72     "Progress_Message_Firefox_AutoFill=Firefox Auto Fill" 
   78 class Firefox extends Extract {
 
   80     private static final Logger logger = Logger.getLogger(Firefox.class.getName());
 
   81     private static final String PLACE_URL_PREFIX = 
"place:";
 
   82     private static final String HISTORY_QUERY = 
"SELECT moz_historyvisits.id, url, title, visit_count,(visit_date/1000000) AS visit_date,from_visit," 
   83             + 
"(SELECT url FROM moz_historyvisits history, moz_places places where history.id = moz_historyvisits.from_visit and history.place_id = places.id ) as ref " 
   84             + 
"FROM moz_places, moz_historyvisits " 
   85             + 
"WHERE moz_places.id = moz_historyvisits.place_id " 
   87     private static final String COOKIE_QUERY = 
"SELECT name,value,host,expiry,(lastAccessed/1000000) AS lastAccessed,(creationTime/1000000) AS creationTime FROM moz_cookies"; 
 
   88     private static final String COOKIE_QUERY_V3 = 
"SELECT name,value,host,expiry,(lastAccessed/1000000) AS lastAccessed FROM moz_cookies"; 
 
   89     private static final String BOOKMARK_QUERY = 
"SELECT fk, moz_bookmarks.title, url, (moz_bookmarks.dateAdded/1000000) AS dateAdded FROM moz_bookmarks INNER JOIN moz_places ON moz_bookmarks.fk=moz_places.id"; 
 
   90     private static final String DOWNLOAD_QUERY = 
"SELECT target, source,(startTime/1000000) AS startTime, maxBytes FROM moz_downloads"; 
 
   91     private static final String DOWNLOAD_QUERY_V24 = 
"SELECT url, content AS target, (lastModified/1000000) AS lastModified " 
   92             + 
" FROM moz_places, moz_annos, moz_anno_attributes " 
   93             + 
" WHERE moz_places.id = moz_annos.place_id" 
   94             + 
" AND moz_annos.anno_attribute_id = moz_anno_attributes.id" 
   95             + 
" AND moz_anno_attributes.name='downloads/destinationFileURI'"; 
 
   96     private static final String FORMHISTORY_QUERY = 
"SELECT fieldname, value FROM moz_formhistory";
 
   97     private static final String FORMHISTORY_QUERY_V64 = 
"SELECT fieldname, value, timesUsed, firstUsed, lastUsed FROM moz_formhistory";
 
   98     private Content dataSource;
 
   99     private final IngestJobContext context;
 
  101     Firefox(IngestJobContext context) {
 
  102         super(NbBundle.getMessage(Firefox.class, 
"Firefox.moduleName"), context);
 
  103         this.context = context;
 
  107     public void process(Content dataSource, DataSourceIngestModuleProgress progressBar) {
 
  108         this.dataSource = dataSource;
 
  110         long ingestJobId = context.getJobId();
 
  112         progressBar.progress(Bundle.Progress_Message_Firefox_History());
 
  113         this.getHistory(context.getJobId());
 
  115         if (context.dataSourceIngestIsCancelled()) {
 
  119         progressBar.progress(Bundle.Progress_Message_Firefox_Bookmarks());
 
  120         this.getBookmark(ingestJobId);
 
  122         if (context.dataSourceIngestIsCancelled()) {
 
  126         progressBar.progress(Bundle.Progress_Message_Firefox_Downloads());
 
  127         this.getDownload(ingestJobId);
 
  129         if (context.dataSourceIngestIsCancelled()) {
 
  133         progressBar.progress(Bundle.Progress_Message_Firefox_Cookies());
 
  134         this.getCookie(ingestJobId);
 
  136         if (context.dataSourceIngestIsCancelled()) {
 
  140         progressBar.progress(Bundle.Progress_Message_Firefox_FormHistory());
 
  141         this.getFormsHistory(ingestJobId);
 
  143         if (context.dataSourceIngestIsCancelled()) {
 
  147         progressBar.progress(Bundle.Progress_Message_Firefox_AutoFill());
 
  148         this.getAutofillProfiles(ingestJobId);
 
  156     private void getHistory(
long ingestJobId) {
 
  157         FileManager fileManager = currentCase.getServices().getFileManager();
 
  158         List<AbstractFile> historyFiles;
 
  160             historyFiles = fileManager.findFiles(dataSource, 
"places.sqlite", 
"Firefox"); 
 
  161         } 
catch (TskCoreException ex) {
 
  162             String msg = NbBundle.getMessage(this.getClass(), 
"Firefox.getHistory.errMsg.errFetchingFiles");
 
  163             logger.log(Level.WARNING, msg);
 
  164             this.addErrorMessage(this.getDisplayName() + 
": " + msg);
 
  168         if (historyFiles.isEmpty()) {
 
  169             String msg = NbBundle.getMessage(this.getClass(), 
"Firefox.getHistory.errMsg.noFilesFound");
 
  170             logger.log(Level.INFO, msg);
 
  175         Collection<BlackboardArtifact> bbartifacts = 
new ArrayList<>();
 
  177         for (AbstractFile historyFile : historyFiles) {
 
  179             if (context.dataSourceIngestIsCancelled()) {
 
  183             if (historyFile.getSize() == 0) {
 
  187             String fileName = historyFile.getName();
 
  188             String temps = RAImageIngestModule.getRATempPath(currentCase, 
"firefox", ingestJobId) + File.separator + fileName + j + 
".db"; 
 
  190                 ContentUtils.writeToFile(historyFile, 
new File(temps), context::dataSourceIngestIsCancelled);
 
  191             } 
catch (ReadContentInputStreamException ex) {
 
  192                 logger.log(Level.WARNING, String.format(
"Error reading Firefox web history artifacts file '%s' (id=%d).",
 
  193                         fileName, historyFile.getId()), ex); 
 
  194                 this.addErrorMessage(
 
  195                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getHistory.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  198             } 
catch (IOException ex) {
 
  199                 logger.log(Level.SEVERE, String.format(
"Error writing temp sqlite db file '%s' for Firefox web history artifacts file '%s' (id=%d).",
 
  200                         temps, fileName, historyFile.getId()), ex); 
 
  201                 this.addErrorMessage(
 
  202                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getHistory.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  206             File dbFile = 
new File(temps);
 
  207             if (context.dataSourceIngestIsCancelled()) {
 
  211             List<HashMap<String, Object>> tempList = this.querySQLiteDb(temps, HISTORY_QUERY);
 
  212             logger.log(Level.INFO, 
"{0} - Now getting history from {1} with {2} artifacts identified.", 
new Object[]{getDisplayName(), temps, tempList.size()}); 
 
  213             for (HashMap<String, Object> result : tempList) {
 
  215                 if (context.dataSourceIngestIsCancelled()) {
 
  219                 String url = result.get(
"url").toString();
 
  220                 String domain = extractDomain(url);
 
  223                     Collection<BlackboardAttribute> bbattributes = createHistoryAttributes(
 
  225                             Long.valueOf(result.get(
"visit_date").toString()), 
 
  226                             result.get(
"ref").toString(), 
 
  227                             result.get(
"title").toString(), 
 
  228                             NbBundle.getMessage(this.getClass(), 
"Firefox.moduleName"), 
 
  233                     bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_HISTORY, historyFile, bbattributes));
 
  234                 } 
catch (TskCoreException ex) {
 
  235                     logger.log(Level.SEVERE, String.format(
"Failed to create TSK_WEB_HISTORY artifact for file %d", historyFile.getId()), ex);
 
  242         if (!context.dataSourceIngestIsCancelled()) {
 
  243             postArtifacts(bbartifacts);
 
  252     private void getBookmark(
long ingestJobId) {
 
  254         FileManager fileManager = currentCase.getServices().getFileManager();
 
  255         List<AbstractFile> bookmarkFiles;
 
  257             bookmarkFiles = fileManager.findFiles(dataSource, 
"places.sqlite", 
"Firefox"); 
 
  258         } 
catch (TskCoreException ex) {
 
  259             String msg = NbBundle.getMessage(this.getClass(), 
"Firefox.getBookmark.errMsg.errFetchFiles");
 
  260             logger.log(Level.WARNING, msg);
 
  261             this.addErrorMessage(this.getDisplayName() + 
": " + msg);
 
  265         if (bookmarkFiles.isEmpty()) {
 
  266             logger.log(Level.INFO, 
"Didn't find any firefox bookmark files."); 
 
  271         Collection<BlackboardArtifact> bbartifacts = 
new ArrayList<>();
 
  273         for (AbstractFile bookmarkFile : bookmarkFiles) {
 
  274             if (bookmarkFile.getSize() == 0) {
 
  277             String fileName = bookmarkFile.getName();
 
  278             String temps = RAImageIngestModule.getRATempPath(currentCase, 
"firefox", ingestJobId) + File.separator + fileName + j + 
".db"; 
 
  280                 ContentUtils.writeToFile(bookmarkFile, 
new File(temps), context::dataSourceIngestIsCancelled);
 
  281             } 
catch (ReadContentInputStreamException ex) {
 
  282                 logger.log(Level.WARNING, String.format(
"Error reading Firefox bookmark artifacts file '%s' (id=%d).",
 
  283                         fileName, bookmarkFile.getId()), ex); 
 
  284                 this.addErrorMessage(
 
  285                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getHistory.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  288             } 
catch (IOException ex) {
 
  289                 logger.log(Level.SEVERE, String.format(
"Error writing temp sqlite db file '%s' for Firefox bookmark artifacts file '%s' (id=%d).",
 
  290                         temps, fileName, bookmarkFile.getId()), ex); 
 
  291                 this.addErrorMessage(NbBundle.getMessage(
this.getClass(), 
"Firefox.getBookmark.errMsg.errAnalyzeFile",
 
  292                         this.getDisplayName(), fileName));
 
  295             File dbFile = 
new File(temps);
 
  296             if (context.dataSourceIngestIsCancelled()) {
 
  300             List<HashMap<String, Object>> tempList = this.querySQLiteDb(temps, BOOKMARK_QUERY);
 
  301             logger.log(Level.INFO, 
"{0} - Now getting bookmarks from {1} with {2} artifacts identified.", 
new Object[]{getDisplayName(), temps, tempList.size()}); 
 
  302             for (HashMap<String, Object> result : tempList) {
 
  304                 if (context.dataSourceIngestIsCancelled()) {
 
  308                 String url = result.get(
"url").toString();
 
  310                 Collection<BlackboardAttribute> bbattributes = 
new ArrayList<>();
 
  311                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
 
  312                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  313                         ((url != null) ? url : 
""))); 
 
  314                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_TITLE,
 
  315                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  316                         ((result.get(
"title").toString() != null) ? result.get(
"title").toString() : 
""))); 
 
  317                 if (Long.valueOf(result.get(
"dateAdded").toString()) > 0) { 
 
  318                     bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
 
  319                             RecentActivityExtracterModuleFactory.getModuleName(),
 
  320                             (Long.valueOf(result.get(
"dateAdded").toString())))); 
 
  322                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
 
  323                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  324                         NbBundle.getMessage(this.getClass(), 
"Firefox.moduleName")));
 
  325                 String domain = extractDomain(url);
 
  326                 if (domain != null && domain.isEmpty() == 
false) {
 
  327                     bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
 
  328                             RecentActivityExtracterModuleFactory.getModuleName(), domain)); 
 
  332                     bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_BOOKMARK, bookmarkFile, bbattributes));
 
  333                 } 
catch (TskCoreException ex) {
 
  334                     logger.log(Level.SEVERE, String.format(
"Failed to create TSK_WEB_BOOKMARK artifact for file %d", bookmarkFile.getId()), ex);
 
  341         if (!context.dataSourceIngestIsCancelled()) {
 
  342             postArtifacts(bbartifacts);
 
  351     private void getCookie(
long ingestJobId) {
 
  352         FileManager fileManager = currentCase.getServices().getFileManager();
 
  353         List<AbstractFile> cookiesFiles;
 
  355             cookiesFiles = fileManager.findFiles(dataSource, 
"cookies.sqlite", 
"Firefox"); 
 
  356         } 
catch (TskCoreException ex) {
 
  357             String msg = NbBundle.getMessage(this.getClass(), 
"Firefox.getCookie.errMsg.errFetchFile");
 
  358             logger.log(Level.WARNING, msg);
 
  359             this.addErrorMessage(this.getDisplayName() + 
": " + msg);
 
  363         if (cookiesFiles.isEmpty()) {
 
  364             logger.log(Level.INFO, 
"Didn't find any Firefox cookie files."); 
 
  369         Collection<BlackboardArtifact> bbartifacts = 
new ArrayList<>();
 
  371         for (AbstractFile cookiesFile : cookiesFiles) {
 
  372             if (context.dataSourceIngestIsCancelled()) {
 
  376             if (cookiesFile.getSize() == 0) {
 
  379             String fileName = cookiesFile.getName();
 
  380             String temps = RAImageIngestModule.getRATempPath(currentCase, 
"firefox", ingestJobId) + File.separator + fileName + j + 
".db"; 
 
  382                 ContentUtils.writeToFile(cookiesFile, 
new File(temps), context::dataSourceIngestIsCancelled);
 
  383             } 
catch (ReadContentInputStreamException ex) {
 
  384                 logger.log(Level.WARNING, String.format(
"Error reading Firefox cookie artifacts file '%s' (id=%d).",
 
  385                         fileName, cookiesFile.getId()), ex); 
 
  386                 this.addErrorMessage(
 
  387                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getHistory.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  390             } 
catch (IOException ex) {
 
  391                 logger.log(Level.SEVERE, String.format(
"Error writing temp sqlite db file '%s' for Firefox cookie artifacts file '%s' (id=%d).",
 
  392                         temps, fileName, cookiesFile.getId()), ex); 
 
  393                 this.addErrorMessage(
 
  394                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getCookie.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  398             File dbFile = 
new File(temps);
 
  399             if (context.dataSourceIngestIsCancelled()) {
 
  403             boolean checkColumn = Util.checkColumn(
"creationTime", 
"moz_cookies", temps); 
 
  406                 query = COOKIE_QUERY;
 
  408                 query = COOKIE_QUERY_V3;
 
  411             List<HashMap<String, Object>> tempList = this.querySQLiteDb(temps, query);
 
  412             logger.log(Level.INFO, 
"{0} - Now getting cookies from {1} with {2} artifacts identified.", 
new Object[]{getDisplayName(), temps, tempList.size()}); 
 
  413             for (HashMap<String, Object> result : tempList) {
 
  415                 if (context.dataSourceIngestIsCancelled()) {
 
  419                 String host = result.get(
"host").toString();
 
  421                 Collection<BlackboardAttribute> bbattributes = 
new ArrayList<>();
 
  422                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
 
  423                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  424                         ((host != null) ? host : 
""))); 
 
  425                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
 
  426                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  427                         (Long.valueOf(result.get(
"lastAccessed").toString())))); 
 
  428                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
 
  429                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  430                         ((result.get(
"name").toString() != null) ? result.get(
"name").toString() : 
""))); 
 
  431                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
 
  432                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  433                         ((result.get(
"value").toString() != null) ? result.get(
"value").toString() : 
""))); 
 
  434                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
 
  435                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  436                         NbBundle.getMessage(this.getClass(), 
"Firefox.moduleName")));
 
  438                 if (checkColumn == 
true) {
 
  439                     String value = result.get(
"creationTime").toString();
 
  440                     if(value != null && !value.isEmpty()) {
 
  441                         bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
 
  442                                 RecentActivityExtracterModuleFactory.getModuleName(),
 
  443                                 (Long.valueOf(result.get(
"creationTime").toString())))); 
 
  446                 String domain = extractDomain(host);
 
  447                 if (domain != null && domain.isEmpty() == 
false) {
 
  448                     domain = domain.replaceFirst(
"^\\.+(?!$)", 
"");
 
  449                     bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
 
  450                             RecentActivityExtracterModuleFactory.getModuleName(), domain));
 
  454                     bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_COOKIE, cookiesFile, bbattributes));
 
  455                 } 
catch (TskCoreException ex) {
 
  456                     logger.log(Level.SEVERE, String.format(
"Failed to create TSK_WEB_COOKIE artifact for file %d", cookiesFile.getId()), ex);
 
  463         if (!context.dataSourceIngestIsCancelled()) {
 
  464             postArtifacts(bbartifacts);
 
  473     private void getDownload(
long ingestJobId) {
 
  474         getDownloadPreVersion24(ingestJobId);
 
  475         getDownloadVersion24(ingestJobId);
 
  485     private void getDownloadPreVersion24(
long ingestJobId) {
 
  487         FileManager fileManager = currentCase.getServices().getFileManager();
 
  488         List<AbstractFile> downloadsFiles;
 
  490             downloadsFiles = fileManager.findFiles(dataSource, 
"downloads.sqlite", 
"Firefox"); 
 
  491         } 
catch (TskCoreException ex) {
 
  492             String msg = NbBundle.getMessage(this.getClass(), 
"Firefox.getDlPre24.errMsg.errFetchFiles");
 
  493             logger.log(Level.WARNING, msg);
 
  494             this.addErrorMessage(this.getDisplayName() + 
": " + msg);
 
  498         if (downloadsFiles.isEmpty()) {
 
  499             logger.log(Level.INFO, 
"Didn't find any pre-version-24.0 Firefox download files."); 
 
  504         Collection<BlackboardArtifact> bbartifacts = 
new ArrayList<>();
 
  506         for (AbstractFile downloadsFile : downloadsFiles) {
 
  507             if (downloadsFile.getSize() == 0) {
 
  510             String fileName = downloadsFile.getName();
 
  511             String temps = RAImageIngestModule.getRATempPath(currentCase, 
"firefox", ingestJobId) + File.separator + fileName + j + 
".db"; 
 
  514                 ContentUtils.writeToFile(downloadsFile, 
new File(temps), context::dataSourceIngestIsCancelled);
 
  515             } 
catch (ReadContentInputStreamException ex) {
 
  516                 logger.log(Level.WARNING, String.format(
"Error reading Firefox download artifacts file '%s' (id=%d).",
 
  517                         fileName, downloadsFile.getId()), ex); 
 
  518                 this.addErrorMessage(
 
  519                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getHistory.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  522             } 
catch (IOException ex) {
 
  523                 logger.log(Level.SEVERE, String.format(
"Error writing temp sqlite db file '%s' for Firefox download artifacts file '%s' (id=%d).",
 
  524                         temps, fileName, downloadsFile.getId()), ex); 
 
  525                 this.addErrorMessage(NbBundle.getMessage(
this.getClass(), 
"Firefox.getDlPre24.errMsg.errAnalyzeFiles",
 
  526                         this.getDisplayName(), fileName));
 
  529             File dbFile = 
new File(temps);
 
  530             if (context.dataSourceIngestIsCancelled()) {
 
  535             List<HashMap<String, Object>> tempList = this.querySQLiteDb(temps, DOWNLOAD_QUERY);
 
  536             logger.log(Level.INFO, 
"{0}- Now getting downloads from {1} with {2} artifacts identified.", 
new Object[]{getDisplayName(), temps, tempList.size()}); 
 
  537             for (HashMap<String, Object> result : tempList) {
 
  539                 if (context.dataSourceIngestIsCancelled()) {
 
  543                 String source = result.get(
"source").toString();
 
  545                 Collection<BlackboardAttribute> bbattributes = 
new ArrayList<>();
 
  547                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
 
  548                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  551                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
 
  552                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  553                         (Long.valueOf(result.get(
"startTime").toString())))); 
 
  555                 String target = result.get(
"target").toString(); 
 
  556                 String downloadedFilePath = 
"";
 
  557                 if (target != null) {
 
  559                         downloadedFilePath = URLDecoder.decode(target.replaceAll(
"file:///", 
""), 
"UTF-8"); 
 
  560                         bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH,
 
  561                                 RecentActivityExtracterModuleFactory.getModuleName(),
 
  562                                 downloadedFilePath));
 
  563                         long pathID = Util.findID(dataSource, downloadedFilePath);
 
  565                             bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH_ID,
 
  566                                     RecentActivityExtracterModuleFactory.getModuleName(),
 
  569                     } 
catch (UnsupportedEncodingException ex) {
 
  570                         logger.log(Level.SEVERE, 
"Error decoding Firefox download URL in " + temps, ex); 
 
  575                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
 
  576                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  577                         NbBundle.getMessage(this.getClass(), 
"Firefox.moduleName")));
 
  578                 String domain = extractDomain(source);
 
  579                 if (domain != null && domain.isEmpty() == 
false) {
 
  580                     bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
 
  581                             RecentActivityExtracterModuleFactory.getModuleName(),
 
  585                     BlackboardArtifact webDownloadArtifact = createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_DOWNLOAD, downloadsFile, bbattributes);
 
  586                     bbartifacts.add(webDownloadArtifact);
 
  589                     for (AbstractFile downloadedFile : currentCase.getSleuthkitCase().getFileManager().findFilesExactNameExactPath(dataSource,
 
  590                             FilenameUtils.getName(downloadedFilePath), FilenameUtils.getPath(downloadedFilePath))) {
 
  591                         bbartifacts.add(createAssociatedArtifact(downloadedFile, webDownloadArtifact));
 
  594                 } 
catch (TskCoreException ex) {
 
  595                     logger.log(Level.SEVERE, String.format(
"Error creating TSK_WEB_DOWNLOAD or TSK_ASSOCIATED_ARTIFACT artifact for file '%d'",
 
  596                             downloadsFile.getId()), ex); 
 
  601                 this.addErrorMessage(
 
  602                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getDlPre24.errMsg.errParsingArtifacts",
 
  603                                 this.getDisplayName(), errors));
 
  609         if (!context.dataSourceIngestIsCancelled()) {
 
  610             postArtifacts(bbartifacts);
 
  621     private void getDownloadVersion24(
long ingestJobId) {
 
  622         FileManager fileManager = currentCase.getServices().getFileManager();
 
  623         List<AbstractFile> downloadsFiles;
 
  625             downloadsFiles = fileManager.findFiles(dataSource, 
"places.sqlite", 
"Firefox"); 
 
  626         } 
catch (TskCoreException ex) {
 
  627             String msg = NbBundle.getMessage(this.getClass(), 
"Firefox.getDlV24.errMsg.errFetchFiles");
 
  628             logger.log(Level.WARNING, msg);
 
  629             this.addErrorMessage(this.getDisplayName() + 
": " + msg);
 
  633         if (downloadsFiles.isEmpty()) {
 
  634             logger.log(Level.INFO, 
"Didn't find any version-24.0 Firefox download files."); 
 
  639         Collection<BlackboardArtifact> bbartifacts = 
new ArrayList<>();
 
  641         for (AbstractFile downloadsFile : downloadsFiles) {
 
  642             if (downloadsFile.getSize() == 0) {
 
  645             String fileName = downloadsFile.getName();
 
  646             String temps = RAImageIngestModule.getRATempPath(currentCase, 
"firefox", ingestJobId) + File.separator + fileName + 
"-downloads" + j + 
".db"; 
 
  649                 ContentUtils.writeToFile(downloadsFile, 
new File(temps), context::dataSourceIngestIsCancelled);
 
  650             } 
catch (ReadContentInputStreamException ex) {
 
  651                 logger.log(Level.WARNING, String.format(
"Error reading Firefox download artifacts file '%s' (id=%d).",
 
  652                         fileName, downloadsFile.getId()), ex); 
 
  653                 this.addErrorMessage(
 
  654                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getHistory.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  657             } 
catch (IOException ex) {
 
  658                 logger.log(Level.SEVERE, String.format(
"Error writing temp sqlite db file '%s' for Firefox download artifacts file '%s' (id=%d).",
 
  659                         temps, fileName, downloadsFile.getId()), ex); 
 
  660                 this.addErrorMessage(
 
  661                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getDlV24.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  665             File dbFile = 
new File(temps);
 
  666             if (context.dataSourceIngestIsCancelled()) {
 
  671             List<HashMap<String, Object>> tempList = this.querySQLiteDb(temps, DOWNLOAD_QUERY_V24);
 
  673             logger.log(Level.INFO, 
"{0} - Now getting downloads from {1} with {2} artifacts identified.", 
new Object[]{getDisplayName(), temps, tempList.size()}); 
 
  674             for (HashMap<String, Object> result : tempList) {
 
  676                 if (context.dataSourceIngestIsCancelled()) {
 
  680                 String url = result.get(
"url").toString();
 
  682                 Collection<BlackboardAttribute> bbattributes = 
new ArrayList<>();
 
  684                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
 
  685                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  691                 String target = result.get(
"target").toString(); 
 
  692                 String downloadedFilePath = 
"";
 
  693                 if (target != null) {
 
  695                         downloadedFilePath = URLDecoder.decode(target.replaceAll(
"file:///", 
""), 
"UTF-8"); 
 
  696                         bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH,
 
  697                                 RecentActivityExtracterModuleFactory.getModuleName(),
 
  698                                 downloadedFilePath));
 
  699                         long pathID = Util.findID(dataSource, downloadedFilePath);
 
  701                             bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PATH_ID,
 
  702                                     RecentActivityExtracterModuleFactory.getModuleName(),
 
  705                     } 
catch (UnsupportedEncodingException ex) {
 
  706                         logger.log(Level.SEVERE, 
"Error decoding Firefox download URL in " + temps, ex); 
 
  710                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
 
  711                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  712                         Long.valueOf(result.get(
"lastModified").toString()))); 
 
  713                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
 
  714                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  715                         NbBundle.getMessage(this.getClass(), 
"Firefox.moduleName")));
 
  716                 String domain = extractDomain(url);
 
  717                 if (domain != null && domain.isEmpty() == 
false) {
 
  718                     bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
 
  719                             RecentActivityExtracterModuleFactory.getModuleName(), domain)); 
 
  722                     BlackboardArtifact webDownloadArtifact = createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_DOWNLOAD, downloadsFile, bbattributes);
 
  723                     bbartifacts.add(webDownloadArtifact);
 
  726                     for (AbstractFile downloadedFile : currentCase.getSleuthkitCase().getFileManager().findFilesExactNameExactPath(dataSource,
 
  727                             FilenameUtils.getName(downloadedFilePath), FilenameUtils.getPath(downloadedFilePath))) {
 
  728                         bbartifacts.add(createAssociatedArtifact(downloadedFile, webDownloadArtifact));
 
  731                 } 
catch (TskCoreException ex) {
 
  732                     logger.log(Level.SEVERE, String.format(
"Error creating associated object artifact for file  '%s'",
 
  733                             downloadedFilePath), ex); 
 
  737                 this.addErrorMessage(NbBundle.getMessage(
this.getClass(), 
"Firefox.getDlV24.errMsg.errParsingArtifacts",
 
  738                         this.getDisplayName(), errors));
 
  744         if (!context.dataSourceIngestIsCancelled()) {
 
  745             postArtifacts(bbartifacts);
 
  755     private void getFormsHistory(
long ingestJobId) {
 
  756         FileManager fileManager = currentCase.getServices().getFileManager();
 
  757         List<AbstractFile> formHistoryFiles;
 
  760         Set<String> excludedFieldNames = 
new HashSet<>(Arrays.asList(
 
  766             formHistoryFiles = fileManager.findFiles(dataSource, 
"formhistory.sqlite", 
"Firefox"); 
 
  767         } 
catch (TskCoreException ex) {
 
  768             String msg = NbBundle.getMessage(this.getClass(), 
"Firefox.getFormsAutofill.errMsg.errFetchingFiles");
 
  769             logger.log(Level.WARNING, msg);
 
  770             this.addErrorMessage(this.getDisplayName() + 
": " + msg);
 
  774         if (formHistoryFiles.isEmpty()) {
 
  775             String msg = NbBundle.getMessage(this.getClass(), 
"Firefox.getFormsAutofill.errMsg.noFilesFound");
 
  776             logger.log(Level.INFO, msg);
 
  781         Collection<BlackboardArtifact> bbartifacts = 
new ArrayList<>();
 
  783         for (AbstractFile formHistoryFile : formHistoryFiles) {
 
  784             if (formHistoryFile.getSize() == 0) {
 
  788             String fileName = formHistoryFile.getName();
 
  789             String tempFilePath = RAImageIngestModule.getRATempPath(currentCase, 
"firefox", ingestJobId) + File.separator + fileName + j + 
".db"; 
 
  791                 ContentUtils.writeToFile(formHistoryFile, 
new File(tempFilePath), context::dataSourceIngestIsCancelled);
 
  792             } 
catch (ReadContentInputStreamException ex) {
 
  793                 logger.log(Level.WARNING, String.format(
"Error reading Firefox web history artifacts file '%s' (id=%d).",
 
  794                         fileName, formHistoryFile.getId()), ex); 
 
  795                 this.addErrorMessage(
 
  796                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getFormsAutofill.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  799             } 
catch (IOException ex) {
 
  800                 logger.log(Level.SEVERE, String.format(
"Error writing temp sqlite db file '%s' for Firefox web history artifacts file '%s' (id=%d).",
 
  801                         tempFilePath, fileName, formHistoryFile.getId()), ex); 
 
  802                 this.addErrorMessage(
 
  803                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getFormsAutofill.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  807             File dbFile = 
new File(tempFilePath);
 
  808             if (context.dataSourceIngestIsCancelled()) {
 
  814             boolean isFirefoxV64 = Util.checkColumn(
"timesUsed", 
"moz_formhistory", tempFilePath);
 
  815             String formHistoryQuery = (isFirefoxV64) ? FORMHISTORY_QUERY_V64 : FORMHISTORY_QUERY;
 
  817             List<HashMap<String, Object>> tempList = this.querySQLiteDb(tempFilePath, formHistoryQuery);
 
  818             logger.log(Level.INFO, 
"{0} - Now getting history from {1} with {2} artifacts identified.", 
new Object[]{getDisplayName(), tempFilePath, tempList.size()}); 
 
  819             for (HashMap<String, Object> result : tempList) {
 
  821                 if (context.dataSourceIngestIsCancelled()) {
 
  825                 Collection<BlackboardAttribute> bbattributes = 
new ArrayList<>();
 
  827                 String fieldName = ((result.get(
"fieldname").toString() != null) ? result.get(
"fieldname").toString() : 
"");
 
  829                 if (excludedFieldNames.contains(fieldName.toLowerCase())) {
 
  833                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
 
  834                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  837                 bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
 
  838                         RecentActivityExtracterModuleFactory.getModuleName(),
 
  839                         ((result.get(
"value").toString() != null) ? result.get(
"value").toString() : 
""))); 
 
  843                     bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
 
  844                             RecentActivityExtracterModuleFactory.getModuleName(),
 
  845                             (Long.valueOf(result.get(
"firstUsed").toString()) / 1000000))); 
 
  847                     bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_ACCESSED,
 
  848                             RecentActivityExtracterModuleFactory.getModuleName(),
 
  849                             (Long.valueOf(result.get(
"lastUsed").toString()) / 1000000))); 
 
  851                     bbattributes.add(
new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COUNT,
 
  852                             RecentActivityExtracterModuleFactory.getModuleName(),
 
  853                             (Integer.valueOf(result.get(
"timesUsed").toString())))); 
 
  858                     bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_FORM_AUTOFILL, formHistoryFile, bbattributes));
 
  859                 } 
catch (TskCoreException ex) {
 
  860                     logger.log(Level.SEVERE, String.format(
"Failed to create TSK_WEB_FORM_AUTOFILL artifact for file %d", formHistoryFile.getId()), ex);
 
  867         if (!context.dataSourceIngestIsCancelled()) {
 
  868             postArtifacts(bbartifacts);
 
  878     private void getAutofillProfiles(
long ingestJobId) {
 
  879         FileManager fileManager = currentCase.getServices().getFileManager();
 
  880         List<AbstractFile> autofillProfilesFiles;
 
  882             autofillProfilesFiles = fileManager.findFiles(dataSource, 
"autofill-profiles.json", 
"Firefox"); 
 
  883         } 
catch (TskCoreException ex) {
 
  884             String msg = NbBundle.getMessage(this.getClass(), 
"Firefox.getAutofillProfiles.errMsg.errGettingFiles");
 
  885             logger.log(Level.SEVERE, msg, ex);
 
  886             this.addErrorMessage(this.getDisplayName() + 
": " + msg);
 
  890         if (autofillProfilesFiles.isEmpty()) {
 
  891             logger.log(Level.INFO, 
"Didn't find any Firefox Autofill Profiles files."); 
 
  897         while (j < autofillProfilesFiles.size()) {
 
  898             AbstractFile profileFile = autofillProfilesFiles.get(j++);
 
  899             if (profileFile.getSize() == 0) {
 
  902             String temps = RAImageIngestModule.getRATempPath(currentCase, 
"Firefox", ingestJobId) + File.separator + profileFile.getName() + j + 
".json"; 
 
  904                 ContentUtils.writeToFile(profileFile, 
new File(temps), context::dataSourceIngestIsCancelled);
 
  905             } 
catch (ReadContentInputStreamException ex) {
 
  906                 logger.log(Level.WARNING, String.format(
"Error reading Firefox Autofill profiles artifacts file '%s' (id=%d).",
 
  907                         profileFile.getName(), profileFile.getId()), ex); 
 
  908                 this.addErrorMessage(NbBundle.getMessage(
this.getClass(), 
"Firefox.getAutofillProfiles.errMsg.errAnalyzingFile",
 
  909                         this.getDisplayName(), profileFile.getName()));
 
  911             } 
catch (IOException ex) {
 
  912                 logger.log(Level.SEVERE, String.format(
"Error writing temp file '%s' for Firefox Autofill profiles file '%s' (id=%d).",
 
  913                         temps, profileFile.getName(), profileFile.getId()), ex); 
 
  914                 this.addErrorMessage(NbBundle.getMessage(
this.getClass(), 
"Firefox.getAutofillProfiles.errMsg.errAnalyzingFile",
 
  915                         this.getDisplayName(), profileFile.getName()));
 
  919             logger.log(Level.INFO, 
"{0}- Now getting Bookmarks from {1}", 
new Object[]{getDisplayName(), temps}); 
 
  920             File dbFile = 
new File(temps);
 
  921             if (context.dataSourceIngestIsCancelled()) {
 
  926             FileReader tempReader;
 
  928                 tempReader = 
new FileReader(temps);
 
  929             } 
catch (FileNotFoundException ex) {
 
  930                 logger.log(Level.SEVERE, 
"Error while trying to read the Autofill profiles json file for Firefox.", ex); 
 
  931                 this.addErrorMessage(
 
  932                         NbBundle.getMessage(
this.getClass(), 
"Firefox.getAutofillProfiles.errMsg.errAnalyzeFile", this.getDisplayName(),
 
  933                                 profileFile.getName()));
 
  937             JsonObject jsonRootObject;
 
  938             JsonArray jAddressesArray;
 
  941                 jsonRootObject = JsonParser.parseReader(tempReader).getAsJsonObject();
 
  942                 jAddressesArray = jsonRootObject.getAsJsonArray(
"addresses"); 
 
  943             } 
catch (JsonIOException | JsonSyntaxException | IllegalStateException ex) {
 
  944                 logger.log(Level.WARNING, 
"Error parsing Json for Firefox Autofill profiles.", ex); 
 
  945                 this.addErrorMessage(NbBundle.getMessage(
this.getClass(), 
"Firefox.getAutofillProfiles.errMsg.errAnalyzingFile3",
 
  946                         this.getDisplayName(), profileFile.getName()));
 
  950             WebBrowserArtifactsHelper helper;
 
  953                 helper = 
new WebBrowserArtifactsHelper(
 
  954                         Case.getCurrentCaseThrows().getSleuthkitCase(),
 
  955                         NbBundle.getMessage(this.getClass(), 
"Firefox.parentModuleName"),
 
  959             } 
catch (NoCurrentCaseException ex) {
 
  960                 logger.log(Level.SEVERE, 
"No case open, bailing.", ex); 
 
  964             for (JsonElement result : jAddressesArray) {
 
  965                 JsonObject address = result.getAsJsonObject();
 
  966                 if (address == null) {
 
  970                 JsonElement nameEl = address.get(
"name"); 
 
  971                 String name = (nameEl != null) ? nameEl.getAsString() : 
"";
 
  973                 JsonElement emailEl = address.get(
"email"); 
 
  974                 String email = (emailEl != null) ? emailEl.getAsString() : 
"";
 
  976                 JsonElement telEl = address.get(
"tel"); 
 
  977                 String tel = (telEl != null) ? telEl.getAsString() : 
"";
 
  978                 JsonElement telCountryCodeEl = address.get(
"tel-country-code"); 
 
  979                 String telCountryCode = (telCountryCodeEl != null) ? telCountryCodeEl.getAsString() : 
"";
 
  980                 JsonElement telNationalEl = address.get(
"tel-national"); 
 
  981                 String telNational = (telNationalEl != null) ? telNationalEl.getAsString() : 
"";
 
  983                 String phoneNumber = makeTelNumber(tel, telCountryCode, telNational);
 
  985                 JsonElement createdEl = address.get(
"timeCreated"); 
 
  986                 Long datetimeCreated = (createdEl != null) ? createdEl.getAsLong() / 1000 : Long.valueOf(0);
 
  987                 JsonElement lastusedEl = address.get(
"timeLastUsed"); 
 
  988                 Long datetimeLastUsed = (lastusedEl != null) ? lastusedEl.getAsLong() / 1000 : Long.valueOf(0);
 
  989                 JsonElement timesUsedEl = address.get(
"timesUsed"); 
 
  990                 Integer timesUsed = (timesUsedEl != null) ? timesUsedEl.getAsShort() : Integer.valueOf(0);
 
  992                 JsonElement addressLine1El = address.get(
"address-line1"); 
 
  993                 String addressLine1 = (addressLine1El != null) ? addressLine1El.getAsString() : 
"";
 
  994                 JsonElement addressLine2El = address.get(
"address-line2"); 
 
  995                 String addressLine2 = (addressLine2El != null) ? addressLine2El.getAsString() : 
"";
 
  996                 JsonElement addressLine3El = address.get(
"address-line3"); 
 
  997                 String addressLine3 = (addressLine3El != null) ? addressLine3El.getAsString() : 
"";
 
  999                 JsonElement postalCodeEl = address.get(
"postal-code"); 
 
 1000                 String postalCode = (postalCodeEl != null) ? postalCodeEl.getAsString() : 
"";
 
 1001                 JsonElement countryEl = address.get(
"country"); 
 
 1002                 String country = (countryEl != null) ? countryEl.getAsString() : 
"";
 
 1004                 String mailingAddress = makeFullAddress(addressLine1, addressLine2, addressLine3, postalCode, country);
 
 1007                     helper.addWebFormAddress(name, email, phoneNumber,
 
 1008                             mailingAddress, datetimeCreated, datetimeLastUsed, timesUsed);
 
 1009                 } 
catch (TskCoreException | Blackboard.BlackboardException ex) {
 
 1010                     logger.log(Level.SEVERE, 
"Error while trying to insert Firefox Autofill profile artifact{0}", ex); 
 
 1011                     this.addErrorMessage(
 
 1012                             NbBundle.getMessage(
this.getClass(), 
"Firefox.getAutofillProfiles.errMsg.errAnalyzingFile4",
 
 1013                                     this.getDisplayName(), profileFile.getName()));
 
 1028     private String extractDomain(String url) {
 
 1029         if (url == null || url.isEmpty()) {
 
 1033         if (url.toLowerCase().startsWith(PLACE_URL_PREFIX)) {
 
 1040         return NetworkUtils.extractDomain(url);
 
 1054     private String makeTelNumber(String tel, String telCountryCode, String telNational) {
 
 1056         if (tel != null && !tel.isEmpty()) {
 
 1060         if ((telCountryCode != null && !telCountryCode.isEmpty())
 
 1061                 && (telNational != null && !telNational.isEmpty())) {
 
 1062             return telCountryCode + telNational;
 
 1079     private String makeFullAddress(String addressLine1, String addressLine2, String addressLine3, String postalCode, String country) {
 
 1080         String fullAddress = 
"";
 
 1081         fullAddress = appendAddressField(fullAddress, addressLine1);
 
 1082         fullAddress = appendAddressField(fullAddress, addressLine2);
 
 1083         fullAddress = appendAddressField(fullAddress, addressLine3);
 
 1084         fullAddress = appendAddressField(fullAddress, postalCode);
 
 1085         fullAddress = appendAddressField(fullAddress, country);
 
 1099     private String appendAddressField(String address, String addressfield) {
 
 1101         String updatedAddress = address;
 
 1102         if (addressfield != null && !addressfield.isEmpty()) {
 
 1103             if (!updatedAddress.isEmpty()) {
 
 1104                 updatedAddress += 
", ";
 
 1106             updatedAddress += addressfield;
 
 1109         return updatedAddress;