Autopsy  4.14.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
ChromeCacheExtractor.java
Go to the documentation of this file.
1 /*
2  *
3  * Autopsy Forensic Browser
4  *
5  * Copyright 2019 Basis Technology Corp.
6  *
7  * Project Contact/Architect: carrier <at> sleuthkit <dot> org
8  *
9  * Licensed under the Apache License, Version 2.0 (the "License");
10  * you may not use this file except in compliance with the License.
11  * You may obtain a copy of the License at
12  *
13  * http://www.apache.org/licenses/LICENSE-2.0
14  *
15  * Unless required by applicable law or agreed to in writing, software
16  * distributed under the License is distributed on an "AS IS" BASIS,
17  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18  * See the License for the specific language governing permissions and
19  * limitations under the License.
20  */
21 package org.sleuthkit.autopsy.recentactivity;
22 
23 import java.io.File;
24 import java.io.FileOutputStream;
25 import java.io.IOException;
26 import java.io.RandomAccessFile;
27 import java.nio.ByteBuffer;
28 import java.nio.ByteOrder;
29 import java.nio.channels.FileChannel;
30 import java.nio.charset.Charset;
31 import java.nio.file.Path;
32 import java.nio.file.Paths;
33 import java.util.ArrayList;
34 import java.util.Collection;
35 import java.util.HashMap;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Optional;
40 import java.util.logging.Level;
41 import org.openide.util.NbBundle;
42 import org.openide.util.NbBundle.Messages;
53 import org.sleuthkit.datamodel.AbstractFile;
54 import org.sleuthkit.datamodel.Blackboard;
55 import org.sleuthkit.datamodel.BlackboardArtifact;
56 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
57 import org.sleuthkit.datamodel.BlackboardAttribute;
58 import org.sleuthkit.datamodel.Content;
59 import org.sleuthkit.datamodel.DerivedFile;
60 import org.sleuthkit.datamodel.TimeUtilities;
61 import org.sleuthkit.datamodel.TskCoreException;
62 import org.sleuthkit.datamodel.TskData;
63 import org.sleuthkit.datamodel.TskException;
64 
89 final class ChromeCacheExtractor {
90 
91  private final static String DEFAULT_CACHE_PATH_STR = "default/cache"; //NON-NLS
92  private final static String BROTLI_MIMETYPE ="application/x-brotli"; //NON-NLS
93 
94  private final static long UINT32_MASK = 0xFFFFFFFFl;
95 
96  private final static int INDEXFILE_HDR_SIZE = 92*4;
97  private final static int DATAFILE_HDR_SIZE = 8192;
98 
99  private final static Logger logger = Logger.getLogger(ChromeCacheExtractor.class.getName());
100 
101  private static final String VERSION_NUMBER = "1.0.0"; //NON-NLS
102  private final String moduleName;
103 
104  private String absOutputFolderName;
105  private String relOutputFolderName;
106 
107  private final Content dataSource;
108  private final IngestJobContext context;
109  private final DataSourceIngestModuleProgress progressBar;
110  private final IngestServices services = IngestServices.getInstance();
111  private Case currentCase;
112  private FileManager fileManager;
113 
114  // A file table to cache copies of index and data_n files.
115  private final Map<String, FileWrapper> fileCopyCache = new HashMap<>();
116 
117  // A file table to cache the f_* files.
118  private final Map<String, AbstractFile> externalFilesTable = new HashMap<>();
119 
125  final class FileWrapper {
126  private final AbstractFile abstractFile;
127  private final RandomAccessFile fileCopy;
128  private final ByteBuffer byteBuffer;
129 
130  FileWrapper (AbstractFile abstractFile, RandomAccessFile fileCopy, ByteBuffer buffer ) {
131  this.abstractFile = abstractFile;
132  this.fileCopy = fileCopy;
133  this.byteBuffer = buffer;
134  }
135 
136  public RandomAccessFile getFileCopy() {
137  return fileCopy;
138  }
139  public ByteBuffer getByteBuffer() {
140  return byteBuffer;
141  }
142  AbstractFile getAbstractFile() {
143  return abstractFile;
144  }
145  }
146 
147  @NbBundle.Messages({
148  "ChromeCacheExtractor.moduleName=ChromeCacheExtractor",
149  "# {0} - module name",
150  "# {1} - row number",
151  "# {2} - table length",
152  "# {3} - cache path",
153  "ChromeCacheExtractor.progressMsg={0}: Extracting cache entry {1} of {2} entries from {3}"
154  })
155  ChromeCacheExtractor(Content dataSource, IngestJobContext context, DataSourceIngestModuleProgress progressBar ) {
156  moduleName = Bundle.ChromeCacheExtractor_moduleName();
157  this.dataSource = dataSource;
158  this.context = context;
159  this.progressBar = progressBar;
160  }
161 
162 
168  private void moduleInit() throws IngestModuleException {
169 
170  try {
171  currentCase = Case.getCurrentCaseThrows();
172  fileManager = currentCase.getServices().getFileManager();
173 
174  // Create an output folder to save any derived files
175  absOutputFolderName = RAImageIngestModule.getRAOutputPath(currentCase, moduleName);
176  relOutputFolderName = Paths.get( RAImageIngestModule.getRelModuleOutputPath(), moduleName).normalize().toString();
177 
178  File dir = new File(absOutputFolderName);
179  if (dir.exists() == false) {
180  dir.mkdirs();
181  }
182  } catch (NoCurrentCaseException ex) {
183  String msg = "Failed to get current case."; //NON-NLS
184  throw new IngestModuleException(msg, ex);
185  }
186  }
187 
195  private void resetForNewCacheFolder(String cachePath) throws IngestModuleException {
196 
197  fileCopyCache.clear();
198  externalFilesTable.clear();
199 
200  String cacheAbsOutputFolderName = this.getAbsOutputFolderName() + cachePath;
201  File outDir = new File(cacheAbsOutputFolderName);
202  if (outDir.exists() == false) {
203  outDir.mkdirs();
204  }
205 
206  String cacheTempPath = RAImageIngestModule.getRATempPath(currentCase, moduleName) + cachePath;
207  File tempDir = new File(cacheTempPath);
208  if (tempDir.exists() == false) {
209  tempDir.mkdirs();
210  }
211  }
212 
219  private void cleanup () {
220 
221  for (Entry<String, FileWrapper> entry : this.fileCopyCache.entrySet()) {
222  Path tempFilePath = Paths.get(RAImageIngestModule.getRATempPath(currentCase, moduleName), entry.getKey() );
223  try {
224  entry.getValue().getFileCopy().getChannel().close();
225  entry.getValue().getFileCopy().close();
226 
227  File tmpFile = tempFilePath.toFile();
228  if (!tmpFile.delete()) {
229  tmpFile.deleteOnExit();
230  }
231  } catch (IOException ex) {
232  logger.log(Level.WARNING, String.format("Failed to delete cache file copy %s", tempFilePath.toString()), ex); //NON-NLS
233  }
234  }
235  }
236 
242  private String getAbsOutputFolderName() {
243  return absOutputFolderName;
244  }
245 
251  private String getRelOutputFolderName() {
252  return relOutputFolderName;
253  }
254 
261  void processCaches() {
262 
263  try {
264  moduleInit();
265  } catch (IngestModuleException ex) {
266  String msg = "Failed to initialize ChromeCacheExtractor."; //NON-NLS
267  logger.log(Level.SEVERE, msg, ex);
268  return;
269  }
270 
271  // Find and process the cache folders. There could be one per user
272  try {
273  // Identify each cache folder by searching for the index files in each
274  List<AbstractFile> indexFiles = findIndexFiles();
275 
276  // Process each of the cache folders
277  for (AbstractFile indexFile: indexFiles) {
278 
279  if (context.dataSourceIngestIsCancelled()) {
280  return;
281  }
282 
283  processCacheFolder(indexFile);
284  }
285 
286  } catch (TskCoreException ex) {
287  String msg = "Failed to find cache index files"; //NON-NLS
288  logger.log(Level.WARNING, msg, ex);
289  }
290  }
291 
292  @Messages({
293  "ChromeCacheExtract_adding_extracted_files_msg=Chrome Cache: Adding %d extracted files for analysis.",
294  "ChromeCacheExtract_adding_artifacts_msg=Chrome Cache: Adding %d artifacts for analysis.",
295  "ChromeCacheExtract_loading_files_msg=Chrome Cache: Loading files from %s."
296  })
297 
304  private void processCacheFolder(AbstractFile indexFile) {
305 
306  String cacheFolderName = indexFile.getParentPath();
307  Optional<FileWrapper> indexFileWrapper;
308 
309  /*
310  * The first part of this method is all about finding the needed files in the cache
311  * folder and making internal copies/caches of them so that we can later process them
312  * and effeciently look them up.
313  */
314  try {
315  progressBar.progress(String.format(Bundle.ChromeCacheExtract_loading_files_msg(), cacheFolderName));
316  resetForNewCacheFolder(cacheFolderName);
317 
318  // @@@ This is little ineffecient because we later in this call search for the AbstractFile that we currently have
319  // Load the index file into the caches
320  indexFileWrapper = findDataOrIndexFile(indexFile.getName(), cacheFolderName);
321  if (!indexFileWrapper.isPresent()) {
322  String msg = String.format("Failed to find copy cache index file %s", indexFile.getUniquePath());
323  logger.log(Level.WARNING, msg);
324  return;
325  }
326 
327 
328  // load the data files into the internal cache. We do this because we often
329  // jump in between the various data_X files resolving segments
330  for (int i = 0; i < 4; i ++) {
331  Optional<FileWrapper> dataFile = findDataOrIndexFile(String.format("data_%1d",i), cacheFolderName );
332  if (!dataFile.isPresent()) {
333  return;
334  }
335  }
336 
337  // find all f_* files in a single query and load them into the cache
338  // we do this here so that it is a single query instead of hundreds of individual ones
339  findExternalFiles(cacheFolderName);
340 
341  } catch (TskCoreException | IngestModuleException ex) {
342  String msg = "Failed to find cache files in path " + cacheFolderName; //NON-NLS
343  logger.log(Level.WARNING, msg, ex);
344  return;
345  }
346 
347  /*
348  * Now the analysis begins. We parse the index file and that drives parsing entries
349  * from data_X or f_XXXX files.
350  */
351  logger.log(Level.INFO, "{0}- Now reading Cache index file from path {1}", new Object[]{moduleName, cacheFolderName }); //NON-NLS
352 
353  List<AbstractFile> derivedFiles = new ArrayList<>();
354  Collection<BlackboardArtifact> artifactsAdded = new ArrayList<>();
355 
356  ByteBuffer indexFileROBuffer = indexFileWrapper.get().getByteBuffer();
357  IndexFileHeader indexHdr = new IndexFileHeader(indexFileROBuffer);
358 
359  // seek past the header
360  indexFileROBuffer.position(INDEXFILE_HDR_SIZE);
361 
362  /* Cycle through index and get the CacheAddress for each CacheEntry. Process each entry
363  * to extract data, add artifacts, etc. from the f_XXXX and data_x files */
364  for (int i = 0; i < indexHdr.getTableLen(); i++) {
365 
366  if (context.dataSourceIngestIsCancelled()) {
367  cleanup();
368  return;
369  }
370 
371  CacheAddress addr = new CacheAddress(indexFileROBuffer.getInt() & UINT32_MASK, cacheFolderName);
372  if (addr.isInitialized()) {
373  progressBar.progress(NbBundle.getMessage(this.getClass(),
374  "ChromeCacheExtractor.progressMsg",
375  moduleName, i, indexHdr.getTableLen(), cacheFolderName) );
376  try {
377  List<DerivedFile> addedFiles = processCacheEntry(addr, artifactsAdded);
378  derivedFiles.addAll(addedFiles);
379  }
380  catch (TskCoreException | IngestModuleException ex) {
381  logger.log(Level.WARNING, String.format("Failed to get cache entry at address %s", addr), ex); //NON-NLS
382  }
383  }
384  }
385 
386  if (context.dataSourceIngestIsCancelled()) {
387  cleanup();
388  return;
389  }
390 
391 
392  // notify listeners of new files and schedule for analysis
393  progressBar.progress(String.format(Bundle.ChromeCacheExtract_adding_extracted_files_msg(), derivedFiles.size()));
394  derivedFiles.forEach((derived) -> {
395  services.fireModuleContentEvent(new ModuleContentEvent(derived));
396  });
397  context.addFilesToJob(derivedFiles);
398 
399  // notify listeners about new artifacts
400  progressBar.progress(String.format(Bundle.ChromeCacheExtract_adding_artifacts_msg(), artifactsAdded.size()));
401  Blackboard blackboard = currentCase.getSleuthkitCase().getBlackboard();
402  try {
403  blackboard.postArtifacts(artifactsAdded, moduleName);
404  } catch (Blackboard.BlackboardException ex) {
405  logger.log(Level.WARNING, String.format("Failed to post cacheIndex artifacts "), ex); //NON-NLS
406  }
407 
408  cleanup();
409  }
410 
423  private List<DerivedFile> processCacheEntry(CacheAddress cacheAddress, Collection<BlackboardArtifact> artifactsAdded ) throws TskCoreException, IngestModuleException {
424 
425  List<DerivedFile> derivedFiles = new ArrayList<>();
426 
427  // get the path to the corresponding data_X file for the cache entry
428  String cacheEntryFileName = cacheAddress.getFilename();
429  String cachePath = cacheAddress.getCachePath();
430 
431  Optional<FileWrapper> cacheEntryFileOptional = findDataOrIndexFile(cacheEntryFileName, cachePath);
432  if (!cacheEntryFileOptional.isPresent()) {
433  String msg = String.format("Failed to find data file %s", cacheEntryFileName); //NON-NLS
434  throw new IngestModuleException(msg);
435  }
436 
437  // Load the entry to get its metadata, segments, etc.
438  CacheEntry cacheEntry = new CacheEntry(cacheAddress, cacheEntryFileOptional.get() );
439  List<CacheDataSegment> dataSegments = cacheEntry.getDataSegments();
440 
441  // Only process the first payload data segment in each entry
442  // first data segement has the HTTP headers, 2nd is the payload
443  if (dataSegments.size() < 2) {
444  return derivedFiles;
445  }
446  CacheDataSegment dataSegment = dataSegments.get(1);
447 
448  // Name where segment is located (could be diffrent from where entry was located)
449  String segmentFileName = dataSegment.getCacheAddress().getFilename();
450  Optional<AbstractFile> segmentFileAbstractFile = findAbstractFile(segmentFileName, cachePath);
451  if (!segmentFileAbstractFile.isPresent()) {
452  logger.log(Level.WARNING, "Error finding segment file: " + cachePath + "/" + segmentFileName); //NON-NLS
453  return derivedFiles;
454  }
455 
456  boolean isBrotliCompressed = false;
457  if (dataSegment.getType() != CacheDataTypeEnum.HTTP_HEADER && cacheEntry.isBrotliCompressed() ) {
458  isBrotliCompressed = true;
459  }
460 
461 
462  // Make artifacts around the cached item and extract data from data_X file
463  try {
464  AbstractFile cachedItemFile; //
465  /* If the cached data is in a f_XXXX file, we only need to make artifacts. */
466  if (dataSegment.isInExternalFile() ) {
467  cachedItemFile = segmentFileAbstractFile.get();
468  }
469  /* If the data is in a data_X file, we need to extract it out and then make the similar artifacts */
470  else {
471 
472  // Data segments in "data_x" files are saved in individual files and added as derived files
473  String filename = dataSegment.save();
474  String relPathname = getRelOutputFolderName() + dataSegment.getCacheAddress().getCachePath() + filename;
475 
476  // @@@ We should batch these up and do them in one big insert / transaction
477  DerivedFile derivedFile = fileManager.addDerivedFile(filename, relPathname,
478  dataSegment.getDataLength(),
479  cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), cacheEntry.getCreationTime(), // TBD
480  true,
481  segmentFileAbstractFile.get(),
482  "",
483  moduleName,
484  VERSION_NUMBER,
485  "",
486  TskData.EncodingType.NONE);
487 
488  derivedFiles.add(derivedFile);
489  cachedItemFile = derivedFile;
490  }
491 
492  addArtifacts(cacheEntry, cacheEntryFileOptional.get().getAbstractFile(), cachedItemFile, artifactsAdded);
493 
494  // Tika doesn't detect these types. So, make sure they have the correct MIME type */
495  if (isBrotliCompressed) {
496  cachedItemFile.setMIMEType(BROTLI_MIMETYPE);
497  cachedItemFile.save();
498  }
499 
500  } catch (TskException ex) {
501  logger.log(Level.SEVERE, "Error while trying to add an artifact", ex); //NON-NLS
502  }
503 
504  return derivedFiles;
505  }
506 
516  private void addArtifacts(CacheEntry cacheEntry, AbstractFile cacheEntryFile, AbstractFile cachedItemFile, Collection<BlackboardArtifact> artifactsAdded) throws TskCoreException {
517 
518  // Create a TSK_WEB_CACHE entry with the parent as data_X file that had the cache entry
519  BlackboardArtifact webCacheArtifact = cacheEntryFile.newArtifact(ARTIFACT_TYPE.TSK_WEB_CACHE);
520  if (webCacheArtifact != null) {
521  Collection<BlackboardAttribute> webAttr = new ArrayList<>();
522  webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_URL,
523  moduleName,
524  ((cacheEntry.getKey() != null) ? cacheEntry.getKey() : "")));
525  webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
526  moduleName, cacheEntry.getCreationTime()));
527  webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HEADERS,
528  moduleName, cacheEntry.getHTTPHeaders()));
529  webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH,
530  moduleName, cachedItemFile.getUniquePath()));
531  webAttr.add(new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PATH_ID,
532  moduleName, cachedItemFile.getId()));
533  webCacheArtifact.addAttributes(webAttr);
534  artifactsAdded.add(webCacheArtifact);
535 
536  // Create a TSK_ASSOCIATED_OBJECT on the f_XXX or derived file file back to the CACHE entry
537  BlackboardArtifact associatedObjectArtifact = cachedItemFile.newArtifact(ARTIFACT_TYPE.TSK_ASSOCIATED_OBJECT);
538  if (associatedObjectArtifact != null) {
539  associatedObjectArtifact.addAttribute(
540  new BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_ASSOCIATED_ARTIFACT,
541  moduleName, webCacheArtifact.getArtifactID()));
542  artifactsAdded.add(associatedObjectArtifact);
543  }
544  }
545  }
546 
555  private void findExternalFiles(String cachePath) throws TskCoreException {
556 
557  List<AbstractFile> effFiles = fileManager.findFiles(dataSource, "f_%", cachePath); //NON-NLS
558  for (AbstractFile abstractFile : effFiles ) {
559  this.externalFilesTable.put(cachePath + abstractFile.getName(), abstractFile);
560  }
561  }
570  private Optional<AbstractFile> findAbstractFile(String cacheFileName, String cacheFolderName) throws TskCoreException {
571 
572  // see if it is cached
573  String fileTableKey = cacheFolderName + cacheFileName;
574  if (cacheFileName.startsWith("f_") && externalFilesTable.containsKey(fileTableKey)) {
575  return Optional.of(externalFilesTable.get(fileTableKey));
576  }
577 
578  if (fileCopyCache.containsKey(fileTableKey)) {
579  return Optional.of(fileCopyCache.get(fileTableKey).getAbstractFile());
580  }
581 
582 
583  List<AbstractFile> cacheFiles = fileManager.findFiles(dataSource, cacheFileName, cacheFolderName); //NON-NLS
584  if (!cacheFiles.isEmpty()) {
585  for (AbstractFile abstractFile: cacheFiles ) {
586  if (abstractFile.getUniquePath().trim().endsWith(DEFAULT_CACHE_PATH_STR)) {
587  return Optional.of(abstractFile);
588  }
589  }
590  return Optional.of(cacheFiles.get(0));
591  }
592 
593  return Optional.empty();
594  }
595 
603  private List<AbstractFile> findIndexFiles() throws TskCoreException {
604  return fileManager.findFiles(dataSource, "index", DEFAULT_CACHE_PATH_STR); //NON-NLS
605  }
606 
607 
608 
620  private Optional<FileWrapper> findDataOrIndexFile(String cacheFileName, String cacheFolderName) throws TskCoreException, IngestModuleException {
621 
622  // Check if the file is already in the cache
623  String fileTableKey = cacheFolderName + cacheFileName;
624  if (fileCopyCache.containsKey(fileTableKey)) {
625  return Optional.of(fileCopyCache.get(fileTableKey));
626  }
627 
628  // Use Autopsy to get the AbstractFile
629  Optional<AbstractFile> abstractFileOptional = findAbstractFile(cacheFileName, cacheFolderName);
630  if (!abstractFileOptional.isPresent()) {
631  return Optional.empty();
632  }
633 
634  // Wrap the file so that we can get the ByteBuffer later.
635  // @@@ BC: I think this should nearly all go into FileWrapper and be done lazily and perhaps based on size.
636  // Many of the files are small enough to keep in memory for the ByteBuffer
637 
638  // write the file to disk so that we can have a memory-mapped ByteBuffer
639  AbstractFile cacheFile = abstractFileOptional.get();
640  RandomAccessFile randomAccessFile = null;
641  String tempFilePathname = RAImageIngestModule.getRATempPath(currentCase, moduleName) + cacheFolderName + cacheFile.getName(); //NON-NLS
642  try {
643  File newFile = new File(tempFilePathname);
644  ContentUtils.writeToFile(cacheFile, newFile, context::dataSourceIngestIsCancelled);
645 
646  randomAccessFile = new RandomAccessFile(tempFilePathname, "r");
647  FileChannel roChannel = randomAccessFile.getChannel();
648  ByteBuffer cacheFileROBuf = roChannel.map(FileChannel.MapMode.READ_ONLY, 0,
649  (int) roChannel.size());
650 
651  cacheFileROBuf.order(ByteOrder.nativeOrder());
652  FileWrapper cacheFileWrapper = new FileWrapper(cacheFile, randomAccessFile, cacheFileROBuf );
653 
654  if (!cacheFileName.startsWith("f_")) {
655  fileCopyCache.put(cacheFolderName + cacheFileName, cacheFileWrapper);
656  }
657 
658  return Optional.of(cacheFileWrapper);
659  }
660  catch (IOException ex) {
661 
662  try {
663  if (randomAccessFile != null) {
664  randomAccessFile.close();
665  }
666  }
667  catch (IOException ex2) {
668  logger.log(Level.SEVERE, "Error while trying to close temp file after exception.", ex2); //NON-NLS
669  }
670  String msg = String.format("Error reading/copying Chrome cache file '%s' (id=%d).", //NON-NLS
671  cacheFile.getName(), cacheFile.getId());
672  throw new IngestModuleException(msg, ex);
673  }
674  }
675 
679  final class IndexFileHeader {
680 
681  private final long magic;
682  private final int version;
683  private final int numEntries;
684  private final int numBytes;
685  private final int lastFile;
686  private final int tableLen;
687 
688  IndexFileHeader(ByteBuffer indexFileROBuf) {
689 
690  magic = indexFileROBuf.getInt() & UINT32_MASK;
691 
692  indexFileROBuf.position(indexFileROBuf.position()+2);
693 
694  version = indexFileROBuf.getShort();
695  numEntries = indexFileROBuf.getInt();
696  numBytes = indexFileROBuf.getInt();
697  lastFile = indexFileROBuf.getInt();
698 
699  indexFileROBuf.position(indexFileROBuf.position()+4); // this_id
700  indexFileROBuf.position(indexFileROBuf.position()+4); // stats cache cacheAddress
701 
702  tableLen = indexFileROBuf.getInt();
703  }
704 
705  public long getMagic() {
706  return magic;
707  }
708 
709  public int getVersion() {
710  return version;
711  }
712 
713  public int getNumEntries() {
714  return numEntries;
715  }
716 
717  public int getNumBytes() {
718  return numBytes;
719  }
720 
721  public int getLastFile() {
722  return lastFile;
723  }
724 
725  public int getTableLen() {
726  return tableLen;
727  }
728 
729  @Override
730  public String toString() {
731  StringBuilder sb = new StringBuilder();
732 
733  sb.append(String.format("Index Header:"))
734  .append(String.format("\tMagic = %x" , getMagic()) )
735  .append(String.format("\tVersion = %x" , getVersion()) )
736  .append(String.format("\tNumEntries = %x" , getNumEntries()) )
737  .append(String.format("\tNumBytes = %x" , getNumBytes()) )
738  .append(String.format("\tLastFile = %x" , getLastFile()) )
739  .append(String.format("\tTableLen = %x" , getTableLen()) );
740 
741  return sb.toString();
742  }
743  }
744 
748  enum CacheFileTypeEnum {
749  EXTERNAL,
750  RANKINGS,
751  BLOCK_256,
752  BLOCK_1K,
753  BLOCK_4K,
754  BLOCK_FILES,
755  BLOCK_ENTRIES,
756  BLOCK_EVICTED
757  }
758 
759 
760 
783  final class CacheAddress {
784  // sundry constants to parse the bit fields
785  private static final long ADDR_INITIALIZED_MASK = 0x80000000l;
786  private static final long FILE_TYPE_MASK = 0x70000000;
787  private static final long FILE_TYPE_OFFSET = 28;
788  private static final long NUM_BLOCKS_MASK = 0x03000000;
789  private static final long NUM_BLOCKS_OFFSET = 24;
790  private static final long FILE_SELECTOR_MASK = 0x00ff0000;
791  private static final long FILE_SELECTOR_OFFSET = 16;
792  private static final long START_BLOCK_MASK = 0x0000FFFF;
793  private static final long EXTERNAL_FILE_NAME_MASK = 0x0FFFFFFF;
794 
795  private final long uint32CacheAddr;
796  private final CacheFileTypeEnum fileType;
797  private final int numBlocks;
798  private final int startBlock;
799  private final String fileName;
800  private final int fileNumber;
801 
802  private final String cachePath;
803 
804 
810  CacheAddress(long uint32, String cachePath) {
811 
812  uint32CacheAddr = uint32;
813  this.cachePath = cachePath;
814 
815 
816  // analyze the
817  int fileTypeEnc = (int)(uint32CacheAddr & FILE_TYPE_MASK) >> FILE_TYPE_OFFSET;
818  fileType = CacheFileTypeEnum.values()[fileTypeEnc];
819 
820  if (isInitialized()) {
821  if (isInExternalFile()) {
822  fileNumber = (int)(uint32CacheAddr & EXTERNAL_FILE_NAME_MASK);
823  fileName = String.format("f_%06x", getFileNumber() );
824  numBlocks = 0;
825  startBlock = 0;
826  } else {
827  fileNumber = (int)((uint32CacheAddr & FILE_SELECTOR_MASK) >> FILE_SELECTOR_OFFSET);
828  fileName = String.format("data_%d", getFileNumber() );
829  numBlocks = (int)(uint32CacheAddr & NUM_BLOCKS_MASK >> NUM_BLOCKS_OFFSET);
830  startBlock = (int)(uint32CacheAddr & START_BLOCK_MASK);
831  }
832  }
833  else {
834  fileName = null;
835  fileNumber = 0;
836  numBlocks = 0;
837  startBlock = 0;
838  }
839  }
840 
841  boolean isInitialized() {
842  return ((uint32CacheAddr & ADDR_INITIALIZED_MASK) != 0);
843  }
844 
845  CacheFileTypeEnum getFileType() {
846  return fileType;
847  }
848 
853  String getFilename() {
854  return fileName;
855  }
856 
857  String getCachePath() {
858  return cachePath;
859  }
860 
861  boolean isInExternalFile() {
862  return (fileType == CacheFileTypeEnum.EXTERNAL);
863  }
864 
865  int getFileNumber() {
866  return fileNumber;
867  }
868 
869  int getStartBlock() {
870  return startBlock;
871  }
872 
873  int getNumBlocks() {
874  return numBlocks;
875  }
876 
877  int getBlockSize() {
878  switch (fileType) {
879  case RANKINGS:
880  return 36;
881  case BLOCK_256:
882  return 256;
883  case BLOCK_1K:
884  return 1024;
885  case BLOCK_4K:
886  return 4096;
887  case BLOCK_FILES:
888  return 8;
889  case BLOCK_ENTRIES:
890  return 104;
891  case BLOCK_EVICTED:
892  return 48;
893  default:
894  return 0;
895  }
896  }
897 
898  public long getUint32CacheAddr() {
899  return uint32CacheAddr;
900  }
901 
902  @Override
903  public String toString() {
904  StringBuilder sb = new StringBuilder();
905  sb.append(String.format("CacheAddr %08x : %s : filename %s",
906  uint32CacheAddr,
907  isInitialized() ? "Initialized" : "UnInitialized",
908  getFilename()));
909 
910  if ((fileType == CacheFileTypeEnum.BLOCK_256) ||
911  (fileType == CacheFileTypeEnum.BLOCK_1K) ||
912  (fileType == CacheFileTypeEnum.BLOCK_4K) ) {
913  sb.append(String.format(" (%d blocks starting at %08X)",
914  this.getNumBlocks(),
915  this.getStartBlock()
916  ));
917  }
918 
919  return sb.toString();
920  }
921 
922  }
923 
927  enum CacheDataTypeEnum {
928  HTTP_HEADER,
929  UNKNOWN,
930  };
931 
941  final class CacheDataSegment {
942 
943  private int length;
944  private final CacheAddress cacheAddress;
945  private CacheDataTypeEnum type;
946 
947  private boolean isHTTPHeaderHint;
948 
949  private FileWrapper cacheFileCopy = null;
950  private byte[] data = null;
951 
952  private String httpResponse;
953  private final Map<String, String> httpHeaders = new HashMap<>();
954 
955  CacheDataSegment(CacheAddress cacheAddress, int len) {
956  this(cacheAddress, len, false);
957  }
958 
959  CacheDataSegment(CacheAddress cacheAddress, int len, boolean isHTTPHeader ) {
960  this.type = CacheDataTypeEnum.UNKNOWN;
961  this.length = len;
962  this.cacheAddress = cacheAddress;
963  this.isHTTPHeaderHint = isHTTPHeader;
964  }
965 
966  boolean isInExternalFile() {
967  return cacheAddress.isInExternalFile();
968  }
969 
970  boolean hasHTTPHeaders() {
971  return this.type == CacheDataTypeEnum.HTTP_HEADER;
972  }
973 
974  String getHTTPHeader(String key) {
975  return this.httpHeaders.get(key);
976  }
977 
983  String getHTTPHeaders() {
984  if (!hasHTTPHeaders()) {
985  return "";
986  }
987 
988  StringBuilder sb = new StringBuilder();
989  httpHeaders.entrySet().forEach((entry) -> {
990  if (sb.length() > 0) {
991  sb.append(" \n");
992  }
993  sb.append(String.format("%s : %s",
994  entry.getKey(), entry.getValue()));
995  });
996 
997  return sb.toString();
998  }
999 
1000  String getHTTPRespone() {
1001  return httpResponse;
1002  }
1003 
1009  void extract() throws TskCoreException, IngestModuleException {
1010 
1011  // do nothing if already extracted,
1012  if (data != null) {
1013  return;
1014  }
1015 
1016  // Don't extract data from external files.
1017  if (!cacheAddress.isInExternalFile() ) {
1018 
1019  cacheFileCopy = findDataOrIndexFile(cacheAddress.getFilename(), cacheAddress.getCachePath()).get();
1020 
1021  this.data = new byte [length];
1022  ByteBuffer buf = cacheFileCopy.getByteBuffer();
1023  int dataOffset = DATAFILE_HDR_SIZE + cacheAddress.getStartBlock() * cacheAddress.getBlockSize();
1024  buf.position(dataOffset);
1025  buf.get(data, 0, length);
1026 
1027  // if this might be a HTPP header, lets try to parse it as such
1028  if ((isHTTPHeaderHint)) {
1029  String strData = new String(data);
1030  if (strData.contains("HTTP")) {
1031 
1032  // Http headers if present, are usually in frst data segment in an entry
1033  // General Parsing algo:
1034  // - Find start of HTTP header by searching for string "HTTP"
1035  // - Skip to the first 0x00 to get to the end of the HTTP response line, this makrs start of headers section
1036  // - Find the end of the header by searching for 0x00 0x00 bytes
1037  // - Extract the headers section
1038  // - Parse the headers section - each null terminated string is a header
1039  // - Each header is of the format "name: value" e.g.
1040 
1041  type = CacheDataTypeEnum.HTTP_HEADER;
1042 
1043  int startOff = strData.indexOf("HTTP");
1044  Charset charset = Charset.forName("UTF-8");
1045  boolean done = false;
1046  int i = startOff;
1047  int hdrNum = 1;
1048 
1049  while (!done) {
1050  // each header is null terminated
1051  int start = i;
1052  while (i < data.length && data[i] != 0) {
1053  i++;
1054  }
1055 
1056  // http headers are terminated by 0x00 0x00
1057  if (i == data.length || data[i+1] == 0) {
1058  done = true;
1059  }
1060 
1061  int len = (i - start);
1062  String headerLine = new String(data, start, len, charset);
1063 
1064  // first line is the http response
1065  if (hdrNum == 1) {
1066  httpResponse = headerLine;
1067  } else {
1068  int nPos = headerLine.indexOf(':');
1069  if (nPos > 0 ) {
1070  String key = headerLine.substring(0, nPos);
1071  String val= headerLine.substring(nPos+1);
1072  httpHeaders.put(key.toLowerCase(), val);
1073  }
1074  }
1075 
1076  i++;
1077  hdrNum++;
1078  }
1079  }
1080  }
1081  }
1082  }
1083 
1084  String getDataString() throws TskCoreException, IngestModuleException {
1085  if (data == null) {
1086  extract();
1087  }
1088  return new String(data);
1089  }
1090 
1091  byte[] getDataBytes() throws TskCoreException, IngestModuleException {
1092  if (data == null) {
1093  extract();
1094  }
1095  return data.clone();
1096  }
1097 
1098  int getDataLength() {
1099  return this.length;
1100  }
1101 
1102  CacheDataTypeEnum getType() {
1103  return type;
1104  }
1105 
1106  CacheAddress getCacheAddress() {
1107  return cacheAddress;
1108  }
1109 
1110 
1119  String save() throws TskCoreException, IngestModuleException {
1120  String fileName;
1121 
1122  if (cacheAddress.isInExternalFile()) {
1123  fileName = cacheAddress.getFilename();
1124  } else {
1125  fileName = String.format("%s__%08x", cacheAddress.getFilename(), cacheAddress.getUint32CacheAddr());
1126  }
1127 
1128  String filePathName = getAbsOutputFolderName() + cacheAddress.getCachePath() + fileName;
1129  save(filePathName);
1130 
1131  return fileName;
1132  }
1133 
1143  void save(String filePathName) throws TskCoreException, IngestModuleException {
1144 
1145  // Save the data to specified file
1146  if (data == null) {
1147  extract();
1148  }
1149 
1150  // Data in external files is not saved in local files
1151  if (!this.isInExternalFile()) {
1152  // write the
1153  try (FileOutputStream stream = new FileOutputStream(filePathName)) {
1154  stream.write(data);
1155  } catch (IOException ex) {
1156  throw new TskCoreException(String.format("Failed to write output file %s", filePathName), ex);
1157  }
1158  }
1159  }
1160 
1161  @Override
1162  public String toString() {
1163  StringBuilder strBuilder = new StringBuilder();
1164  strBuilder.append(String.format("\t\tData type = : %s, Data Len = %d ",
1165  this.type.toString(), this.length ));
1166 
1167  if (hasHTTPHeaders()) {
1168  String str = getHTTPHeader("content-encoding");
1169  if (str != null) {
1170  strBuilder.append(String.format("\t%s=%s", "content-encoding", str ));
1171  }
1172  }
1173 
1174  return strBuilder.toString();
1175  }
1176 
1177  }
1178 
1179 
1183  enum EntryStateEnum {
1184  ENTRY_NORMAL,
1185  ENTRY_EVICTED,
1186  ENTRY_DOOMED
1187  };
1188 
1189 
1190 // Main structure for an entry on the backing storage.
1191 //
1192 // Each entry has a key, identifying the URL the cache entry pertains to.
1193 // If the key is longer than
1194 // what can be stored on this structure, it will be extended on consecutive
1195 // blocks (adding 256 bytes each time), up to 4 blocks (1024 - 32 - 1 chars).
1196 // After that point, the whole key will be stored as a data block or external
1197 // file.
1198 //
1199 // Each entry can have upto 4 data segments
1200 //
1201 // struct EntryStore {
1202 // uint32 hash; // Full hash of the key.
1203 // CacheAddr next; // Next entry with the same hash or bucket.
1204 // CacheAddr rankings_node; // Rankings node for this entry.
1205 // int32 reuse_count; // How often is this entry used.
1206 // int32 refetch_count; // How often is this fetched from the net.
1207 // int32 state; // Current state.
1208 // uint64 creation_time;
1209 // int32 key_len;
1210 // CacheAddr long_key; // Optional cacheAddress of a long key.
1211 // int32 data_size[4]; // We can store up to 4 data streams for each
1212 // CacheAddr data_addr[4]; // entry.
1213 // uint32 flags; // Any combination of EntryFlags.
1214 // int32 pad[4];
1215 // uint32 self_hash; // The hash of EntryStore up to this point.
1216 // char key[256 - 24 * 4]; // null terminated
1217 // };
1218 
1222  final class CacheEntry {
1223 
1224  // each entry is 256 bytes. The last section of the entry, after all the other fields is a null terminated key
1225  private static final int MAX_KEY_LEN = 256-24*4;
1226 
1227  private final CacheAddress selfAddress;
1228  private final FileWrapper cacheFileCopy;
1229 
1230  private final long hash;
1231  private final CacheAddress nextAddress;
1232  private final CacheAddress rankingsNodeAddress;
1233 
1234  private final int reuseCount;
1235  private final int refetchCount;
1236  private final EntryStateEnum state;
1237 
1238  private final long creationTime;
1239  private final int keyLen;
1240 
1241  private final CacheAddress longKeyAddresses; // cacheAddress of the key, if the key is external to the entry
1242 
1243  private final int[] dataSegmentSizes;
1244  private final CacheAddress[] dataSegmentIndexFileEntries;
1245  private List<CacheDataSegment> dataSegments;
1246 
1247  private final long flags;
1248 
1249  private String key; // Key may be found within the entry or may be external
1250 
1251  CacheEntry(CacheAddress cacheAdress, FileWrapper cacheFileCopy ) {
1252  this.selfAddress = cacheAdress;
1253  this.cacheFileCopy = cacheFileCopy;
1254 
1255  ByteBuffer fileROBuf = cacheFileCopy.getByteBuffer();
1256 
1257  int entryOffset = DATAFILE_HDR_SIZE + cacheAdress.getStartBlock() * cacheAdress.getBlockSize();
1258 
1259  // reposition the buffer to the the correct offset
1260  fileROBuf.position(entryOffset);
1261 
1262  hash = fileROBuf.getInt() & UINT32_MASK;
1263 
1264  long uint32 = fileROBuf.getInt() & UINT32_MASK;
1265  nextAddress = (uint32 != 0) ? new CacheAddress(uint32, selfAddress.getCachePath()) : null;
1266 
1267  uint32 = fileROBuf.getInt() & UINT32_MASK;
1268  rankingsNodeAddress = (uint32 != 0) ? new CacheAddress(uint32, selfAddress.getCachePath()) : null;
1269 
1270  reuseCount = fileROBuf.getInt();
1271  refetchCount = fileROBuf.getInt();
1272 
1273  state = EntryStateEnum.values()[fileROBuf.getInt()];
1274  creationTime = (fileROBuf.getLong() / 1000000) - Long.valueOf("11644473600");
1275 
1276  keyLen = fileROBuf.getInt();
1277 
1278  uint32 = fileROBuf.getInt() & UINT32_MASK;
1279  longKeyAddresses = (uint32 != 0) ? new CacheAddress(uint32, selfAddress.getCachePath()) : null;
1280 
1281  dataSegments = null;
1282  dataSegmentSizes= new int[4];
1283  for (int i = 0; i < 4; i++) {
1284  dataSegmentSizes[i] = fileROBuf.getInt();
1285  }
1286  dataSegmentIndexFileEntries = new CacheAddress[4];
1287  for (int i = 0; i < 4; i++) {
1288  dataSegmentIndexFileEntries[i] = new CacheAddress(fileROBuf.getInt() & UINT32_MASK, selfAddress.getCachePath());
1289  }
1290 
1291  flags = fileROBuf.getInt() & UINT32_MASK;
1292  // skip over pad
1293  for (int i = 0; i < 4; i++) {
1294  fileROBuf.getInt();
1295  }
1296 
1297  // skip over self hash
1298  fileROBuf.getInt();
1299 
1300  // get the key
1301  if (longKeyAddresses != null) {
1302  // Key is stored outside of the entry
1303  try {
1304  CacheDataSegment data = new CacheDataSegment(longKeyAddresses, this.keyLen, true);
1305  key = data.getDataString();
1306  } catch (TskCoreException | IngestModuleException ex) {
1307  logger.log(Level.WARNING, String.format("Failed to get external key from address %s", longKeyAddresses)); //NON-NLS
1308  }
1309  }
1310  else { // key stored within entry
1311  StringBuilder strBuilder = new StringBuilder(MAX_KEY_LEN);
1312  int keyLen = 0;
1313  while (fileROBuf.remaining() > 0 && keyLen < MAX_KEY_LEN) {
1314  char keyChar = (char)fileROBuf.get();
1315  if (keyChar == '\0') {
1316  break;
1317  }
1318  strBuilder.append(keyChar);
1319  keyLen++;
1320  }
1321 
1322  key = strBuilder.toString();
1323  }
1324  }
1325 
1326  public CacheAddress getCacheAddress() {
1327  return selfAddress;
1328  }
1329 
1330  public long getHash() {
1331  return hash;
1332  }
1333 
1334  public CacheAddress getNextCacheAddress() {
1335  return nextAddress;
1336  }
1337 
1338  public int getReuseCount() {
1339  return reuseCount;
1340  }
1341 
1342  public int getRefetchCount() {
1343  return refetchCount;
1344  }
1345 
1346  public EntryStateEnum getState() {
1347  return state;
1348  }
1349 
1350  public long getCreationTime() {
1351  return creationTime;
1352  }
1353 
1354  public long getFlags() {
1355  return flags;
1356  }
1357 
1358  public String getKey() {
1359  return key;
1360  }
1361 
1370  public List<CacheDataSegment> getDataSegments() throws TskCoreException, IngestModuleException {
1371 
1372  if (dataSegments == null) {
1373  dataSegments = new ArrayList<>();
1374  for (int i = 0; i < 4; i++) {
1375  if (dataSegmentSizes[i] > 0) {
1376  CacheDataSegment cacheData = new CacheDataSegment(dataSegmentIndexFileEntries[i], dataSegmentSizes[i], true );
1377 
1378  cacheData.extract();
1379  dataSegments.add(cacheData);
1380  }
1381  }
1382  }
1383  return dataSegments;
1384  }
1385 
1393  boolean hasHTTPHeaders() {
1394  if ((dataSegments == null) || dataSegments.isEmpty()) {
1395  return false;
1396  }
1397  return dataSegments.get(0).hasHTTPHeaders();
1398  }
1399 
1406  String getHTTPHeader(String key) {
1407  if ((dataSegments == null) || dataSegments.isEmpty()) {
1408  return null;
1409  }
1410  // First data segment has the HTTP headers, if any
1411  return dataSegments.get(0).getHTTPHeader(key);
1412  }
1413 
1419  String getHTTPHeaders() {
1420  if ((dataSegments == null) || dataSegments.isEmpty()) {
1421  return null;
1422  }
1423  // First data segment has the HTTP headers, if any
1424  return dataSegments.get(0).getHTTPHeaders();
1425  }
1426 
1435  boolean isBrotliCompressed() {
1436 
1437  if (hasHTTPHeaders() ) {
1438  String encodingHeader = getHTTPHeader("content-encoding");
1439  if (encodingHeader!= null) {
1440  return encodingHeader.trim().equalsIgnoreCase("br");
1441  }
1442  }
1443 
1444  return false;
1445  }
1446 
1447  @Override
1448  public String toString() {
1449  StringBuilder sb = new StringBuilder();
1450  sb.append(String.format("Entry = Hash: %08x, State: %s, ReuseCount: %d, RefetchCount: %d",
1451  this.hash, this.state.toString(), this.reuseCount, this.refetchCount ))
1452  .append(String.format("\n\tKey: %s, Keylen: %d",
1453  this.key, this.keyLen, this.reuseCount, this.refetchCount ))
1454  .append(String.format("\n\tCreationTime: %s",
1455  TimeUtilities.epochToTime(this.creationTime) ))
1456  .append(String.format("\n\tNext Address: %s",
1457  (nextAddress != null) ? nextAddress.toString() : "None"));
1458 
1459  for (int i = 0; i < 4; i++) {
1460  if (dataSegmentSizes[i] > 0) {
1461  sb.append(String.format("\n\tData %d: cache address = %s, Data = %s",
1462  i, dataSegmentIndexFileEntries[i].toString(),
1463  (dataSegments != null)
1464  ? dataSegments.get(i).toString()
1465  : "Data not retrived yet."));
1466  }
1467  }
1468 
1469  return sb.toString();
1470  }
1471  }
1472 }

Copyright © 2012-2020 Basis Technology. Generated on: Wed Apr 8 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.