Autopsy  4.6.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestJobSettings.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.FileOutputStream;
24 import java.io.IOException;
25 import java.nio.file.Files;
26 import java.nio.file.Path;
27 import java.nio.file.Paths;
28 import java.util.ArrayList;
29 import java.util.Collection;
30 import java.util.Collections;
31 import java.util.HashSet;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Objects;
35 import java.util.logging.Level;
36 import org.openide.util.NbBundle;
37 import org.openide.util.io.NbObjectInputStream;
38 import org.openide.util.io.NbObjectOutputStream;
39 import org.python.util.PythonObjectInputStream;
45 
49 public final class IngestJobSettings {
50 
51  private static final String ENABLED_MODULES_PROPERTY = "Enabled_Ingest_Modules"; //NON-NLS
52  private static final String DISABLED_MODULES_PROPERTY = "Disabled_Ingest_Modules"; //NON-NLS
53  private static final String LAST_FILE_INGEST_FILTER_PROPERTY = "Last_File_Ingest_Filter"; //NON-NLS
54  private static final String MODULE_SETTINGS_FOLDER = "IngestModuleSettings"; //NON-NLS
55  private static final String MODULE_SETTINGS_FOLDER_PATH = Paths.get(PlatformUtil.getUserConfigDirectory(), IngestJobSettings.MODULE_SETTINGS_FOLDER).toAbsolutePath().toString();
56  private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS
57  private static final CharSequence PYTHON_CLASS_PROXY_PREFIX = "org.python.proxies.".subSequence(0, "org.python.proxies.".length() - 1); //NON-NLS
58  private static final Logger logger = Logger.getLogger(IngestJobSettings.class.getName());
59  private final IngestType ingestType;
60  private final List<IngestModuleTemplate> moduleTemplates = new ArrayList<>();
61  private final List<String> warnings = new ArrayList<>();
62  private String executionContext;
64  private String moduleSettingsFolderPath;
65 
78  static Path getSavedModuleSettingsFolder(String executionContext) {
79  return Paths.get(IngestJobSettings.MODULE_SETTINGS_FOLDER_PATH, executionContext);
80  }
81 
93  public IngestJobSettings(final String executionContext) {
95  }
96 
111  public IngestJobSettings(String executionContext, IngestType ingestType) {
112  this.ingestType = ingestType;
113  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
114  this.executionContext = executionContext;
115  } else {
116  this.executionContext = executionContext + "." + this.ingestType.name();
117  }
119  this.load();
120  }
121 
135  public IngestJobSettings(String executionContext, IngestType ingestType, Collection<IngestModuleTemplate> moduleTemplates) {
136  this.ingestType = ingestType;
137  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
138  this.executionContext = executionContext;
139  } else {
140  this.executionContext = executionContext + "." + this.ingestType.name();
141  }
142  this.moduleTemplates.addAll(moduleTemplates);
143  }
144 
159  public IngestJobSettings(String executionContext, IngestType ingestType, Collection<IngestModuleTemplate> moduleTemplates, FilesSet fileFilter) {
161  this.setFileFilter(fileFilter);
162  }
163 
171  return Paths.get(IngestJobSettings.MODULE_SETTINGS_FOLDER_PATH, executionContext);
172  }
173 
177  public void save() {
179  this.store();
180  }
181 
191  public void saveAs(String executionContext) {
192  this.executionContext = executionContext;
194  this.store();
195  }
196 
203  public List<String> getWarnings() {
204  List<String> warningMessages = new ArrayList<>(this.warnings);
205  this.warnings.clear();
206  return warningMessages;
207  }
208 
218  public String getExecutionContext() {
219  return this.executionContext;
220  }
221 
228  if (fileFilter == null) {
229  fileFilter = FilesSetsManager.getDefaultFilter();
230  }
231  return fileFilter;
232  }
233 
239  public void setFileFilter(FilesSet fileIngestFilter) {
240  this.fileFilter = fileIngestFilter;
241  }
242 
248  public List<IngestModuleTemplate> getIngestModuleTemplates() {
249  return Collections.unmodifiableList(this.moduleTemplates);
250  }
251 
257  public void setIngestModuleTemplates(List<IngestModuleTemplate> moduleTemplates) {
258  this.moduleTemplates.clear();
259  this.moduleTemplates.addAll(moduleTemplates);
260  }
261 
267  public List<IngestModuleTemplate> getEnabledIngestModuleTemplates() {
268  List<IngestModuleTemplate> enabledModuleTemplates = new ArrayList<>();
269  for (IngestModuleTemplate moduleTemplate : this.moduleTemplates) {
270  if (moduleTemplate.isEnabled()) {
271  enabledModuleTemplates.add(moduleTemplate);
272  }
273  }
274  return enabledModuleTemplates;
275  }
276 
283  public boolean getProcessUnallocatedSpace() {
284  boolean processUnallocated = true;
285  if (!Objects.isNull(this.fileFilter)) {
286  processUnallocated = (this.fileFilter.ingoresUnallocatedSpace() == false);
287  }
288  return processUnallocated;
289  }
290 
296  try {
297  Path folder = getSavedModuleSettingsFolder();
298  Files.createDirectories(folder);
299  this.moduleSettingsFolderPath = folder.toAbsolutePath().toString();
300  } catch (IOException | SecurityException ex) {
301  logger.log(Level.SEVERE, "Failed to create ingest module settings directory " + this.moduleSettingsFolderPath, ex); //NON-NLS
302  this.warnings.add(NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.createModuleSettingsFolder.warning")); //NON-NLS
303  }
304  }
305 
310  private void load() {
315  List<IngestModuleFactory> moduleFactories = new ArrayList<>();
316  List<IngestModuleFactory> allModuleFactories = IngestModuleFactoryLoader.getIngestModuleFactories();
317  HashSet<String> loadedModuleNames = new HashSet<>();
318 
319  // Add modules that are going to be used for this ingest depending on type.
320  for (IngestModuleFactory moduleFactory : allModuleFactories) {
321  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
322  moduleFactories.add(moduleFactory);
323  } else if (this.ingestType.equals(IngestType.DATA_SOURCE_ONLY) && moduleFactory.isDataSourceIngestModuleFactory()) {
324  moduleFactories.add(moduleFactory);
325  } else if (this.ingestType.equals(IngestType.FILES_ONLY) && moduleFactory.isFileIngestModuleFactory()) {
326  moduleFactories.add(moduleFactory);
327  }
328  }
329 
330  for (IngestModuleFactory moduleFactory : moduleFactories) {
331  loadedModuleNames.add(moduleFactory.getModuleDisplayName());
332  }
333 
338  HashSet<String> enabledModuleNames = getModulesNames(executionContext, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(loadedModuleNames));
339  HashSet<String> disabledModuleNames = getModulesNames(executionContext, IngestJobSettings.DISABLED_MODULES_PROPERTY, ""); //NON-NLS
340 
344  List<String> missingModuleNames = new ArrayList<>();
345  for (String moduleName : enabledModuleNames) {
346  if (!loadedModuleNames.contains(moduleName)) {
347  missingModuleNames.add(moduleName);
348  }
349  }
350  for (String moduleName : disabledModuleNames) {
351  if (!loadedModuleNames.contains(moduleName)) {
352  missingModuleNames.add(moduleName);
353  }
354  }
355  for (String moduleName : missingModuleNames) {
356  enabledModuleNames.remove(moduleName);
357  disabledModuleNames.remove(moduleName);
358  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.missingModule.warning", moduleName); //NON-NLS
359  logger.log(Level.WARNING, warning);
360  this.warnings.add(warning);
361  }
362 
367  for (IngestModuleFactory moduleFactory : moduleFactories) {
368  IngestModuleTemplate moduleTemplate = new IngestModuleTemplate(moduleFactory, loadModuleSettings(moduleFactory));
369  String moduleName = moduleTemplate.getModuleName();
370  if (enabledModuleNames.contains(moduleName)) {
371  moduleTemplate.setEnabled(true);
372  } else if (disabledModuleNames.contains(moduleName)) {
373  moduleTemplate.setEnabled(false);
374  } else {
375  // The module factory was loaded, but the module name does not
376  // appear in the enabled/disabled module settings. Treat the
377  // module as a new module and enable it by default.
378  moduleTemplate.setEnabled(true);
379  enabledModuleNames.add(moduleName);
380  }
381  this.moduleTemplates.add(moduleTemplate);
382  }
383 
388  ModuleSettings.setConfigSetting(this.executionContext, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(enabledModuleNames));
389  ModuleSettings.setConfigSetting(this.executionContext, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(disabledModuleNames));
390 
396  }
397  try {
398  Map<String, FilesSet> fileIngestFilters = FilesSetsManager.getInstance()
401  fileIngestFilters.put(fSet.getName(), fSet);
402  }
403  this.fileFilter = fileIngestFilters.get(ModuleSettings.getConfigSetting(this.executionContext, IngestJobSettings.LAST_FILE_INGEST_FILTER_PROPERTY));
405  this.fileFilter = FilesSetsManager.getDefaultFilter();
406  logger.log(Level.SEVERE, "Failed to get file filter from .properties file, default filter being used", ex); //NON-NLS
407  }
408  }
409 
421  private static HashSet<String> getModulesNames(String executionContext, String propertyName, String defaultSetting) {
422  if (ModuleSettings.settingExists(executionContext, propertyName) == false) {
423  ModuleSettings.setConfigSetting(executionContext, propertyName, defaultSetting);
424  }
425  HashSet<String> moduleNames = new HashSet<>();
426  String modulesSetting = ModuleSettings.getConfigSetting(executionContext, propertyName);
427  if (!modulesSetting.isEmpty()) {
428  String[] settingNames = modulesSetting.split(", ");
429  for (String name : settingNames) {
430  /*
431  * Map some obsolete core ingest module names to the current
432  * core ingest module names.
433  */
434  switch (name) {
435  case "Thunderbird Parser": //NON-NLS
436  case "MBox Parser": //NON-NLS
437  moduleNames.add("Email Parser"); //NON-NLS
438  break;
439  case "File Extension Mismatch Detection": //NON-NLS
440  moduleNames.add("Extension Mismatch Detector"); //NON-NLS
441  break;
442  case "EWF Verify": //NON-NLS
443  case "E01 Verify": //NON-NLS
444  moduleNames.add("E01 Verifier"); //NON-NLS
445  break;
446  case "Archive Extractor": //NON-NLS
447  moduleNames.add("Embedded File Extractor"); //NON-NLS
448  break;
449  default:
450  moduleNames.add(name);
451  }
452  }
453  }
454  return moduleNames;
455  }
456 
465  static List<String> getEnabledModules(String context) {
466  return new ArrayList<>(getModulesNames(context, ENABLED_MODULES_PROPERTY, ""));
467  }
468 
479  private boolean isPythonModuleSettingsFile(String moduleSettingsFilePath) {
480  return moduleSettingsFilePath.contains(PYTHON_CLASS_PROXY_PREFIX);
481  }
482 
491  IngestModuleIngestJobSettings settings = null;
492  String moduleSettingsFilePath = getModuleSettingsFilePath(factory);
493  File settingsFile = new File(moduleSettingsFilePath);
494  if (settingsFile.exists()) {
495  if (!isPythonModuleSettingsFile(moduleSettingsFilePath)) {
496  try (NbObjectInputStream in = new NbObjectInputStream(new FileInputStream(settingsFile.getAbsolutePath()))) {
497  settings = (IngestModuleIngestJobSettings) in.readObject();
498  } catch (IOException | ClassNotFoundException ex) {
499  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsLoad.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
500  logger.log(Level.WARNING, warning, ex);
501  this.warnings.add(warning);
502  }
503  } else {
504  // @@@ BC Jython serialization is currently broken and this
505  // throws an exception. (-2323). Commenting out so that
506  // Python modules will at least load with default settings.
507 // try (PythonObjectInputStream in = new PythonObjectInputStream(new FileInputStream(settingsFile.getAbsolutePath()))) {
508 // settings = (IngestModuleIngestJobSettings) in.readObject();
509 // } catch (IOException | ClassNotFoundException exception) {
510 // String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsLoad.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
511 // logger.log(Level.WARNING, warning, exception);
512 // this.warnings.add(warning);
513 // }
514  }
515  }
516  if (settings == null) {
517  settings = factory.getDefaultIngestJobSettings();
518  }
519  return settings;
520  }
521 
531  String fileName = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + IngestJobSettings.MODULE_SETTINGS_FILE_EXT;
532  Path path = Paths.get(this.moduleSettingsFolderPath, fileName);
533  return path.toAbsolutePath().toString();
534  }
535 
539  private void store() {
543  HashSet<String> enabledModuleNames = new HashSet<>();
544  HashSet<String> disabledModuleNames = new HashSet<>();
545  for (IngestModuleTemplate moduleTemplate : moduleTemplates) {
546  saveModuleSettings(moduleTemplate.getModuleFactory(), moduleTemplate.getModuleSettings());
547  String moduleName = moduleTemplate.getModuleName();
548  if (moduleTemplate.isEnabled()) {
549  enabledModuleNames.add(moduleName);
550  } else {
551  disabledModuleNames.add(moduleName);
552  }
553  }
554  ModuleSettings.setConfigSetting(this.executionContext, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(enabledModuleNames));
555  ModuleSettings.setConfigSetting(this.executionContext, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(disabledModuleNames));
556 
560  ModuleSettings.setConfigSetting(this.executionContext, LAST_FILE_INGEST_FILTER_PROPERTY, fileFilter.getName());
561  }
562 
570  String moduleSettingsFilePath = getModuleSettingsFilePath(factory);
571  try (NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(moduleSettingsFilePath))) {
572  out.writeObject(settings);
573  } catch (IOException ex) {
574  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsSave.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
575  logger.log(Level.SEVERE, warning, ex);
576  this.warnings.add(warning);
577  }
578  }
579 
588  private static String makeCsvList(Collection<String> collection) {
589  if (collection == null || collection.isEmpty()) {
590  return "";
591  }
592 
593  ArrayList<String> list = new ArrayList<>();
594  list.addAll(collection);
595  StringBuilder csvList = new StringBuilder();
596  for (int i = 0; i < list.size() - 1; ++i) {
597  csvList.append(list.get(i)).append(", ");
598  }
599  csvList.append(list.get(list.size() - 1));
600  return csvList.toString();
601  }
602 
606  public enum IngestType {
607 
619  FILES_ONLY
620  }
621 
622 }
IngestJobSettings(final String executionContext)
void saveModuleSettings(IngestModuleFactory factory, IngestModuleIngestJobSettings settings)
List< IngestModuleTemplate > getEnabledIngestModuleTemplates()
static HashSet< String > getModulesNames(String executionContext, String propertyName, String defaultSetting)
IngestModuleIngestJobSettings getDefaultIngestJobSettings()
static String makeCsvList(Collection< String > collection)
void setIngestModuleTemplates(List< IngestModuleTemplate > moduleTemplates)
final List< IngestModuleTemplate > moduleTemplates
IngestJobSettings(String executionContext, IngestType ingestType, Collection< IngestModuleTemplate > moduleTemplates, FilesSet fileFilter)
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
IngestModuleIngestJobSettings loadModuleSettings(IngestModuleFactory factory)
String getModuleSettingsFilePath(IngestModuleFactory factory)
static String getConfigSetting(String moduleName, String settingName)
List< IngestModuleTemplate > getIngestModuleTemplates()
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
IngestJobSettings(String executionContext, IngestType ingestType, Collection< IngestModuleTemplate > moduleTemplates)
boolean isPythonModuleSettingsFile(String moduleSettingsFilePath)
static boolean settingExists(String moduleName, String settingName)
IngestJobSettings(String executionContext, IngestType ingestType)

Copyright © 2012-2016 Basis Technology. Generated on: Mon May 7 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.