Autopsy  4.14.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
IngestJobSettings.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2014-2018 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.ingest;
20 
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.FileOutputStream;
24 import java.io.IOException;
25 import java.nio.file.Files;
26 import java.nio.file.Path;
27 import java.nio.file.Paths;
28 import java.util.ArrayList;
29 import java.util.Collection;
30 import java.util.Collections;
31 import java.util.HashSet;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Objects;
35 import java.util.logging.Level;
36 import org.openide.util.NbBundle;
37 import org.openide.util.io.NbObjectInputStream;
38 import org.openide.util.io.NbObjectOutputStream;
45 
49 public final class IngestJobSettings {
50 
51  private static final String ENABLED_MODULES_PROPERTY = "Enabled_Ingest_Modules"; //NON-NLS
52  private static final String DISABLED_MODULES_PROPERTY = "Disabled_Ingest_Modules"; //NON-NLS
53  private static final String LAST_FILE_INGEST_FILTER_PROPERTY = "Last_File_Ingest_Filter"; //NON-NLS
54  private static final String MODULE_SETTINGS_FOLDER = "IngestModuleSettings"; //NON-NLS
55  private static final String MODULE_SETTINGS_FOLDER_PATH = Paths.get(PlatformUtil.getUserConfigDirectory(), IngestJobSettings.MODULE_SETTINGS_FOLDER).toAbsolutePath().toString();
56  private static final String MODULE_SETTINGS_FILE_EXT = ".settings"; //NON-NLS
57  private static final CharSequence PYTHON_CLASS_PROXY_PREFIX = "org.python.proxies.".subSequence(0, "org.python.proxies.".length() - 1); //NON-NLS
58  private static final Logger logger = Logger.getLogger(IngestJobSettings.class.getName());
59  private final IngestType ingestType;
60  private final List<IngestModuleTemplate> moduleTemplates = new ArrayList<>();
61  private final List<String> warnings = new ArrayList<>();
62  private String executionContext;
64  private String moduleSettingsFolderPath;
65 
78  static Path getSavedModuleSettingsFolder(String executionContext) {
79  return Paths.get(IngestJobSettings.MODULE_SETTINGS_FOLDER_PATH, executionContext);
80  }
81 
93  public IngestJobSettings(final String executionContext) {
95  }
96 
111  public IngestJobSettings(String executionContext, IngestType ingestType) {
112  this.ingestType = ingestType;
113  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
114  this.executionContext = executionContext;
115  } else {
116  this.executionContext = executionContext + "." + this.ingestType.name();
117  }
119  this.load();
120  }
121 
135  public IngestJobSettings(String executionContext, IngestType ingestType, Collection<IngestModuleTemplate> moduleTemplates) {
136  this.ingestType = ingestType;
137  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
138  this.executionContext = executionContext;
139  } else {
140  this.executionContext = executionContext + "." + this.ingestType.name();
141  }
142  this.moduleTemplates.addAll(moduleTemplates);
143  }
144 
159  public IngestJobSettings(String executionContext, IngestType ingestType, Collection<IngestModuleTemplate> moduleTemplates, FilesSet fileFilter) {
161  this.setFileFilter(fileFilter);
162  }
163 
171  return Paths.get(IngestJobSettings.MODULE_SETTINGS_FOLDER_PATH, executionContext);
172  }
173 
177  public void save() {
179  this.store();
180  }
181 
191  public void saveAs(String executionContext) {
192  this.executionContext = executionContext;
194  this.store();
195  }
196 
203  public List<String> getWarnings() {
204  List<String> warningMessages = new ArrayList<>(this.warnings);
205  this.warnings.clear();
206  return warningMessages;
207  }
208 
218  public String getExecutionContext() {
219  return this.executionContext;
220  }
221 
228  if (fileFilter == null) {
229  fileFilter = FilesSetsManager.getDefaultFilter();
230  }
231  return fileFilter;
232  }
233 
239  public void setFileFilter(FilesSet fileIngestFilter) {
240  this.fileFilter = fileIngestFilter;
241  }
242 
248  public List<IngestModuleTemplate> getIngestModuleTemplates() {
249  return Collections.unmodifiableList(this.moduleTemplates);
250  }
251 
257  public void setIngestModuleTemplates(List<IngestModuleTemplate> moduleTemplates) {
258  this.moduleTemplates.clear();
259  this.moduleTemplates.addAll(moduleTemplates);
260  }
261 
267  public List<IngestModuleTemplate> getEnabledIngestModuleTemplates() {
268  List<IngestModuleTemplate> enabledModuleTemplates = new ArrayList<>();
269  for (IngestModuleTemplate moduleTemplate : this.moduleTemplates) {
270  if (moduleTemplate.isEnabled()) {
271  enabledModuleTemplates.add(moduleTemplate);
272  }
273  }
274  return enabledModuleTemplates;
275  }
276 
283  public boolean getProcessUnallocatedSpace() {
284  boolean processUnallocated = true;
285  if (!Objects.isNull(this.fileFilter)) {
286  processUnallocated = (this.fileFilter.ingoresUnallocatedSpace() == false);
287  }
288  return processUnallocated;
289  }
290 
296  try {
297  Path folder = getSavedModuleSettingsFolder();
298  Files.createDirectories(folder);
299  this.moduleSettingsFolderPath = folder.toAbsolutePath().toString();
300  } catch (IOException | SecurityException ex) {
301  logger.log(Level.SEVERE, "Failed to create ingest module settings directory " + this.moduleSettingsFolderPath, ex); //NON-NLS
302  this.warnings.add(NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.createModuleSettingsFolder.warning")); //NON-NLS
303  }
304  }
305 
310  private void load() {
315  List<IngestModuleFactory> moduleFactories = new ArrayList<>();
316  List<IngestModuleFactory> allModuleFactories = IngestModuleFactoryLoader.getIngestModuleFactories();
317  HashSet<String> loadedModuleNames = new HashSet<>();
318 
319  // Add modules that are going to be used for this ingest depending on type.
320  for (IngestModuleFactory moduleFactory : allModuleFactories) {
321  if (this.ingestType.equals(IngestType.ALL_MODULES)) {
322  moduleFactories.add(moduleFactory);
323  } else if (this.ingestType.equals(IngestType.DATA_SOURCE_ONLY) && moduleFactory.isDataSourceIngestModuleFactory()) {
324  moduleFactories.add(moduleFactory);
325  } else if (this.ingestType.equals(IngestType.FILES_ONLY) && moduleFactory.isFileIngestModuleFactory()) {
326  moduleFactories.add(moduleFactory);
327  }
328  }
329 
330  for (IngestModuleFactory moduleFactory : moduleFactories) {
331  loadedModuleNames.add(moduleFactory.getModuleDisplayName());
332  }
333 
341  final String plasoModuleName = "Plaso";
342  boolean plasoLoaded = loadedModuleNames.contains(plasoModuleName);
343  if (plasoLoaded) {
344  loadedModuleNames.remove(plasoModuleName);
345  }
346 
351  HashSet<String> enabledModuleNames = getModulesNames(executionContext, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(loadedModuleNames));
352  HashSet<String> disabledModuleNames = getModulesNames(executionContext, IngestJobSettings.DISABLED_MODULES_PROPERTY, plasoModuleName); //NON-NLS
353 
354  // If plaso was loaded, but appears in neither the enabled nor the
355  // disabled list, add it to the disabled list.
356  if (!enabledModuleNames.contains(plasoModuleName) && !disabledModuleNames.contains(plasoModuleName)) {
357  disabledModuleNames.add(plasoModuleName);
358  }
359 
360  //Put plaso back into loadedModuleNames
361  if (plasoLoaded) {
362  loadedModuleNames.add(plasoModuleName);
363  }
364 
368  List<String> missingModuleNames = new ArrayList<>();
369  for (String moduleName : enabledModuleNames) {
370  if (!loadedModuleNames.contains(moduleName)) {
371  missingModuleNames.add(moduleName);
372  }
373  }
374  for (String moduleName : disabledModuleNames) {
375  if (!loadedModuleNames.contains(moduleName)) {
376  missingModuleNames.add(moduleName);
377  }
378  }
379  for (String moduleName : missingModuleNames) {
380  enabledModuleNames.remove(moduleName);
381  disabledModuleNames.remove(moduleName);
382  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.missingModule.warning", moduleName); //NON-NLS
383  logger.log(Level.WARNING, warning);
384  this.warnings.add(warning);
385  }
386 
391  for (IngestModuleFactory moduleFactory : moduleFactories) {
392  IngestModuleTemplate moduleTemplate = new IngestModuleTemplate(moduleFactory, loadModuleSettings(moduleFactory));
393  String moduleName = moduleTemplate.getModuleName();
394  if (enabledModuleNames.contains(moduleName)) {
395  moduleTemplate.setEnabled(true);
396  } else if (disabledModuleNames.contains(moduleName)) {
397  moduleTemplate.setEnabled(false);
398  } else {
399  // The module factory was loaded, but the module name does not
400  // appear in the enabled/disabled module settings. Treat the
401  // module as a new module and enable it by default.
402  moduleTemplate.setEnabled(true);
403  enabledModuleNames.add(moduleName);
404  }
405  this.moduleTemplates.add(moduleTemplate);
406  }
407 
412  ModuleSettings.setConfigSetting(this.executionContext, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(enabledModuleNames));
413  ModuleSettings.setConfigSetting(this.executionContext, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(disabledModuleNames));
414 
420  }
421  try {
422  Map<String, FilesSet> fileIngestFilters = FilesSetsManager.getInstance()
425  fileIngestFilters.put(fSet.getName(), fSet);
426  }
427  this.fileFilter = fileIngestFilters.get(ModuleSettings.getConfigSetting(this.executionContext, IngestJobSettings.LAST_FILE_INGEST_FILTER_PROPERTY));
429  this.fileFilter = FilesSetsManager.getDefaultFilter();
430  logger.log(Level.SEVERE, "Failed to get file filter from .properties file, default filter being used", ex); //NON-NLS
431  }
432  }
433 
445  private static HashSet<String> getModulesNames(String executionContext, String propertyName, String defaultSetting) {
446  if (ModuleSettings.settingExists(executionContext, propertyName) == false) {
447  ModuleSettings.setConfigSetting(executionContext, propertyName, defaultSetting);
448  }
449  HashSet<String> moduleNames = new HashSet<>();
450  String modulesSetting = ModuleSettings.getConfigSetting(executionContext, propertyName);
451  if (!modulesSetting.isEmpty()) {
452  String[] settingNames = modulesSetting.split(", ");
453  for (String name : settingNames) {
454  /*
455  * Map some obsolete core ingest module names to the current
456  * core ingest module names.
457  */
458  switch (name) {
459  case "Thunderbird Parser": //NON-NLS
460  case "MBox Parser": //NON-NLS
461  moduleNames.add("Email Parser"); //NON-NLS
462  break;
463  case "File Extension Mismatch Detection": //NON-NLS
464  moduleNames.add("Extension Mismatch Detector"); //NON-NLS
465  break;
466  case "EWF Verify": //NON-NLS
467  case "E01 Verify": //NON-NLS
468  case "E01 Verifier": // NON-NLS
469  moduleNames.add("Data Source Integrity"); //NON-NLS
470  break;
471  case "Archive Extractor": //NON-NLS
472  moduleNames.add("Embedded File Extractor"); //NON-NLS
473  break;
474  default:
475  moduleNames.add(name);
476  }
477  }
478  }
479  return moduleNames;
480  }
481 
490  static List<String> getEnabledModules(String context) {
491  return new ArrayList<>(getModulesNames(context, ENABLED_MODULES_PROPERTY, ""));
492  }
493 
504  private boolean isPythonModuleSettingsFile(String moduleSettingsFilePath) {
505  return moduleSettingsFilePath.contains(PYTHON_CLASS_PROXY_PREFIX);
506  }
507 
516  IngestModuleIngestJobSettings settings = null;
517  String moduleSettingsFilePath = getModuleSettingsFilePath(factory);
518  File settingsFile = new File(moduleSettingsFilePath);
519  if (settingsFile.exists()) {
520  try (NbObjectInputStream in = new NbObjectInputStream(new FileInputStream(settingsFile.getAbsolutePath()))) {
521  settings = (IngestModuleIngestJobSettings) in.readObject();
522  } catch (IOException | ClassNotFoundException ex) {
523  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsLoad.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
524  logger.log(Level.WARNING, warning, ex);
525  this.warnings.add(warning);
526  }
527  }
528  if (settings == null) {
529  settings = factory.getDefaultIngestJobSettings();
530  }
531  return settings;
532  }
533 
543  String fileName = FactoryClassNameNormalizer.normalize(factory.getClass().getCanonicalName()) + IngestJobSettings.MODULE_SETTINGS_FILE_EXT;
544  Path path = Paths.get(this.moduleSettingsFolderPath, fileName);
545  return path.toAbsolutePath().toString();
546  }
547 
551  private void store() {
555  HashSet<String> enabledModuleNames = new HashSet<>();
556  HashSet<String> disabledModuleNames = new HashSet<>();
557  for (IngestModuleTemplate moduleTemplate : moduleTemplates) {
558  saveModuleSettings(moduleTemplate.getModuleFactory(), moduleTemplate.getModuleSettings());
559  String moduleName = moduleTemplate.getModuleName();
560  if (moduleTemplate.isEnabled()) {
561  enabledModuleNames.add(moduleName);
562  } else {
563  disabledModuleNames.add(moduleName);
564  }
565  }
566  ModuleSettings.setConfigSetting(this.executionContext, IngestJobSettings.ENABLED_MODULES_PROPERTY, makeCsvList(enabledModuleNames));
567  ModuleSettings.setConfigSetting(this.executionContext, IngestJobSettings.DISABLED_MODULES_PROPERTY, makeCsvList(disabledModuleNames));
568 
572  ModuleSettings.setConfigSetting(this.executionContext, LAST_FILE_INGEST_FILTER_PROPERTY, fileFilter.getName());
573  }
574 
582  String moduleSettingsFilePath = getModuleSettingsFilePath(factory);
583  try (NbObjectOutputStream out = new NbObjectOutputStream(new FileOutputStream(moduleSettingsFilePath))) {
584  out.writeObject(settings);
585  } catch (IOException ex) {
586  String warning = NbBundle.getMessage(IngestJobSettings.class, "IngestJobSettings.moduleSettingsSave.warning", factory.getModuleDisplayName(), this.executionContext); //NON-NLS
587  logger.log(Level.SEVERE, warning, ex);
588  this.warnings.add(warning);
589  }
590  }
591 
600  private static String makeCsvList(Collection<String> collection) {
601  if (collection == null || collection.isEmpty()) {
602  return "";
603  }
604 
605  ArrayList<String> list = new ArrayList<>();
606  list.addAll(collection);
607  StringBuilder csvList = new StringBuilder();
608  for (int i = 0; i < list.size() - 1; ++i) {
609  csvList.append(list.get(i)).append(", ");
610  }
611  csvList.append(list.get(list.size() - 1));
612  return csvList.toString();
613  }
614 
618  public enum IngestType {
619 
631  FILES_ONLY
632  }
633 
634 }
static synchronized String getConfigSetting(String moduleName, String settingName)
IngestJobSettings(final String executionContext)
void saveModuleSettings(IngestModuleFactory factory, IngestModuleIngestJobSettings settings)
List< IngestModuleTemplate > getEnabledIngestModuleTemplates()
static HashSet< String > getModulesNames(String executionContext, String propertyName, String defaultSetting)
IngestModuleIngestJobSettings getDefaultIngestJobSettings()
static String makeCsvList(Collection< String > collection)
void setIngestModuleTemplates(List< IngestModuleTemplate > moduleTemplates)
static synchronized boolean settingExists(String moduleName, String settingName)
final List< IngestModuleTemplate > moduleTemplates
IngestJobSettings(String executionContext, IngestType ingestType, Collection< IngestModuleTemplate > moduleTemplates, FilesSet fileFilter)
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
IngestModuleIngestJobSettings loadModuleSettings(IngestModuleFactory factory)
String getModuleSettingsFilePath(IngestModuleFactory factory)
List< IngestModuleTemplate > getIngestModuleTemplates()
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
IngestJobSettings(String executionContext, IngestType ingestType, Collection< IngestModuleTemplate > moduleTemplates)
boolean isPythonModuleSettingsFile(String moduleSettingsFilePath)
IngestJobSettings(String executionContext, IngestType ingestType)

Copyright © 2012-2020 Basis Technology. Generated on: Wed Apr 8 2020
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.