STIXReportModule.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 * 
 * Copyright 2013 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *     http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.modules.stix;

import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.List;
import java.util.logging.Level;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.util.Arrays;
import javax.swing.JPanel;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.namespace.QName;

import org.mitre.cybox.cybox_2.ObjectType;
import org.mitre.cybox.cybox_2.Observable;
import org.mitre.cybox.cybox_2.ObservableCompositionType;
import org.mitre.stix.common_1.IndicatorBaseType;
import org.mitre.stix.indicator_2.Indicator;
import org.mitre.stix.stix_1.STIXPackage;

import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.report.GeneralReportModule;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.report.ReportProgressPanel;
import org.sleuthkit.datamodel.TskCoreException;

import org.mitre.cybox.cybox_2.OperatorTypeEnum;
import org.mitre.cybox.objects.Address;
import org.mitre.cybox.objects.FileObjectType;
import org.mitre.cybox.objects.URIObjectType;
import org.mitre.cybox.objects.EmailMessage;
import org.mitre.cybox.objects.WindowsNetworkShare;
import org.mitre.cybox.objects.AccountObjectType;
import org.mitre.cybox.objects.SystemObjectType;
import org.mitre.cybox.objects.URLHistory;
import org.mitre.cybox.objects.DomainName;
import org.mitre.cybox.objects.WindowsRegistryKey;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.coreutils.ModuleSettings;
import org.sleuthkit.autopsy.report.ReportProgressPanel.ReportStatus;

public class STIXReportModule implements GeneralReportModule {

    private static final Logger logger = Logger.getLogger(STIXReportModule.class.getName());
    private STIXReportModuleConfigPanel configPanel;
    private static STIXReportModule instance = null;
    private String reportPath;
    private boolean reportAllResults;

    private Map<String, ObjectType> idToObjectMap = new HashMap<String, ObjectType>();
    private Map<String, ObservableResult> idToResult = new HashMap<String, ObservableResult>();

    private List<EvalRegistryObj.RegistryFileInfo> registryFileData = null;

    private final boolean skipShortCircuit = true;

    private BufferedWriter output = null;

    // Hidden constructor for the report
    private STIXReportModule() {
    }

    // Get the default implementation of this report
    public static synchronized STIXReportModule getDefault() {
        if (instance == null) {
            instance = new STIXReportModule();
        }
        return instance;
    }

    @Override
    public void generateReport(String baseReportDir, ReportProgressPanel progressPanel) {
        // Start the progress bar and setup the report
        progressPanel.setIndeterminate(false);
        progressPanel.start();
        progressPanel.updateStatusLabel(NbBundle.getMessage(this.getClass(), "STIXReportModule.progress.readSTIX"));
        reportPath = baseReportDir + getRelativeFilePath();

        // Check if the user wants to display all output or just hits
        reportAllResults = configPanel.getShowAllResults();

        // Set up the output file
        try {
            File file = new File(reportPath);
            output = new BufferedWriter(new FileWriter(file));
        } catch (IOException ex) {
            logger.log(Level.SEVERE, String.format("Unable to open STIX report file %s", reportPath), ex); //NON-NLS
            MessageNotifyUtil.Notify.show("STIXReportModule", //NON-NLS
                                          NbBundle.getMessage(this.getClass(),
                                                              "STIXReportModule.notifyMsg.unableToOpenReportFile",
                                                              reportPath),
                    MessageNotifyUtil.MessageType.ERROR);
            progressPanel.complete(ReportStatus.ERROR);
            progressPanel.updateStatusLabel(
                    NbBundle.getMessage(this.getClass(), "STIXReportModule.progress.completedWithErrors"));
            return;
        }

        // Keep track of whether any errors occur during processing
        boolean hadErrors = false;

        // Process the file/directory name entry
        String stixFileName = configPanel.getStixFile();
        if (stixFileName == null) {
            logger.log(Level.SEVERE, "STIXReportModuleConfigPanel.stixFile not initialized "); //NON-NLS
            MessageNotifyUtil.Message.error(
                    NbBundle.getMessage(this.getClass(), "STIXReportModule.notifyErr.noFildDirProvided"));
            progressPanel.complete(ReportStatus.ERROR);
            progressPanel.updateStatusLabel(
                    NbBundle.getMessage(this.getClass(), "STIXReportModule.progress.noFildDirProvided"));
            return;
        }
        if (stixFileName.isEmpty()) {
            logger.log(Level.SEVERE, "No STIX file/directory provided "); //NON-NLS
            MessageNotifyUtil.Message.error(
                    NbBundle.getMessage(this.getClass(), "STIXReportModule.notifyErr.noFildDirProvided"));
            progressPanel.complete(ReportStatus.ERROR);
            progressPanel.updateStatusLabel(
                    NbBundle.getMessage(this.getClass(), "STIXReportModule.progress.noFildDirProvided"));
            return;
        }
        File stixFile = new File(stixFileName);

        if (!stixFile.exists()) {
            logger.log(Level.SEVERE, String.format("Unable to open STIX file/directory %s", stixFileName)); //NON-NLS
            MessageNotifyUtil.Message.error(NbBundle.getMessage(this.getClass(),
                                                                "STIXReportModule.notifyMsg.unableToOpenFileDir",
                                                                stixFileName));
            progressPanel.complete(ReportStatus.ERROR);
            progressPanel.updateStatusLabel(
                    NbBundle.getMessage(this.getClass(), "STIXReportModule.progress.couldNotOpenFileDir", stixFileName));
            return;
        }

        // Store the path
        ModuleSettings.setConfigSetting("STIX", "defaultPath", stixFileName); //NON-NLS

        // Create array of stix file(s)
        File[] stixFiles;
        if (stixFile.isFile()) {
            stixFiles = new File[1];
            stixFiles[0] = stixFile;
        } else {
            stixFiles = stixFile.listFiles();
        }

        // Set the length of the progress bar - we increment twice for each file
        progressPanel.setMaximumProgress(stixFiles.length * 2 + 1);

        // Process each STIX file
        for (File file : stixFiles) {
            try {
                processFile(file.getAbsolutePath(), progressPanel);
            } catch (TskCoreException ex) {
                logger.log(Level.SEVERE, String.format("Unable to process STIX file %s", file), ex); //NON-NLS
                MessageNotifyUtil.Notify.show("STIXReportModule", //NON-NLS
                        ex.getLocalizedMessage(),
                        MessageNotifyUtil.MessageType.ERROR);
                hadErrors = true;
            }
            
            // Clear out the ID maps before loading the next file
            idToObjectMap = new HashMap<String, ObjectType>();
            idToResult = new HashMap<String, ObservableResult>();
        }

        // Close the output file
        if (output != null) {
            try {
                output.close();
            } catch (IOException ex) {
                logger.log(Level.SEVERE, String.format("Error closing STIX report file %s", reportPath), ex); //NON-NLS
            }
        }

        // Set the progress bar to done. If any errors occurred along the way, modify
        // the "complete" message to indicate this.
        if (hadErrors) {
            progressPanel.complete(ReportStatus.ERROR);
            progressPanel.updateStatusLabel(
                    NbBundle.getMessage(this.getClass(), "STIXReportModule.progress.completedWithErrors"));
        } else {
            progressPanel.complete(ReportStatus.COMPLETE);
        }
    }

    private void processFile(String stixFile, ReportProgressPanel progressPanel) throws
            TskCoreException {

        // Load the STIX file
        STIXPackage stix;
        stix = loadSTIXFile(stixFile);

        printFileHeader(stixFile);

        // Save any observables listed up front
        processObservables(stix);
        progressPanel.increment();

        // Make copies of the registry files
        registryFileData = EvalRegistryObj.copyRegistryFiles();

        // Process the indicators
        processIndicators(stix);
        progressPanel.increment();

    }

    private STIXPackage loadSTIXFile(String stixFileName) throws TskCoreException {
        try {
            // Create STIXPackage object from xml.
            File file = new File(stixFileName);
            JAXBContext jaxbContext = JAXBContext.newInstance("org.mitre.stix.stix_1:org.mitre.stix.common_1:org.mitre.stix.indicator_2:" //NON-NLS
                    + "org.mitre.cybox.objects:org.mitre.cybox.cybox_2:org.mitre.cybox.common_2"); //NON-NLS
            Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
            STIXPackage stix = (STIXPackage) jaxbUnmarshaller.unmarshal(file);
            return stix;
        } catch (JAXBException ex) {
            logger.log(Level.SEVERE, String.format("Unable to load STIX file %s", stixFileName), ex.getLocalizedMessage()); //NON-NLS
            throw new TskCoreException("Error loading STIX file (" + ex.toString() + ")"); //NON-NLS
        }
    }

    private void processObservables(STIXPackage stix) {
        if (stix.getObservables() != null) {
            List<Observable> obs = stix.getObservables().getObservables();
            for (Observable o : obs) {
                if (o.getId() != null) {
                    saveToObjectMap(o);
                }
            }
        }
    }

    private void processIndicators(STIXPackage stix) throws TskCoreException {
        if (stix.getIndicators() != null) {
            List<IndicatorBaseType> s = stix.getIndicators().getIndicators();
            for (IndicatorBaseType t : s) {
                if (t instanceof Indicator) {
                    Indicator ind = (Indicator) t;
                    if (ind.getObservable() != null) {
                        if (ind.getObservable().getObject() != null) {
                            ObservableResult result = evaluateSingleObservable(ind.getObservable(), "");
                            if (result.isTrue() || reportAllResults) {
                                writeResultsToFile(ind, result.getDescription(), result.isTrue());
                            }
                            if (result.isTrue()) {
                                saveResultsAsArtifacts(ind, result);
                            }
                        } else if (ind.getObservable().getObservableComposition() != null) {
                            ObservableResult result = evaluateObservableComposition(ind.getObservable().getObservableComposition(), "  ");

                            if (result.isTrue() || reportAllResults) {
                                writeResultsToFile(ind, result.getDescription(), result.isTrue());
                            }
                            if (result.isTrue()) {
                                saveResultsAsArtifacts(ind, result);
                            }
                        }
                    }
                }
            }
        }
    }

    private void saveResultsAsArtifacts(Indicator ind, ObservableResult result) throws TskCoreException {

        if (result.getArtifacts() == null) {
            return;
        }

        // Count of how many artifacts have been created for this indicator. 
        int count = 0;

        for (StixArtifactData s : result.getArtifacts()) {

            // Figure out what name to use for this indicator. If it has a title, 
            // use that. Otherwise use the ID. If both are missing, use a
            // generic heading.
            if (ind.getTitle() != null) {
                s.createArtifact(ind.getTitle());
            } else if (ind.getId() != null) {
                s.createArtifact(ind.getId().toString());
            } else {
                s.createArtifact("Unnamed indicator(s)"); //NON-NLS
            }

            // Trying to protect against the case where we end up with tons of artifacts
            // for a single observable because the condition was not restrictive enough
            count++;
            if (count > 1000) {
                MessageNotifyUtil.Notify.show("STIXReportModule", //NON-NLS
                                              NbBundle.getMessage(this.getClass(),
                                                                  "STIXReportModule.notifyMsg.tooManyArtifactsgt1000",
                                                                  ind.getId()),
                        MessageNotifyUtil.MessageType.INFO);
                break;
            }
        }

    }

    private void writeResultsToFile(Indicator ind, String resultStr, boolean found) {
        if (output != null) {
            try {
                if (found) {
                    output.write("----------------\r\n"
                            + "Found indicator:\r\n"); //NON-NLS
                } else {
                    output.write("-----------------------\r\n"
                            + "Did not find indicator:\r\n"); //NON-NLS
                }
                if (ind.getTitle() != null) {
                    output.write("Title: " + ind.getTitle() + "\r\n"); //NON-NLS
                } else {
                    output.write("\r\n");
                }
                if (ind.getId() != null) {
                    output.write("ID: " + ind.getId() + "\r\n"); //NON-NLS
                }

                if (ind.getDescription() != null) {
                    String desc = ind.getDescription().getValue();
                    desc = desc.trim();
                    output.write("Description: " + desc + "\r\n"); //NON-NLS
                }
                output.write("\r\nObservable results:\r\n" + resultStr + "\r\n\r\n"); //NON-NLS
            } catch (IOException ex) {
                logger.log(Level.SEVERE, String.format("Error writing to STIX report file %s", reportPath), ex); //NON-NLS
            }
        }
    }

    private void printFileHeader(String a_fileName) {
        if (output != null) {
            try {
                char[] chars = new char[a_fileName.length() + 8];
                Arrays.fill(chars, '#');
                String header = new String(chars);
                output.write("\r\n" + header);
                output.write("\r\n");
                output.write("### " + a_fileName + " ###\r\n");
                output.write(header + "\r\n\r\n");
            } catch (IOException ex) {
                logger.log(Level.SEVERE, String.format("Error writing to STIX report file %s", reportPath), ex); //NON-NLS
            }

        }

    }

    private String makeMapKey(Observable obs) {
        QName idQ;
        if (obs.getId() != null) {
            idQ = obs.getId();
        } else if (obs.getIdref() != null) {
            idQ = obs.getIdref();
        } else {
            return "";
        }

        return idQ.getLocalPart();
    }

    private void saveToObjectMap(Observable obs) {

        if (obs.getObject() != null) {
            idToObjectMap.put(makeMapKey(obs), obs.getObject());
        }
    }

    private ObservableResult evaluateObservableComposition(ObservableCompositionType comp, String spacing) throws TskCoreException {
        if (comp.getOperator() == null) {
            throw new TskCoreException("No operator found in composition"); //NON-NLS
        }

        if (comp.getObservables() != null) {
            List<Observable> obsList = comp.getObservables();

            // Split based on the type of composition (AND vs OR)
            if (comp.getOperator() == OperatorTypeEnum.AND) {
                ObservableResult result = new ObservableResult(OperatorTypeEnum.AND, spacing);
                for (Observable o : obsList) {

                    ObservableResult newResult; // The combined result for the composition
                    if (o.getObservableComposition() != null) {
                        newResult = evaluateObservableComposition(o.getObservableComposition(), spacing + "  ");
                        if (result == null) {
                            result = newResult;
                        } else {
                            result.addResult(newResult, OperatorTypeEnum.AND);
                        }
                    } else {
                        newResult = evaluateSingleObservable(o, spacing + "  ");
                        if (result == null) {
                            result = newResult;
                        } else {
                            result.addResult(newResult, OperatorTypeEnum.AND);
                        }
                    }

                    if ((!skipShortCircuit) && !result.isFalse()) {
                        // For testing purposes (and maybe in general), may not want to short-circuit
                        return result;
                    }
                }
                // At this point, all comparisions should have been true (or indeterminate)
                if (result == null) {
                    // This really shouldn't happen, but if we have an empty composition,
                    // indeterminate seems like a reasonable result
                    return new ObservableResult("", "", spacing, ObservableResult.ObservableState.INDETERMINATE, null);
                }

                return result;

            } else {
                ObservableResult result = new ObservableResult(OperatorTypeEnum.OR, spacing);
                for (Observable o : obsList) {

                    ObservableResult newResult;// The combined result for the composition

                    if (o.getObservableComposition() != null) {
                        newResult = evaluateObservableComposition(o.getObservableComposition(), spacing + "  ");
                        if (result == null) {
                            result = newResult;
                        } else {
                            result.addResult(newResult, OperatorTypeEnum.OR);
                        }
                    } else {
                        newResult = evaluateSingleObservable(o, spacing + "  ");
                        if (result == null) {
                            result = newResult;
                        } else {
                            result.addResult(newResult, OperatorTypeEnum.OR);
                        }
                    }

                    if ((!skipShortCircuit) && result.isTrue()) {
                        // For testing (and maybe in general), may not want to short-circuit
                        return result;
                    }
                }
                // At this point, all comparisions were false (or indeterminate)
                if (result == null) {
                    // This really shouldn't happen, but if we have an empty composition,
                    // indeterminate seems like a reasonable result
                    return new ObservableResult("", "", spacing, ObservableResult.ObservableState.INDETERMINATE, null);
                }

                return result;
            }
        } else {
            throw new TskCoreException("No observables found in list"); //NON-NLS
        }
    }

    private ObservableResult evaluateSingleObservable(Observable obs, String spacing) throws TskCoreException {

        // If we've already calculated this one, return the saved value
        if (idToResult.containsKey(makeMapKey(obs))) {
            return idToResult.get(makeMapKey(obs));
        }

        if (obs.getIdref() == null) {

            // We should have the object data right here (as opposed to elsewhere in the STIX file). 
            // Save it to the map.
            if (obs.getId() != null) {
                saveToObjectMap(obs);
            }

            if (obs.getObject() != null) {

                ObservableResult result = evaluateObject(obs.getObject(), spacing, makeMapKey(obs));
                idToResult.put(makeMapKey(obs), result);
                return result;
            }
        }

        if (idToObjectMap.containsKey(makeMapKey(obs))) {
            ObservableResult result = evaluateObject(idToObjectMap.get(makeMapKey(obs)), spacing, makeMapKey(obs));
            idToResult.put(makeMapKey(obs), result);
            return result;
        }

        throw new TskCoreException("Error loading/finding object for observable " + obs.getIdref()); //NON-NLS
    }

    private ObservableResult evaluateObject(ObjectType obj, String spacing, String id) {

        EvaluatableObject evalObj;

        if (obj.getProperties() instanceof FileObjectType) {
            evalObj = new EvalFileObj((FileObjectType) obj.getProperties(), id, spacing);
        } else if (obj.getProperties() instanceof Address) {
            evalObj = new EvalAddressObj((Address) obj.getProperties(), id, spacing);
        } else if (obj.getProperties() instanceof URIObjectType) {
            evalObj = new EvalURIObj((URIObjectType) obj.getProperties(), id, spacing);
        } else if (obj.getProperties() instanceof EmailMessage) {
            evalObj = new EvalEmailObj((EmailMessage) obj.getProperties(), id, spacing);
        } else if (obj.getProperties() instanceof WindowsNetworkShare) {
            evalObj = new EvalNetworkShareObj((WindowsNetworkShare) obj.getProperties(), id, spacing);
        } else if (obj.getProperties() instanceof AccountObjectType) {
            evalObj = new EvalAccountObj((AccountObjectType) obj.getProperties(), id, spacing);
        } else if (obj.getProperties() instanceof SystemObjectType) {
            evalObj = new EvalSystemObj((SystemObjectType) obj.getProperties(), id, spacing);
        } else if (obj.getProperties() instanceof URLHistory) {
            evalObj = new EvalURLHistoryObj((URLHistory) obj.getProperties(), id, spacing);
        } else if (obj.getProperties() instanceof DomainName) {
            evalObj = new EvalDomainObj((DomainName) obj.getProperties(), id, spacing);
        } else if (obj.getProperties() instanceof WindowsRegistryKey) {
            evalObj = new EvalRegistryObj((WindowsRegistryKey) obj.getProperties(), id, spacing, registryFileData);
        } else {
            // Try to get the object type as a string
            String type = obj.getProperties().toString();
            type = type.substring(0, type.indexOf("@"));
            if ((type.lastIndexOf(".") + 1) < type.length()) {
                type = type.substring(type.lastIndexOf(".") + 1);
            }
            return new ObservableResult(id, type + " not supported", //NON-NLS
                    spacing, ObservableResult.ObservableState.INDETERMINATE, null);
        }

        // Evalutate the object
        return evalObj.evaluate();
    }

    @Override
    public String getName() {
        String name = NbBundle.getMessage(this.getClass(), "STIXReportModule.getName.text");
        return name;
    }

    @Override
    public String getRelativeFilePath() {
        return "stix.txt"; //NON-NLS
    }

    @Override
    public String getDescription() {
        String desc = NbBundle.getMessage(this.getClass(), "STIXReportModule.getDesc.text");
        return desc;
    }

    @Override
    public JPanel getConfigurationPanel() {
        configPanel = new STIXReportModuleConfigPanel();
        return configPanel;
    }

}