IngestJob.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 *
 * Copyright 2014-2018 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.ingest;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content;
import org.sleuthkit.datamodel.DataSource;

public final class IngestJob {

    /*
     * An ingest job can be cancelled for various reasons.
     */
    public enum CancellationReason {

        NOT_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.notCancelled.text")),
        USER_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.cancelledByUser.text")),
        INGEST_MODULES_STARTUP_FAILED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.ingestModStartFail.text")),
        OUT_OF_DISK_SPACE(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.outOfDiskSpace.text")),
        SERVICES_DOWN(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.servicesDown.text")),
        CASE_CLOSED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.caseClosed.text"));

        private final String displayName;

        private CancellationReason(String displayName) {
            this.displayName = displayName;
        }

        public String getDisplayName() {
            return displayName;
        }
    }
    
    enum Mode {
        BATCH,
        STREAMING
    }
    
    private static final Logger logger = Logger.getLogger(IngestJob.class.getName());
    private final static AtomicLong nextId = new AtomicLong(0L);
    private final long id;
    private final List<Content> dataSources = new ArrayList<>();
    private final List<AbstractFile> files = new ArrayList<>();
    private final Mode ingestMode;
    private final Map<Long, IngestJobPipeline> ingestJobPipelines;
    private final AtomicInteger incompleteJobsCount;
    private final IngestJobSettings settings;
    private volatile CancellationReason cancellationReason;

    IngestJob(Collection<Content> dataSources, IngestJobSettings settings) {
        this.id = IngestJob.nextId.getAndIncrement();
        this.settings = settings;
        this.ingestJobPipelines = new ConcurrentHashMap<>();
        this.ingestMode = Mode.BATCH;
        this.dataSources.addAll(dataSources);
        incompleteJobsCount = new AtomicInteger(dataSources.size());
        cancellationReason = CancellationReason.NOT_CANCELLED;
    }

    IngestJob(Content dataSource, List<AbstractFile> files, IngestJobSettings settings) {
        this(Arrays.asList(dataSource), settings);
        this.files.addAll(files);
    }
    
    IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) {
        this.id = IngestJob.nextId.getAndIncrement();
        this.ingestJobPipelines = new ConcurrentHashMap<>();
        this.dataSources.add(dataSource);
        this.settings = settings;
        this.ingestMode = ingestMode;
        incompleteJobsCount = new AtomicInteger(1);
        cancellationReason = CancellationReason.NOT_CANCELLED;
    }

    public long getId() {
        return this.id;
    }

    boolean hasIngestPipeline() {
        return (!settings.getEnabledIngestModuleTemplates().isEmpty());
    }
    
    void addStreamingIngestFiles(List<Long> fileObjIds) {
        if (ingestJobPipelines.isEmpty()) {
            logger.log(Level.SEVERE, "Attempted to add streaming ingest files with no IngestJobPipeline");
            return;
        }
        // Streaming ingest jobs will only have one data source
        IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next();
        streamingIngestPipeline.addStreamingIngestFiles(fileObjIds);
    }
    
    void processStreamingIngestDataSource() {
        if (ingestJobPipelines.isEmpty()) {
            logger.log(Level.SEVERE, "Attempted to start data source ingest with no IngestJobPipeline");
            return;
        }
        // Streaming ingest jobs will only have one data source
        IngestJobPipeline streamingIngestPipeline = ingestJobPipelines.values().iterator().next();
        streamingIngestPipeline.processStreamingIngestDataSource();
    }

    List<IngestModuleError> start() {
        
        /*
         * Set up the pipeline(s)
         */
        if (files.isEmpty()) {
            for (Content dataSource : dataSources) {
                IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSource, settings);
                this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline);
            }
        } else {
            IngestJobPipeline ingestJobPipeline = new IngestJobPipeline(this, dataSources.get(0), files, settings);
            this.ingestJobPipelines.put(ingestJobPipeline.getId(), ingestJobPipeline);
        }
        incompleteJobsCount.set(ingestJobPipelines.size());
        
        /*
         * Try to start each data source ingest job. Note that there is an
         * assumption here that if there is going to be a module
         * startup failure, it will be for the first ingest job pipeline.
         *
         * TODO (RC): Consider separating module start up from pipeline startup
         * so that no processing is done if this assumption is false.
         */
        List<IngestModuleError> errors = new ArrayList<>();
        for (IngestJobPipeline ingestJobPipeline : this.ingestJobPipelines.values()) {
            errors.addAll(ingestJobPipeline.start());
            if (errors.isEmpty() == false) {
                break;
            }
        }

        /*
         * Handle start up success or failure.
         */
        if (errors.isEmpty()) {
            for (IngestJobPipeline dataSourceJob : this.ingestJobPipelines.values()) {
                IngestManager.getInstance().fireDataSourceAnalysisStarted(id, dataSourceJob.getId(), dataSourceJob.getDataSource());
            }
        } else {
            cancel(CancellationReason.INGEST_MODULES_STARTUP_FAILED);
        }

        return errors;
    }
    
    Mode getIngestMode() {
        return ingestMode;
    }

    public ProgressSnapshot getSnapshot() {
        return new ProgressSnapshot(true);
    }

    public ProgressSnapshot getSnapshot(boolean getIngestTasksSnapshot) {
        return new ProgressSnapshot(getIngestTasksSnapshot);
    }

    List<Snapshot> getDataSourceIngestJobSnapshots() {
        List<Snapshot> snapshots = new ArrayList<>();
        this.ingestJobPipelines.values().stream().forEach((dataSourceJob) -> {
            snapshots.add(dataSourceJob.getSnapshot(true));
        });
        return snapshots;
    }

    @Deprecated
    public void cancel() {
        cancel(CancellationReason.USER_CANCELLED);
    }

    public void cancel(CancellationReason reason) {
        this.cancellationReason = reason;
        this.ingestJobPipelines.values().stream().forEach((job) -> {
            job.cancel(reason);
        });
    }

    public CancellationReason getCancellationReason() {
        return this.cancellationReason;
    }

    public boolean isCancelled() {
        return (CancellationReason.NOT_CANCELLED != this.cancellationReason);
    }

    void ingestJobPipelineFinished(IngestJobPipeline ingestJobPipeline) {
        IngestManager ingestManager = IngestManager.getInstance();
        if (!ingestJobPipeline.isCancelled()) {
            ingestManager.fireDataSourceAnalysisCompleted(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource());
        } else {
            IngestManager.getInstance().fireDataSourceAnalysisCancelled(id, ingestJobPipeline.getId(), ingestJobPipeline.getDataSource());
        }
        if (incompleteJobsCount.decrementAndGet() == 0) {
            ingestManager.finishIngestJob(this);
        }
    }

    public final class ProgressSnapshot {

        private final List<DataSourceProcessingSnapshot> dataSourceProcessingSnapshots;
        private DataSourceIngestModuleHandle dataSourceModule;
        private boolean fileIngestRunning;
        private Date fileIngestStartTime;
        private final boolean jobCancelled;
        private final IngestJob.CancellationReason jobCancellationReason;

        public final class DataSourceProcessingSnapshot {

            private final Snapshot snapshot;

            private DataSourceProcessingSnapshot(Snapshot snapshot) {
                this.snapshot = snapshot;
            }

            public String getDataSource() {
                return snapshot.getDataSource();
            }

            public boolean isCancelled() {
                return snapshot.isCancelled();
            }

            public CancellationReason getCancellationReason() {
                return snapshot.getCancellationReason();
            }

            public List<String> getCancelledDataSourceIngestModules() {
                return snapshot.getCancelledDataSourceIngestModules();
            }

        }

        private ProgressSnapshot(boolean getIngestTasksSnapshot) {
            dataSourceModule = null;
            fileIngestRunning = false;
            fileIngestStartTime = null;
            dataSourceProcessingSnapshots = new ArrayList<>();
            for (IngestJobPipeline pipeline : ingestJobPipelines.values()) {
                Snapshot snapshot = pipeline.getSnapshot(getIngestTasksSnapshot);
                dataSourceProcessingSnapshots.add(new DataSourceProcessingSnapshot(snapshot));
                if (null == dataSourceModule) {
                    DataSourceIngestPipeline.PipelineModule module = snapshot.getDataSourceLevelIngestModule();
                    if (null != module) {
                        dataSourceModule = new DataSourceIngestModuleHandle(ingestJobPipelines.get(snapshot.getJobId()), module);
                    }
                }
                if (snapshot.getFileIngestIsRunning()) {
                    fileIngestRunning = true;
                }
                Date childFileIngestStartTime = snapshot.getFileIngestStartTime();
                if (null != childFileIngestStartTime && (null == fileIngestStartTime || childFileIngestStartTime.before(fileIngestStartTime))) {
                    fileIngestStartTime = childFileIngestStartTime;
                }
            }
            this.jobCancelled = isCancelled();
            this.jobCancellationReason = cancellationReason;
        }

        public DataSourceIngestModuleHandle runningDataSourceIngestModule() {
            return this.dataSourceModule;
        }

        public boolean fileIngestIsRunning() {
            return this.fileIngestRunning;
        }

        public Date fileIngestStartTime() {
            return new Date(this.fileIngestStartTime.getTime());
        }

        public boolean isCancelled() {
            return this.jobCancelled;
        }

        public CancellationReason getCancellationReason() {
            return this.jobCancellationReason;
        }

        public List<DataSourceProcessingSnapshot> getDataSourceSnapshots() {
            return Collections.unmodifiableList(this.dataSourceProcessingSnapshots);
        }

    }

    public static class DataSourceIngestModuleHandle {

        private final IngestJobPipeline ingestJobPipeline;
        private final DataSourceIngestPipeline.PipelineModule module;
        private final boolean cancelled;

        private DataSourceIngestModuleHandle(IngestJobPipeline ingestJobPipeline, DataSourceIngestPipeline.PipelineModule module) {
            this.ingestJobPipeline = ingestJobPipeline;
            this.module = module;
            this.cancelled = ingestJobPipeline.currentDataSourceIngestModuleIsCancelled();
        }

        public String displayName() {
            return this.module.getDisplayName();
        }

        public Date startTime() {
            return this.module.getProcessingStartTime();
        }

        public boolean isCancelled() {
            return this.cancelled;
        }

        public void cancel() {
            if (this.ingestJobPipeline.getCurrentDataSourceIngestModule() == this.module) {
                this.ingestJobPipeline.cancelCurrentDataSourceIngestModule();
            }
        }

    }

}