IngestJob.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 *
 * Copyright 2014-2021 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.ingest;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import org.openide.util.NbBundle;
import org.python.google.common.collect.ImmutableList;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.AnalysisResult;
import org.sleuthkit.datamodel.DataArtifact;
import org.sleuthkit.datamodel.DataSource;

public final class IngestJob {

    public enum CancellationReason {

        NOT_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.notCancelled.text")),
        USER_CANCELLED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.cancelledByUser.text")),
        INGEST_MODULES_STARTUP_FAILED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.ingestModStartFail.text")),
        OUT_OF_DISK_SPACE(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.outOfDiskSpace.text")),
        SERVICES_DOWN(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.servicesDown.text")),
        CASE_CLOSED(NbBundle.getMessage(IngestJob.class, "IngestJob.cancelReason.caseClosed.text"));

        private final String displayName;

        private CancellationReason(String displayName) {
            this.displayName = displayName;
        }

        public String getDisplayName() {
            return displayName;
        }
    }

    enum Mode {
        BATCH,
        STREAMING
    }

    private static final Logger logger = Logger.getLogger(IngestJob.class.getName());
    private final static AtomicLong nextId = new AtomicLong(0L);
    private final long id;
    private final DataSource dataSource;
    private final List<AbstractFile> files = new ArrayList<>();
    private final Mode ingestMode;
    private final IngestJobSettings settings;
    private volatile IngestJobExecutor ingestModuleExecutor;
    private volatile CancellationReason cancellationReason;

    IngestJob(DataSource dataSource, List<AbstractFile> files, IngestJobSettings settings) {
        this(dataSource, Mode.BATCH, settings);
        this.files.addAll(files);
    }

    IngestJob(DataSource dataSource, Mode ingestMode, IngestJobSettings settings) {
        id = IngestJob.nextId.getAndIncrement();
        this.dataSource = dataSource;
        this.settings = settings;
        this.ingestMode = ingestMode;
        cancellationReason = CancellationReason.NOT_CANCELLED;
    }

    public long getId() {
        return this.id;
    }

    DataSource getDataSource() {
        return dataSource;
    }

    List<AbstractFile> getFiles() {
        return ImmutableList.copyOf(files);
    }

    IngestJobSettings getSettings() {
        return settings;
    }

    boolean hasIngestPipeline() {
        return (!settings.getEnabledIngestModuleTemplates().isEmpty());
    }

    void addStreamedFiles(List<Long> fileObjIds) {
        if (ingestMode == Mode.STREAMING) {
            if (ingestModuleExecutor != null) {
                ingestModuleExecutor.addStreamedFiles(fileObjIds);
            } else {
                logger.log(Level.SEVERE, "Attempted to add streamed files with no ingest pipeline");
            }
        } else {
            logger.log(Level.SEVERE, "Attempted to add streamed files to batch ingest job");
        }
    }

    void addDataArtifacts(List<DataArtifact> dataArtifacts) {
        ingestModuleExecutor.addDataArtifacts(dataArtifacts);
    }

    void addAnalysisResults(List<AnalysisResult> results) {
        ingestModuleExecutor.addAnalysisResults(results);
    }

    void addStreamedDataSource() {
        if (ingestMode == Mode.STREAMING) {
            if (ingestModuleExecutor != null) {
                ingestModuleExecutor.addStreamedDataSource();
            } else {
                logger.log(Level.SEVERE, "Attempted to start data source analaysis with no ingest pipeline");
            }
        } else {
            logger.log(Level.SEVERE, "Attempted to add streamed ingest files to batch ingest job");
        }
    }

    synchronized List<IngestModuleError> start() throws InterruptedException {
        if (ingestModuleExecutor != null) {
            logger.log(Level.SEVERE, "Attempt to start ingest job that has already been started");
            return Collections.emptyList();
        }

        ingestModuleExecutor = new IngestJobExecutor(this);
        List<IngestModuleError> errors = new ArrayList<>();
        errors.addAll(ingestModuleExecutor.startUp());
        if (errors.isEmpty()) {
            IngestManager.getInstance().fireDataSourceAnalysisStarted(id, ingestModuleExecutor.getDataSource());
        } else {
            cancel(CancellationReason.INGEST_MODULES_STARTUP_FAILED);
        }
        return errors;
    }

    Mode getIngestMode() {
        return ingestMode;
    }

    public ProgressSnapshot getSnapshot() {
        return getSnapshot(true);
    }

    public ProgressSnapshot getSnapshot(boolean includeIngestTasksSnapshot) {
        ProgressSnapshot snapshot = null;
        if (ingestModuleExecutor != null) {
            return new ProgressSnapshot(includeIngestTasksSnapshot);
        }
        return snapshot;
    }

    IngestJobProgressSnapshot getDiagnosticStatsSnapshot() {
        IngestJobProgressSnapshot snapshot = null;
        if (ingestModuleExecutor != null) {
            snapshot = ingestModuleExecutor.getIngestJobProgressSnapshot(true);
        }
        return snapshot;
    }

    @Deprecated
    public void cancel() {
        cancel(CancellationReason.USER_CANCELLED);
    }

    public void cancel(CancellationReason reason) {
        cancellationReason = reason;
        /*
         * Cancels the running of the ingest module pipelines. This is done in a
         * separate thread to avoid a potential deadlock. The deadlock is
         * possible because this method can be called in a thread that acquires
         * the ingest manager's ingest jobs list lock and then tries to acquire
         * the ingest pipeline stage transition lock, while an ingest thread
         * that has acquired the stage transition lock is trying to acquire the
         * ingest manager's ingest jobs list lock.
         */
        new Thread(() -> {
            if (ingestModuleExecutor != null) {
                ingestModuleExecutor.cancel(reason);
            }
        }).start();
    }

    public CancellationReason getCancellationReason() {
        return cancellationReason;
    }

    public boolean isCancelled() {
        return (CancellationReason.NOT_CANCELLED != cancellationReason);
    }

    void notifyIngestPipelinesShutDown() {
        IngestManager ingestManager = IngestManager.getInstance();
        if (!ingestModuleExecutor.isCancelled()) {
            ingestManager.fireDataSourceAnalysisCompleted(id, dataSource);
        } else {
            IngestManager.getInstance().fireDataSourceAnalysisCancelled(id, dataSource);
        }
        ingestManager.finishIngestJob(this);
    }

    public final class ProgressSnapshot {

        private final DataSourceProcessingSnapshot dataSourceProcessingSnapshot;
        private final boolean jobCancellationRequested;
        private final CancellationReason jobCancellationReason;

        public final class DataSourceProcessingSnapshot {

            private final IngestJobProgressSnapshot snapshot;

            private DataSourceProcessingSnapshot(IngestJobProgressSnapshot snapshot) {
                this.snapshot = snapshot;
            }

            public String getDataSource() {
                return snapshot.getDataSource();
            }

            public boolean fileIngestIsRunning() {
                return snapshot.getFileIngestIsRunning();
            }

            public Date fileIngestStartTime() {
                return new Date(snapshot.getFileIngestStartTime().getTime());
            }

            DataSourceIngestPipeline.DataSourcePipelineModule getDataSourceLevelIngestModule() {
                return snapshot.getDataSourceLevelIngestModule();
            }

            public boolean isCancelled() {
                return snapshot.isCancelled();
            }

            public CancellationReason getCancellationReason() {
                return snapshot.getCancellationReason();
            }

            public List<String> getCancelledDataSourceIngestModules() {
                return snapshot.getCancelledDataSourceIngestModules();
            }

        }

        private ProgressSnapshot(boolean includeIngestTasksSnapshot) {
            IngestJobProgressSnapshot snapshot = ingestModuleExecutor.getIngestJobProgressSnapshot(includeIngestTasksSnapshot);
            dataSourceProcessingSnapshot = new DataSourceProcessingSnapshot(snapshot);
            jobCancellationRequested = IngestJob.this.isCancelled();
            jobCancellationReason = IngestJob.this.getCancellationReason();
        }

        public DataSourceIngestModuleHandle runningDataSourceIngestModule() {
            DataSourceIngestModuleHandle moduleHandle = null;
            DataSourceIngestPipeline.DataSourcePipelineModule module = dataSourceProcessingSnapshot.getDataSourceLevelIngestModule();
            if (module != null) {
                moduleHandle = new DataSourceIngestModuleHandle(ingestModuleExecutor, module);
            }
            return moduleHandle;
        }

        public boolean fileIngestIsRunning() {
            return dataSourceProcessingSnapshot.fileIngestIsRunning();
        }

        public Date fileIngestStartTime() {
            return new Date(dataSourceProcessingSnapshot.fileIngestStartTime().getTime());
        }

        public boolean isCancelled() {
            return jobCancellationRequested;
        }

        public CancellationReason getCancellationReason() {
            return jobCancellationReason;
        }

        public DataSourceProcessingSnapshot getDataSourceProcessingSnapshot() {
            return dataSourceProcessingSnapshot;
        }

    }

    public static class DataSourceIngestModuleHandle {

        private final IngestJobExecutor ingestJobExecutor;
        private final DataSourceIngestPipeline.DataSourcePipelineModule module;
        private final boolean cancelled;

        private DataSourceIngestModuleHandle(IngestJobExecutor ingestJobExecutor, DataSourceIngestPipeline.DataSourcePipelineModule module) {
            this.ingestJobExecutor = ingestJobExecutor;
            this.module = module;
            this.cancelled = ingestJobExecutor.currentDataSourceIngestModuleIsCancelled();
        }

        public String displayName() {
            return module.getDisplayName();
        }

        public Date startTime() {
            return module.getProcessingStartTime();
        }

        public boolean isCancelled() {
            return cancelled;
        }

        public void cancel() {
            /*
             * Note that this cancellation mechanism has a race condition. This
             * could perhaps be solved by adding a cancel() API to the
             * IngestModule interface.
             */
            if (ingestJobExecutor.getCurrentDataSourceIngestModule() == module) {
                ingestJobExecutor.cancelCurrentDataSourceIngestModule();
            }
        }

    }

}