MalwareScanIngestModule.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 *
 * Copyright 2023 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.basistech.df.cybertriage.autopsy.malwarescan;
 
import com.basistech.df.cybertriage.autopsy.ctapi.CTApiDAO;
import com.basistech.df.cybertriage.autopsy.ctapi.CTCloudException;
import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthTokenResponse;
import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthenticatedRequestData;
import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean;
import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HexFormat;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.logging.Level;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.ingest.FileIngestModule;
import org.sleuthkit.autopsy.ingest.IngestJobContext;
import org.sleuthkit.autopsy.ingest.IngestModule;
import org.sleuthkit.autopsy.modules.filetypeid.FileTypeDetector;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.AnalysisResult;
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.ReadContentInputStream;
import org.sleuthkit.datamodel.HashUtility;
import org.sleuthkit.datamodel.HashUtility.HashResult;
import org.sleuthkit.datamodel.HashUtility.HashType;
import org.sleuthkit.datamodel.Score;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.datamodel.TskData;

class MalwareScanIngestModule implements FileIngestModule {
 
    private static final SharedProcessing sharedProcessing = new SharedProcessing();
    private boolean uploadFiles;
    private boolean queryFiles;
    
    MalwareScanIngestModule(MalwareScanIngestSettings settings) {
        uploadFiles = settings.shouldUploadFiles();
        queryFiles = settings.shouldQueryFiles();  
    }
    
    @Override
    public void startUp(IngestJobContext context) throws IngestModuleException {
        sharedProcessing.startUp(context, uploadFiles);
    }
 
    @Override
    public ProcessResult process(AbstractFile af) {
        return sharedProcessing.process(af);
    }
 
    @Override
    public void shutDown() {
        sharedProcessing.shutDown();
    }

    private static class SharedProcessing {
 
        // batch size of 200 files max
        private static final int BATCH_SIZE = 200;
        // 1 day timeout for all API requests
        private static final long FLUSH_SECS_TIMEOUT = 24 * 60 * 60;
 
        //minimum lookups left before issuing warning
        private static final long LOW_LOOKUPS_REMAINING = 250;
 
        //minimum file uploads left before issuing warning
        private static final long LOW_UPLOADS_REMAINING = 25;
 
        // min and max upload size in bytes
        private static final long MIN_UPLOAD_SIZE = 1;
        private static final long MAX_UPLOAD_SIZE = 100_000_000; // 100MB
 
        private static final int NUM_FILE_UPLOAD_RETRIES = 7;
        private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
 
        private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
                "application/x-bat",//NON-NLS
                "application/x-dosexec",//NON-NLS
                "application/vnd.microsoft.portable-executable",//NON-NLS
                "application/x-msdownload",//NON-NLS
                "application/exe",//NON-NLS
                "application/x-exe",//NON-NLS
                "application/dos-exe",//NON-NLS
                "vms/exe",//NON-NLS
                "application/x-winexe",//NON-NLS
                "application/msdos-windows",//NON-NLS
                "application/x-msdos-program"//NON-NLS
        ).collect(Collectors.toSet());
        
        private static final String MALWARE_CONFIG = ""; // NOTE: Adding a configuration complicates NTL branch UI
 
        private static final Logger logger = Logger.getLogger(MalwareScanIngestModule.class.getName());
 
        private final BatchProcessor<FileRecord> batchProcessor = new BatchProcessor<FileRecord>(
                BATCH_SIZE,
                FLUSH_SECS_TIMEOUT,
                (lst) -> SharedProcessing.this.handleBatch(SharedProcessing.this.ingestJobState, lst));
 
        private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
        private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
 
        private IngestJobState ingestJobState = null;
 
        @Messages({
            "MalwareScanIngestModule_malwareTypeDisplayName=Malware",
            "MalwareScanIngestModule_ShareProcessing_noLicense_title=No Cyber Triage License",
            "MalwareScanIngestModule_ShareProcessing_noLicense_desc=No Cyber Triage license could be loaded.  Cyber Triage processing will be disabled.",
            "MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_title=No remaining lookups",
            "MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_desc=There are no more remaining hash lookups for this license at this time.  Malware scanning will be disabled.",
            "MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_title=Hash Lookups Low",
            "# {0} - remainingLookups",
            "MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_desc=This license only has {0} lookups remaining.",
            "MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_title=No remaining file uploads",
            "MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_desc=There are no more remaining file uploads for this license at this time.  File uploading will be disabled.",
            "MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_title=File Uploads Limit Low",
            "# {0} - remainingUploads",
            "MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_desc=This license only has {0} file uploads remaining.",
            "MalwareScanIngestModule_ShareProcessing_startup_generalException_desc=An exception occurred on MalwareScanIngestModule startup.  See the log for more information.",
            "MalwareScanIngestModule_ShareProcessing_startup_invalidLicenseWarning_title=Invalid License",
            "MalwareScanIngestModule_ShareProcessing_startup_invalidLicenseWarning_desc=The current Cyber Triage license is no longer valid.  Please remove the license from the Cyber Triage options panel."})
        synchronized void startUp(IngestJobContext context, boolean uploadFiles) throws IngestModuleException {
            // only run this code once per startup
            if (ingestJobState != null) {
                return;
            }
 
            try {
                ingestJobState = getNewJobState(context, uploadFiles);
            } catch (CTCloudException cloudEx) {
                ingestJobState = IngestJobState.DISABLED;
                logger.log(Level.WARNING, "An error occurred while starting the MalwareScanIngestModule.", cloudEx);
                throw new IngestModuleException(cloudEx.getErrorDetails(), cloudEx);
            } catch (IllegalStateException stateEx) {
                ingestJobState = IngestJobState.DISABLED;
                logger.log(Level.WARNING, "An error occurred while starting the MalwareScanIngestModule.", stateEx);
                throw new IngestModuleException(stateEx.getMessage(), stateEx);
            } catch (Exception ex) {
                ingestJobState = IngestJobState.DISABLED;
                logger.log(Level.WARNING, "An error occurred while starting the MalwareScanIngestModule.", ex);
                throw new IngestModuleException(Bundle.MalwareScanIngestModule_ShareProcessing_startup_generalException_desc(), ex);
            }
        }

        private IngestJobState getNewJobState(IngestJobContext context, boolean uploadFiles) throws Exception {
            // get saved license
            Optional<LicenseInfo> licenseInfoOpt = ctSettingsPersistence.loadLicenseInfo();
            if (licenseInfoOpt.isEmpty() || licenseInfoOpt.get().getDecryptedLicense() == null) {
                throw new IllegalStateException(Bundle.MalwareScanIngestModule_ShareProcessing_noLicense_desc());
            }
 
            AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(licenseInfoOpt.get().getDecryptedLicense());
            // syncronously fetch malware scans info
 
            // determine lookups remaining
            long lookupsRemaining = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
            if (lookupsRemaining <= 0) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_title(),
                        Bundle.MalwareScanIngestModule_ShareProcessing_noLookupsRemaining_desc(),
                        null);
 
                return IngestJobState.DISABLED;
            } else if (lookupsRemaining < LOW_LOOKUPS_REMAINING) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_title(),
                        Bundle.MalwareScanIngestModule_ShareProcessing_lowLookupsLimitWarning_desc(lookupsRemaining),
                        null);
            }
 
            // determine lookups remaining
            if (uploadFiles) {
                long uploadsRemaining = remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount());
                if (uploadsRemaining <= 0) {
                    notifyWarning(
                            Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_title(),
                            Bundle.MalwareScanIngestModule_ShareProcessing_noUploadsRemaining_desc(),
                            null);
                    uploadFiles = false;
                } else if (lookupsRemaining < LOW_UPLOADS_REMAINING) {
                    notifyWarning(
                            Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_title(),
                            Bundle.MalwareScanIngestModule_ShareProcessing_lowUploadsLimitWarning_desc(lookupsRemaining),
                            null);
                }
            }
 
            // setup necessary variables for processing
            SleuthkitCase tskCase = Case.getCurrentCaseThrows().getSleuthkitCase();
            return new IngestJobState(
                    context,
                    tskCase,
                    new PathNormalizer(tskCase),
                    new FileTypeDetector(),
                    licenseInfoOpt.get(),
                    BlackboardArtifact.Type.TSK_MALWARE,
                    uploadFiles,
                    true
            );
        }

        private static long remaining(Long limit, Long used) {
            limit = limit == null ? 0 : limit;
            used = used == null ? 0 : used;
            return limit - used;
        }

        private static String getOrCalcHash(AbstractFile af, HashType hashType) {
            switch (hashType) {
                case MD5:
                    if (StringUtils.isNotBlank(af.getMd5Hash())) {
                        return af.getMd5Hash();
                    }
                    break;
                case SHA256:
                    if (StringUtils.isNotBlank(af.getSha256Hash())) {
                        return af.getSha256Hash();
                    }
            }
 
            try {
                List<HashResult> hashResults = HashUtility.calculateHashes(af, Collections.singletonList(hashType));
                if (CollectionUtils.isNotEmpty(hashResults)) {
                    for (HashResult hashResult : hashResults) {
                        if (hashResult.getType() == hashType) {
                            return hashResult.getValue();
                        }
                    }
                }
            } catch (TskCoreException ex) {
                logger.log(Level.WARNING,
                        MessageFormat.format("An error occurred while processing hash for file name: {0} and obj id: {1} and hash type {2}.",
                                af.getName(),
                                af.getId(),
                                hashType.name()),
                        ex);
            }
 
            return null;
        }

        private static String getOrCalcMd5(AbstractFile af) {
            return getOrCalcHash(af, HashType.MD5);
        }

        private static String getOrCalcSha256(AbstractFile af) {
            return getOrCalcHash(af, HashType.SHA256);
        }

        private static String getOrCalcSha1(AbstractFile af) throws NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
            if (StringUtils.isNotBlank(af.getSha1Hash())) {
                return af.getSha1Hash();
            }
            // taken from https://stackoverflow.com/questions/6293713/java-how-to-create-sha-1-for-a-file
            MessageDigest digest = MessageDigest.getInstance("SHA-1");
            ReadContentInputStream afStream = new ReadContentInputStream(af);
            int n = 0;
            byte[] buffer = new byte[8192];
            while (n != -1) {
                n = afStream.read(buffer);
                if (n > 0) {
                    digest.update(buffer, 0, n);
                }
            }
            byte[] hashBytes = digest.digest();
            String hashString = HexFormat.of().formatHex(hashBytes);
            return hashString;
        }

        @Messages({
            "MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout",
            "MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out"
        })
        IngestModule.ProcessResult process(AbstractFile af) {
            try {
                if (ingestJobState != null
                        && ingestJobState.isDoFileLookups()
                        && !ingestJobState.getIngestJobContext().fileIngestIsCancelled()
                        && af.getKnown() != TskData.FileKnown.KNOWN
                        && EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(ingestJobState.getFileTypeDetector().getMIMEType(af)).trim().toLowerCase())
                        && CollectionUtils.isEmpty(af.getAnalysisResults(ingestJobState.getMalwareType()))) {
 
                    String md5 = getOrCalcMd5(af);
                    if (StringUtils.isNotBlank(md5)) {
                        batchProcessor.add(new FileRecord(af.getId(), md5));
                    }
                }
                return ProcessResult.OK;
            } catch (TskCoreException ex) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
                        Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
                        ex);
                return IngestModule.ProcessResult.ERROR;
            } catch (InterruptedException ex) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_ShareProcessing_batchTimeout_title(),
                        Bundle.MalwareScanIngestModule_ShareProcessing_batchTimeout_desc(),
                        ex);
                return IngestModule.ProcessResult.ERROR;
            }
        }

        @Messages({
            "MalwareScanIngestModule_SharedProcessing_authTokenResponseError_title=Authentication API error",
            "# {0} - errorResponse",
            "MalwareScanIngestModule_SharedProcessing_authTokenResponseError_desc=Received error: ''{0}'' when fetching the API authentication token for the license",
            "MalwareScanIngestModule_SharedProcessing_repServicenResponseError_title=Lookup API error",
            "# {0} - errorResponse",
            "MalwareScanIngestModule_SharedProcessing_repServicenResponseError_desc=Received error: ''{0}'' when fetching hash lookup results",
            "MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title=Hash Lookups Exhausted",
            "MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc=The remaining hash lookups for this license have been exhausted",
            "MalwareScanIngestModule_SharedProcessing_generalProcessingError_title=Hash Lookup Error",
            "MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc=An error occurred while processing hash lookup results",})
        private void handleBatch(IngestJobState ingestJobState, List<FileRecord> fileRecords) {
            if (ingestJobState == null
                    || !ingestJobState.isDoFileLookups()
                    || ingestJobState.getIngestJobContext().fileIngestIsCancelled()
                    || fileRecords == null
                    || fileRecords.isEmpty()) {
                return;
            }
 
            // create mapping of md5 to corresponding object ids as well as just the list of md5's
            Map<String, List<Long>> md5ToObjId = new HashMap<>();
 
            for (FileRecord fr : fileRecords) {
                if (fr == null || StringUtils.isBlank(fr.getMd5hash()) || fr.getObjId() <= 0) {
                    continue;
                }
 
                String sanitizedMd5 = normalizedMd5(fr.getMd5hash());
                md5ToObjId
                        .computeIfAbsent(sanitizedMd5, (k) -> new ArrayList<>())
                        .add(fr.getObjId());
            }
 
            List<String> md5Hashes = new ArrayList<>(md5ToObjId.keySet());
 
            if (md5Hashes.isEmpty()) {
                return;
            }
 
            try {
                List<CTCloudBean> repResult = getHashLookupResults(ingestJobState, md5Hashes);
                handleLookupResults(ingestJobState, md5ToObjId, repResult);
            } catch (Exception ex) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
                        Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
                        ex);
            }
        }

        @Messages({
            "MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_title=Lookup Limits Exceeded",
            "MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_desc=Not all files were processed because hash lookup limits were exceeded. Please try again when your limits reset.",})
        private void handleLookupResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> repResult) throws Blackboard.BlackboardException, TskCoreException, TskCoreException, CTCloudException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
            if (CollectionUtils.isEmpty(repResult)) {
                return;
            }
 
            Map<Status, List<CTCloudBean>> statusGroupings = repResult.stream()
                    .filter(bean -> bean.getMalwareResult() != null)
                    .collect(Collectors.groupingBy(bean -> bean.getMalwareResult().getStatus()));
 
            // for all found items, create analysis results
            List<CTCloudBean> found = statusGroupings.get(Status.FOUND);
            createAnalysisResults(ingestJobState, found, md5ToObjId);
 
            // if being scanned, check list to run later
            handleNonFoundResults(ingestJobState, md5ToObjId, statusGroupings.get(Status.BEING_SCANNED), false);
 
            // if not found, try upload
            handleNonFoundResults(ingestJobState, md5ToObjId, statusGroupings.get(Status.NOT_FOUND), true);
 
            // indicate a general error if some result in an error
            if (CollectionUtils.isNotEmpty(statusGroupings.get(Status.ERROR))) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
                        Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
                        null);
            }
 
            // indicate some results were not processed if limits exceeded in results
            if (CollectionUtils.isNotEmpty(statusGroupings.get(Status.LIMITS_EXCEEDED))) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_title(),
                        Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedResultsHashLookups_desc(),
                        null);
            }
        }

        private void handleNonFoundResults(IngestJobState ingestJobState, Map<String, List<Long>> md5ToObjId, List<CTCloudBean> results, boolean performFileUpload) throws CTCloudException, TskCoreException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
            if (CollectionUtils.isNotEmpty(results)
                    && ingestJobState.isDoFileLookups()
                    && ((performFileUpload && ingestJobState.isUploadUnknownFiles()) || (!performFileUpload && ingestJobState.isQueryForMissing()))) {
 
                for (CTCloudBean beingScanned : CollectionUtils.emptyIfNull(results)) {
 
                    String sanitizedMd5 = normalizedMd5(beingScanned.getMd5HashValue());
                    if (StringUtils.isBlank(sanitizedMd5)) {
                        continue;
                    }
                    List<Long> correspondingObjIds = md5ToObjId.get(sanitizedMd5);
                    if (CollectionUtils.isEmpty(correspondingObjIds)) {
                        continue;
                    }
 
                    if (performFileUpload) {
                        uploadFile(ingestJobState, sanitizedMd5, correspondingObjIds.get(0));
                    }
 
                    ingestJobState.getUnidentifiedHashes().put(sanitizedMd5, correspondingObjIds);
                }
            }
        }

        private List<CTCloudBean> getHashLookupResults(IngestJobState ingestJobState, List<String> md5Hashes) throws CTCloudException {
            if (ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
                return Collections.emptyList();
            }
 
            // get an auth token with the license
            AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense());
 
            // make sure we are in bounds for the remaining scans
            long remainingScans = remaining(authTokenResponse.getHashLookupLimit(), authTokenResponse.getHashLookupCount());
            if (remainingScans <= 0) {
                ingestJobState.disableDoFileLookups();
                notifyWarning(
                        Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
                        Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
                        null);
                return Collections.emptyList();
            } else if (ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
                return Collections.emptyList();
            }
 
            // while we have a valid auth token, also check file uploads.
            if (ingestJobState.isUploadUnknownFiles()) {
                long remainingUploads = remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount());
                if (remainingUploads <= 0) {
                    ingestJobState.disableUploadUnknownFiles();
                    notifyWarning(
                            Bundle.MalwareScanIngestModule_uploadFile_noRemainingFileUploads_title(),
                            Bundle.MalwareScanIngestModule_uploadFile_noRemainingFileUploads_desc(),
                            null);
                }
            }
 
            // using auth token, get results
            return ctApiDAO.getReputationResults(
                    new AuthenticatedRequestData(ingestJobState.getLicenseInfo().getDecryptedLicense(), authTokenResponse),
                    md5Hashes
            );
        }

        private static String normalizedMd5(String orig) {
            return StringUtils.defaultString(orig).trim().toLowerCase();
        }

        private static boolean isUploadable(AbstractFile af) {
            long size = af.getSize();
            return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
        }

        @Messages({
            "MalwareScanIngestModule_uploadFile_notUploadable_title=Not Able to Upload",
            "# {0} - objectId",
            "MalwareScanIngestModule_uploadFile_notUploadable_desc=A file did not meet requirements for upload (object id: {0}).",
            "MalwareScanIngestModule_uploadFile_noRemainingFileUploads_title=No Remaining File Uploads",
            "MalwareScanIngestModule_uploadFile_noRemainingFileUploads_desc=There are no more file uploads on this license at this time.  File uploads will be disabled for remaining uploads.",})
        private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId) throws CTCloudException, TskCoreException, NoSuchAlgorithmException, ReadContentInputStream.ReadContentInputStreamException {
            if (!ingestJobState.isUploadUnknownFiles() || ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
                return false;
            }
 
            AbstractFile af = ingestJobState.getTskCase().getAbstractFileById(objId);
            if (af == null) {
                return false;
            }
 
            if (!isUploadable(af)) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_uploadFile_notUploadable_title(),
                        Bundle.MalwareScanIngestModule_uploadFile_notUploadable_desc(objId),
                        null);
                return false;
            }
 
            // get auth token / file upload url
            AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), af.getSize());
            if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
                throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
            } else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
                // don't proceed with upload if reached limit
                ingestJobState.disableUploadUnknownFiles();
                notifyWarning(
                        Bundle.MalwareScanIngestModule_uploadFile_noRemainingFileUploads_title(),
                        Bundle.MalwareScanIngestModule_uploadFile_noRemainingFileUploads_desc(),
                        null);
 
                return false;
            } else if (ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
                return false;
            }
 
            // upload bytes
            ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
 
            ctApiDAO.uploadFile(new FileUploadRequest()
                    .setContentLength(af.getSize())
                    .setFileInputStream(fileInputStream)
                    .setFileName(af.getName())
                    .setFullUrlPath(authTokenResponse.getFileUploadUrl())
            );
 
            // upload metadata
            MetadataUploadRequest metaRequest = new MetadataUploadRequest()
                    .setCreatedDate(af.getCrtime() == 0 ? null : af.getCrtime())
                    .setFilePath(ingestJobState.getPathNormalizer().normalizePath(af.getUniquePath()))
                    .setFileSizeBytes(af.getSize())
                    .setFileUploadUrl(authTokenResponse.getFileUploadUrl())
                    .setMd5(md5)
                    .setSha1(getOrCalcSha1(af))
                    .setSha256(getOrCalcSha256(af));
 
            ctApiDAO.uploadMeta(new AuthenticatedRequestData(ingestJobState.getLicenseInfo().getDecryptedLicense(), authTokenResponse), metaRequest);
            return true;
        }

        @Messages({
            "MalwareScanIngestModule_longPollForNotFound_fileLookupPolling_title=Waiting for File Upload Results",
            "MalwareScanIngestModule_longPollForNotFound_fileLookupPolling_desc=Waiting for all uploaded files to complete scanning.",
            "MalwareScanIngestModule_longPollForNotFound_timeout_title=File Upload Results Timeout",
            "MalwareScanIngestModule_longPollForNotFound_timeout_desc=There was a timeout while waiting for file uploads to be processed.  Please try again later.",})
        private void longPollForNotFound(IngestJobState ingestJobState) throws InterruptedException, CTCloudException, Blackboard.BlackboardException, TskCoreException {
            if (!ingestJobState.isDoFileLookups()
                    || !ingestJobState.isQueryForMissing()
                    || MapUtils.isEmpty(ingestJobState.getUnidentifiedHashes())
                    || ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
                return;
            }
 
            MessageNotifyUtil.Notify.info(
                    Bundle.MalwareScanIngestModule_longPollForNotFound_fileLookupPolling_title(),
                    Bundle.MalwareScanIngestModule_longPollForNotFound_fileLookupPolling_desc()
            );
            logger.log(Level.INFO, "Begin polling for malware status of file uploads.");
 
            Map<String, List<Long>> remaining = new HashMap<>(ingestJobState.getUnidentifiedHashes());
 
            for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) {
                List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE);
                for (List<String> batch : md5Batches) {
                    // if we have exceeded limits or cancelled, then we're done.
                    if (!ingestJobState.isDoFileLookups() || ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
                        return;
                    }
 
                    List<CTCloudBean> repResult = getHashLookupResults(ingestJobState, batch);
 
                    Map<Status, List<CTCloudBean>> statusGroupings = repResult.stream()
                            .filter(bean -> bean.getMalwareResult() != null)
                            .collect(Collectors.groupingBy(bean -> bean.getMalwareResult().getStatus()));
 
                    // for all found items, create analysis results
                    List<CTCloudBean> found = statusGroupings.get(Status.FOUND);
 
                    createAnalysisResults(ingestJobState, found, remaining);
 
                    // remove any found items from the list of items to long poll for
                    for (CTCloudBean foundItem : CollectionUtils.emptyIfNull(found)) {
                        String normalizedMd5 = normalizedMd5(foundItem.getMd5HashValue());
                        remaining.remove(normalizedMd5);
                    }
                }
 
                if (remaining.isEmpty()) {
                    return;
                }
 
                // exponential backoff before trying again
                long waitMultiplier = ((long) Math.pow(2, retry));
                
                logger.log(Level.INFO, MessageFormat.format("Waiting {0} milliseconds before polling again for malware status of file uploads.", (waitMultiplier * FILE_UPLOAD_RETRY_SLEEP_MILLIS)));
                
                for (int i = 0; i < waitMultiplier; i++) {
                    if (!ingestJobState.isDoFileLookups() || ingestJobState.getIngestJobContext().fileIngestIsCancelled()) {
                        return;
                    }
 
                    Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
                }
            }
 
            notifyWarning(
                    Bundle.MalwareScanIngestModule_longPollForNotFound_timeout_title(),
                    Bundle.MalwareScanIngestModule_longPollForNotFound_timeout_desc(),
                    null
            );
        }

        private void createAnalysisResults(IngestJobState ingestJobState, List<CTCloudBean> repResult, Map<String, List<Long>> md5ToObjId) throws Blackboard.BlackboardException, TskCoreException {
            if (CollectionUtils.isEmpty(repResult)) {
                return;
            }
 
            List<BlackboardArtifact> createdArtifacts = new ArrayList<>();
            SleuthkitCase.CaseDbTransaction trans = null;
            try {
                trans = ingestJobState.getTskCase().beginTransaction();
                for (CTCloudBean result : repResult) {
                    String sanitizedMd5 = normalizedMd5(result.getMd5HashValue());
                    List<Long> objIds = md5ToObjId.remove(sanitizedMd5);
                    if (CollectionUtils.isEmpty(objIds)) {
                        continue;
                    }
 
                    for (Long objId : objIds) {
                        AnalysisResult res = createAnalysisResult(ingestJobState, trans, result, objId);
                        if (res != null) {
                            // only post results that have score NOTABLE or LIKELY_NOTABLE
                            Score score = res.getScore();
                            if (score.getSignificance() == Score.Significance.NOTABLE || score.getSignificance() == Score.Significance.LIKELY_NOTABLE) {
                                createdArtifacts.add(res);
                            }
                        }
                    }
                }
 
                trans.commit();
                trans = null;
            } finally {
                if (trans != null) {
                    trans.rollback();
                    createdArtifacts.clear();
                    trans = null;
                }
            }
 
            if (!CollectionUtils.isEmpty(createdArtifacts)) {
                ingestJobState.getTskCase().getBlackboard().postArtifacts(
                        createdArtifacts,
                        Bundle.MalwareScanIngestModuleFactory_displayName(),
                        ingestJobState.getIngestJobId()
                );
            }
 
        }

        @Messages({
            "MalwareScanIngestModule_SharedProcessing_createAnalysisResult_Yes=YES",
            "MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No=NO"
        })
        private AnalysisResult createAnalysisResult(IngestJobState ingestJobState, SleuthkitCase.CaseDbTransaction trans, CTCloudBean cloudBean, Long objId) throws Blackboard.BlackboardException {
            if (objId == null || cloudBean == null || cloudBean.getMalwareResult() == null || cloudBean.getMalwareResult().getStatus() != Status.FOUND) {
                logger.log(Level.WARNING, MessageFormat.format("Attempting to create analysis result with invalid parameters [objId: {0}, cloud bean status: {1}]",
                        objId == null
                                ? "<null>"
                                : objId,
                        (cloudBean == null || cloudBean.getMalwareResult() == null || cloudBean.getMalwareResult().getStatus() == null)
                        ? "<null>"
                        : cloudBean.getMalwareResult().getStatus().name()
                ));
                return null;
            }
 
            Score score = cloudBean.getMalwareResult().getCTScore() == null
                    ? Score.SCORE_UNKNOWN
                    : cloudBean.getMalwareResult().getCTScore().getTskCore();
 
            String conclusion = score.getSignificance() == Score.Significance.NOTABLE || score.getSignificance() == Score.Significance.LIKELY_NOTABLE
                    ? Bundle.MalwareScanIngestModule_SharedProcessing_createAnalysisResult_Yes()
                    : Bundle.MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No();
 
            String justification = cloudBean.getMalwareResult().getStatusDescription();
 
            return ingestJobState.getTskCase().getBlackboard().newAnalysisResult(
                    ingestJobState.getMalwareType(),
                    objId,
                    ingestJobState.getDsId(),
                    score,
                    conclusion,
                    MALWARE_CONFIG,
                    justification,
                    Collections.emptyList(),
                    trans).getAnalysisResult();
        }

        @Messages({
            "MalwareScanIngestModule_SharedProcessing_flushTimeout_title=Processing Timeout",
            "MalwareScanIngestModule_SharedProcessing_flushTimeout_desc=A timeout occurred while finishing processing"
        })
        synchronized void shutDown() {
            // if already shut down, return
            if (ingestJobState == null) {
                return;
            }
 
            // flush any remaining items
            try {
                batchProcessor.flushAndReset();
                longPollForNotFound(ingestJobState);
            } catch (InterruptedException ex) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_SharedProcessing_flushTimeout_title(),
                        Bundle.MalwareScanIngestModule_SharedProcessing_flushTimeout_desc(),
                        ex);
            } catch (Exception ex) {
                notifyWarning(
                        Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
                        Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
                        ex);
            } finally {
                // set state to shut down and clear any remaining
                ingestJobState = null;
            }
        }

        private static void notifyWarning(String title, String message, Exception ex) {
            MessageNotifyUtil.Notify.warn(title, message);
            logger.log(Level.WARNING, message, ex);
        }
 
        class FileRecord {
 
            private final long objId;
            private final String md5hash;
 
            FileRecord(long objId, String md5hash) {
                this.objId = objId;
                this.md5hash = md5hash;
            }
 
            long getObjId() {
                return objId;
            }
 
            String getMd5hash() {
                return md5hash;
            }
 
        }

        static class IngestJobState {
 
            static final IngestJobState DISABLED = new IngestJobState(
                    null,
                    null,
                    null,
                    null,
                    null,
                    null,
                    false,
                    false
            );
 
            private final SleuthkitCase tskCase;
            private final FileTypeDetector fileTypeDetector;
            private final LicenseInfo licenseInfo;
            private final BlackboardArtifact.Type malwareType;
            private final long dsId;
            private final long ingestJobId;
            private final boolean queryForMissing;
            private final Map<String, List<Long>> unidentifiedHashes = new HashMap<>();
 
            // this can change mid run
            private boolean uploadUnknownFiles;
            private boolean doFileLookups;
            private final IngestJobContext ingestJobContext;
            private final PathNormalizer pathNormalizer;
 
            IngestJobState(IngestJobContext ingestJobContext, SleuthkitCase tskCase, PathNormalizer pathNormalizer, FileTypeDetector fileTypeDetector, LicenseInfo licenseInfo, BlackboardArtifact.Type malwareType, boolean uploadUnknownFiles, boolean doFileLookups) {
                this.tskCase = tskCase;
                this.fileTypeDetector = fileTypeDetector;
                this.pathNormalizer = pathNormalizer;
                this.licenseInfo = licenseInfo;
                this.malwareType = malwareType;
                this.dsId = ingestJobContext == null ? 0L : ingestJobContext.getDataSource().getId();
                this.ingestJobId = ingestJobContext == null ? 0L : ingestJobContext.getJobId();
                this.ingestJobContext = ingestJobContext;
                // for now, querying for any missing files will be tied to whether initially we should upload files and do lookups at all
                this.queryForMissing = uploadUnknownFiles && doFileLookups;
                this.uploadUnknownFiles = uploadUnknownFiles;
                this.doFileLookups = doFileLookups;
            }
 
            SleuthkitCase getTskCase() {
                return tskCase;
            }
 
            IngestJobContext getIngestJobContext() {
                return ingestJobContext;
            }
 
            FileTypeDetector getFileTypeDetector() {
                return fileTypeDetector;
            }
 
            LicenseInfo getLicenseInfo() {
                return licenseInfo;
            }
 
            BlackboardArtifact.Type getMalwareType() {
                return malwareType;
            }
 
            long getDsId() {
                return dsId;
            }
 
            long getIngestJobId() {
                return ingestJobId;
            }
 
            Map<String, List<Long>> getUnidentifiedHashes() {
                return unidentifiedHashes;
            }
 
            boolean isQueryForMissing() {
                return queryForMissing;
            }
 
            boolean isUploadUnknownFiles() {
                return uploadUnknownFiles;
            }
 
            void disableUploadUnknownFiles() {
                this.uploadUnknownFiles = false;
            }
 
            boolean isDoFileLookups() {
                return doFileLookups;
            }
 
            void disableDoFileLookups() {
                this.doFileLookups = false;
            }
 
            public PathNormalizer getPathNormalizer() {
                return pathNormalizer;
            }
 
        }
    }
}