AccountsText.java

Go to the documentation of this file.

/*
 * Autopsy Forensic Browser
 *
 * Copyright 2011-2016 Basis Technology Corp.
 * Contact: carrier <at> sleuthkit <dot> org
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sleuthkit.autopsy.keywordsearch;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest.METHOD;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.coreutils.Logger;
import org.sleuthkit.autopsy.coreutils.Version;

class AccountsText implements IndexedText {

    private static final Logger LOGGER = Logger.getLogger(AccountsText.class.getName());
    private static final boolean DEBUG = (Version.getBuildType() == Version.Type.DEVELOPMENT);

    private static final String HIGHLIGHT_PRE = "<span style='background:yellow'>"; //NON-NLS
    private static final String HIGHLIGHT_POST = "</span>"; //NON-NLS
    private static final String ANCHOR_NAME_PREFIX = AccountsText.class.getName() + "_";

    private static final String INSERT_PREFIX = "<a name='" + ANCHOR_NAME_PREFIX; //NON-NLS
    private static final String INSERT_POSTFIX = "'></a>$0"; //$0 will insert current regex match  //NON-NLS
    private static final Pattern ANCHOR_DETECTION_PATTERN = Pattern.compile(HIGHLIGHT_PRE);

    private static final String HIGHLIGHT_FIELD = LuceneQuery.HIGHLIGHT_FIELD_REGEX;

    private final Server solrServer;
    private final String solrDocumentId;
    private final long solrObjectId;
    private final Integer chunkId;
    private final Set<String> keywords = new HashSet<>();
    private final String displayName;
    private final String queryString;

    private boolean isPageInfoLoaded = false;
    private int numberPagesForFile = 0;
    private int currentPage = 0;
    //list of pages, used for iterating back and forth.  Only stores pages with hits
    private final List<Integer> pages = new ArrayList<>();
    //map from page/chunk to number of hits. value is 0 if not yet known.
    private final LinkedHashMap<Integer, Integer> numberOfHitsPerPage = new LinkedHashMap<>();
    //map from page/chunk number to current hit on that page.
    private final HashMap<Integer, Integer> currentHitPerPage = new HashMap<>();

    @NbBundle.Messages({
        "AccountsText.creditCardNumber=Credit Card Number",
        "AccountsText.creditCardNumbers=Credit Card Numbers"})
    AccountsText(String objectId, Set<String> keywords) {
        this.solrDocumentId = objectId;
        this.keywords.addAll(keywords);

        //build the query string
        this.queryString = HIGHLIGHT_FIELD + ":"
                + keywords.stream()
                .map(keyword -> "/.*?" + KeywordSearchUtil.escapeLuceneQuery(keyword) + ".*?/")//surround each "keyword" with match anything regex.
                .collect(Collectors.joining(" ")); //collect as space separated string

        this.solrServer = KeywordSearch.getServer();

        final int separatorIndex = solrDocumentId.indexOf(Server.CHUNK_ID_SEPARATOR);
        if (-1 == separatorIndex) {
            //no chunk id in solrDocumentId
            this.solrObjectId = Long.parseLong(solrDocumentId);
            this.chunkId = null;
        } else {
            //solrDocumentId includes chunk id
            this.solrObjectId = Long.parseLong(solrDocumentId.substring(0, separatorIndex));
            this.chunkId = Integer.parseInt(solrDocumentId.substring(separatorIndex + 1));
        }

        displayName = keywords.size() == 1
                ? Bundle.AccountsText_creditCardNumber()
                : Bundle.AccountsText_creditCardNumbers();
    }

    long getObjectId() {
        return this.solrObjectId;
    }

    @Override
    public int getNumberPages() {
        return this.numberPagesForFile;
    }

    @Override
    public int getCurrentPage() {
        return this.currentPage;
    }

    @Override
    public boolean hasNextPage() {
        return pages.indexOf(this.currentPage) < pages.size() - 1;

    }

    @Override
    public boolean hasPreviousPage() {
        return pages.indexOf(this.currentPage) > 0;

    }

    @Override
    @NbBundle.Messages("AccountsText.nextPage.exception.msg=No next page.")
    public int nextPage() {
        if (hasNextPage()) {
            currentPage = pages.get(pages.indexOf(this.currentPage) + 1);
            return currentPage;
        } else {
            throw new IllegalStateException(Bundle.AccountsText_nextPage_exception_msg());
        }
    }

    @Override
    @NbBundle.Messages("AccountsText.previousPage.exception.msg=No previous page.")
    public int previousPage() {
        if (hasPreviousPage()) {
            currentPage = pages.get(pages.indexOf(this.currentPage) - 1);
            return currentPage;
        } else {
            throw new IllegalStateException(Bundle.AccountsText_previousPage_exception_msg());
        }
    }

    @Override
    public boolean hasNextItem() {
        if (this.currentHitPerPage.containsKey(currentPage)) {
            return this.currentHitPerPage.get(currentPage) < this.numberOfHitsPerPage.get(currentPage);
        } else {
            return false;
        }
    }

    @Override
    public boolean hasPreviousItem() {
        if (this.currentHitPerPage.containsKey(currentPage)) {
            return this.currentHitPerPage.get(currentPage) > 1;
        } else {
            return false;
        }
    }

    @Override
    @NbBundle.Messages("AccountsText.nextItem.exception.msg=No next item.")
    public int nextItem() {
        if (hasNextItem()) {
            return currentHitPerPage.merge(currentPage, 1, Integer::sum);
        } else {
            throw new IllegalStateException(Bundle.AccountsText_nextItem_exception_msg());
        }
    }

    @Override
    @NbBundle.Messages("AccountsText.previousItem.exception.msg=No previous item.")
    public int previousItem() {
        if (hasPreviousItem()) {
            return currentHitPerPage.merge(currentPage, -1, Integer::sum);
        } else {
            throw new IllegalStateException(Bundle.AccountsText_previousItem_exception_msg());
        }
    }

    @Override
    public int currentItem() {
        if (this.currentHitPerPage.containsKey(currentPage)) {
            return currentHitPerPage.get(currentPage);
        } else {
            return 0;
        }
    }

    @Override
    public LinkedHashMap<Integer, Integer> getHitsPages() {
        return this.numberOfHitsPerPage;
    }

    synchronized private void loadPageInfo() {
        if (isPageInfoLoaded) {
            return;
        }
        if (chunkId != null) {//if a chunk is specified, only show that chunk/page
            this.numberPagesForFile = 1;
            this.currentPage = chunkId;
            this.numberOfHitsPerPage.put(chunkId, 0);
            this.pages.add(chunkId);
            this.currentHitPerPage.put(chunkId, 0);
        } else {
            try {
                this.numberPagesForFile = solrServer.queryNumFileChunks(this.solrObjectId);
            } catch (KeywordSearchModuleException | NoOpenCoreException ex) {
                LOGGER.log(Level.WARNING, "Could not get number pages for content " + this.solrDocumentId, ex); //NON-NLS
                return;
            }

            //if has chunks, get pages with hits
            TreeSet<Integer> sortedPagesWithHits = new TreeSet<>();
            SolrQuery q = new SolrQuery();
            q.setShowDebugInfo(DEBUG); //debug
            q.setQuery(queryString);
            q.setFields(Server.Schema.ID.toString());  //for this case we only need the document ids
            q.addFilterQuery(Server.Schema.ID.toString() + ":" + this.solrObjectId + Server.CHUNK_ID_SEPARATOR + "*");

            try {
                QueryResponse response = solrServer.query(q, METHOD.POST);
                for (SolrDocument resultDoc : response.getResults()) {
                    final String resultDocumentId = resultDoc.getFieldValue(Server.Schema.ID.toString()).toString();
                    // Put the solr chunk id in the map
                    String resultChunkID = StringUtils.substringAfter(resultDocumentId, Server.CHUNK_ID_SEPARATOR);
                    if (StringUtils.isNotBlank(resultChunkID)) {
                        sortedPagesWithHits.add(Integer.parseInt(resultChunkID));
                    } else {
                        sortedPagesWithHits.add(0);
                    }
                }

            } catch (KeywordSearchModuleException | NoOpenCoreException | NumberFormatException ex) {
                LOGGER.log(Level.WARNING, "Error executing Solr highlighting query: " + keywords, ex); //NON-NLS
            }

            //set page to first page having highlights
            if (sortedPagesWithHits.isEmpty()) {
                this.currentPage = 0;
            } else {
                this.currentPage = sortedPagesWithHits.first();
            }

            for (Integer page : sortedPagesWithHits) {
                numberOfHitsPerPage.put(page, 0); //unknown number of matches in the page
                pages.add(page);
                currentHitPerPage.put(page, 0); //set current hit to 0th
            }
        }

        isPageInfoLoaded = true;
    }

    @Override
    @NbBundle.Messages({"AccountsText.getMarkup.noMatchMsg="
        + "<html><pre><span style\\\\='background\\\\:yellow'>There were no keyword hits on this page. <br />"
        + "The keyword could have been in the file name."
        + " <br />Advance to another page if present, or to view the original text, choose File Text"
        + " <br />in the drop down menu to the right...</span></pre></html>",
        "AccountsText.getMarkup.queryFailedMsg="
        + "<html><pre><span style\\\\='background\\\\:yellow'>Failed to retrieve keyword hit results."
        + " <br />Confirm that Autopsy can connect to the Solr server. "
        + "<br /></span></pre></html>"})
    public String getText() {
        loadPageInfo(); //inits once

        SolrQuery q = new SolrQuery();
        q.setShowDebugInfo(DEBUG); //debug
        q.addHighlightField(HIGHLIGHT_FIELD);
        q.setQuery(queryString);

        //set the documentID filter
        String queryDocumentID = this.solrObjectId + Server.CHUNK_ID_SEPARATOR + this.currentPage;
        q.addFilterQuery(Server.Schema.ID.toString() + ":" + queryDocumentID);

        //configure the highlighter
        q.setParam("hl.useFastVectorHighlighter", "true"); //fast highlighter scales better than standard one NON-NLS
        q.setParam("hl.tag.pre", HIGHLIGHT_PRE); //makes sense for FastVectorHighlighter only NON-NLS
        q.setParam("hl.tag.post", HIGHLIGHT_POST); //makes sense for FastVectorHighlighter only NON-NLS
        q.setParam("hl.fragListBuilder", "single"); //makes sense for FastVectorHighlighter only NON-NLS
        q.setParam("hl.maxAnalyzedChars", Server.HL_ANALYZE_CHARS_UNLIMITED); //docs says makes sense for the original Highlighter only, but not really //NON-NLS

        try {
            //extract highlighting and bail early on null responses
            Map<String, Map<String, List<String>>> highlightingPerDocument = solrServer.query(q, METHOD.POST).getHighlighting();
            Map<String, List<String>> highlightingPerField = highlightingPerDocument.get(queryDocumentID);
            if (highlightingPerField == null) {
                return Bundle.AccountsText_getMarkup_noMatchMsg();
            }
            List<String> highlights = highlightingPerField.get(HIGHLIGHT_FIELD);
            if (highlights == null) {
                return Bundle.AccountsText_getMarkup_noMatchMsg();
            }

            //There should only be one item
            String highlighting = highlights.get(0).trim();

            /*
             * use regex matcher to iterate over occurences of HIGHLIGHT_PRE,
             * and prepend them with an anchor tag.
             */
            Matcher m = ANCHOR_DETECTION_PATTERN.matcher(highlighting);
            StringBuffer sb = new StringBuffer(highlighting.length());
            int count = 0;
            while (m.find()) {
                count++;
                m.appendReplacement(sb, INSERT_PREFIX + count + INSERT_POSTFIX);
            }
            m.appendTail(sb);

            //store total hits for this page, now that we know it
            this.numberOfHitsPerPage.put(this.currentPage, count);
            if (this.currentItem() == 0 && this.hasNextItem()) {
                this.nextItem();
            }

            // extracted content (minus highlight tags) is HTML-escaped
            return "<html><pre>" + sb.toString() + "</pre></html>"; //NON-NLS
        } catch (Exception ex) {
            LOGGER.log(Level.WARNING, "Error executing Solr highlighting query: " + keywords, ex); //NON-NLS
            return Bundle.AccountsText_getMarkup_queryFailedMsg();
        }
    }

    @Override
    public String toString() {
        return displayName;
    }

    @Override
    public boolean isSearchable() {
        return true;
    }

    @Override
    public String getAnchorPrefix() {
        return ANCHOR_NAME_PREFIX;
    }

    @Override
    public int getNumberHits() {
        if (!this.numberOfHitsPerPage.containsKey(this.currentPage)) {
            return 0;
        }
        return this.numberOfHitsPerPage.get(this.currentPage);
    }
}