Autopsy  4.5.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
SolrSearchService.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2017 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.io.File;
22 import java.io.IOException;
23 import java.lang.reflect.InvocationTargetException;
24 import java.net.InetAddress;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.MissingResourceException;
28 import java.util.logging.Level;
29 import javax.swing.JDialog;
30 import javax.swing.JOptionPane;
31 import javax.swing.SwingUtilities;
32 import org.apache.commons.lang.math.NumberUtils;
33 import org.apache.commons.io.FileUtils;
34 import org.apache.solr.client.solrj.SolrServerException;
35 import org.apache.solr.client.solrj.impl.HttpSolrServer;
36 import org.openide.util.NbBundle;
37 import org.openide.util.lookup.ServiceProvider;
38 import org.openide.util.lookup.ServiceProviders;
48 import org.sleuthkit.datamodel.BlackboardArtifact;
49 import org.sleuthkit.datamodel.TskCoreException;
50 
55 @ServiceProviders(value = {
56  @ServiceProvider(service = KeywordSearchService.class)
57  ,
58  @ServiceProvider(service = AutopsyService.class)}
59 )
61 
62  private static final String BAD_IP_ADDRESS_FORMAT = "ioexception occurred when talking to server"; //NON-NLS
63  private static final String SERVER_REFUSED_CONNECTION = "server refused connection"; //NON-NLS
64  private static final int IS_REACHABLE_TIMEOUT_MS = 1000;
65  private static final int LARGE_INDEX_SIZE_GB = 50;
66  private static final int GIANT_INDEX_SIZE_GB = 500;
67  private static final Logger logger = Logger.getLogger(SolrSearchService.class.getName());
68 
77  @Override
78  public void indexArtifact(BlackboardArtifact artifact) throws TskCoreException {
79  if (artifact == null) {
80  return;
81  }
82 
83  // We only support artifact indexing for Autopsy versions that use
84  // the negative range for artifact ids.
85  if (artifact.getArtifactID() > 0) {
86  return;
87  }
88  final Ingester ingester = Ingester.getDefault();
89 
90  try {
91  ingester.indexMetaDataOnly(artifact);
92  ingester.indexText(new ArtifactTextExtractor(), artifact, null);
93  } catch (Ingester.IngesterException ex) {
94  throw new TskCoreException(ex.getCause().getMessage(), ex);
95  }
96  }
97 
106  @Override
107  public void tryConnect(String host, int port) throws KeywordSearchServiceException {
108  HttpSolrServer solrServer = null;
109  if (host == null || host.isEmpty()) {
110  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.MissingHostname")); //NON-NLS
111  }
112  try {
113  solrServer = new HttpSolrServer("http://" + host + ":" + Integer.toString(port) + "/solr"); //NON-NLS
114  KeywordSearch.getServer().connectToSolrServer(solrServer);
115  } catch (SolrServerException ex) {
116  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort")); //NON-NLS
117  } catch (IOException ex) {
118  String result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
119  String message = ex.getCause().getMessage().toLowerCase();
120  if (message.startsWith(SERVER_REFUSED_CONNECTION)) {
121  try {
122  if (InetAddress.getByName(host).isReachable(IS_REACHABLE_TIMEOUT_MS)) {
123  // if we can reach the host, then it's probably port problem
124  result = Bundle.SolrConnectionCheck_Port();
125  } else {
126  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
127  }
128  } catch (IOException | MissingResourceException any) {
129  // it may be anything
130  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.HostnameOrPort"); //NON-NLS
131  }
132  } else if (message.startsWith(BAD_IP_ADDRESS_FORMAT)) {
133  result = NbBundle.getMessage(SolrSearchService.class, "SolrConnectionCheck.Hostname"); //NON-NLS
134  }
135  throw new KeywordSearchServiceException(result);
136  } catch (NumberFormatException ex) {
137  throw new KeywordSearchServiceException(Bundle.SolrConnectionCheck_Port());
138  } catch (IllegalArgumentException ex) {
139  throw new KeywordSearchServiceException(ex.getMessage());
140  } finally {
141  if (null != solrServer) {
142  solrServer.shutdown();
143  }
144  }
145  }
146 
152  @NbBundle.Messages({
153  "# {0} - case directory", "SolrSearchService.exceptionMessage.noIndexMetadata=Unable to create IndexMetaData from case directory: {0}",
154  "SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not contain a current Solr core so could not delete the case",
155  "# {0} - index folder path", "SolrSearchService.exceptionMessage.failedToDeleteIndexFiles=Failed to delete text index files at {0}"
156  })
157  @Override
159  String caseDirectory = metadata.getCaseDirectory();
160  IndexMetadata indexMetadata;
161  try {
162  indexMetadata = new IndexMetadata(caseDirectory);
163  } catch (IndexMetadata.TextIndexMetadataException ex) {
164  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.exceptionMessage.noIndexMetadata", caseDirectory), ex);
165  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class, "SolrSearchService.exceptionMessage.noIndexMetadata", caseDirectory), ex);
166  }
167  //find the index for the current version of solr (the one we are connected to) and delete its core using the index name
168  String currentSchema = IndexFinder.getCurrentSchemaVersion();
169  String currentSolr = IndexFinder.getCurrentSolrVersion();
170  for (Index index : indexMetadata.getIndexes()) {
171  if (index.getSolrVersion().equals(currentSolr) && index.getSchemaVersion().equals(currentSchema)) {
172  /*
173  * Unload/delete the core on the server and then delete the text
174  * index files.
175  */
176  KeywordSearch.getServer().deleteCore(index.getIndexName(), metadata.getCaseType());
177  if (!FileUtil.deleteDir(new File(index.getIndexPath()).getParentFile())) {
178  throw new KeywordSearchServiceException(Bundle.SolrSearchService_exceptionMessage_failedToDeleteIndexFiles(index.getIndexPath()));
179  }
180  }
181  return; //only one core exists for each combination of solr and schema version
182  }
183 
184  //this code this code will only execute if an index for the current core was not found
185  logger.log(Level.WARNING, NbBundle.getMessage(SolrSearchService.class,
186  "SolrSearchService.exceptionMessage.noCurrentSolrCore"));
187  throw new KeywordSearchServiceException(NbBundle.getMessage(SolrSearchService.class,
188  "SolrSearchService.exceptionMessage.noCurrentSolrCore"));
189  }
190 
191  @Override
192  public void close() throws IOException {
193  }
194 
195  @Override
196  public String getServiceName() {
197  return NbBundle.getMessage(this.getClass(), "SolrSearchService.ServiceName");
198  }
199 
207  @Override
208  @NbBundle.Messages({
209  "SolrSearch.lookingForMetadata.msg=Looking for text index metadata file",
210  "SolrSearch.readingIndexes.msg=Reading text index metadata file",
211  "SolrSearch.findingIndexes.msg=Looking for existing text index directories",
212  "SolrSearch.creatingNewIndex.msg=Creating new text index",
213  "SolrSearch.checkingForLatestIndex.msg=Looking for text index with latest Solr and schema version",
214  "SolrSearch.indentifyingIndex.msg=Identifying text index to use",
215  "SolrSearch.openCore.msg=Opening text index",
216  "SolrSearch.openLargeCore.msg=Opening text index. This may take several minutes.",
217  "SolrSearch.openGiantCore.msg=Opening text index. Text index for this case is very large and may take long time to load.",
218  "SolrSearch.complete.msg=Text index successfully opened"})
220  if (context.cancelRequested()) {
221  return;
222  }
223 
224  ProgressIndicator progress = context.getProgressIndicator();
225  int totalNumProgressUnits = 7;
226  int progressUnitsCompleted = 0;
227 
228  String caseDirPath = context.getCase().getCaseDirectory();
229  Case theCase = context.getCase();
230  List<Index> indexes = new ArrayList<>();
231  progress.start(Bundle.SolrSearch_lookingForMetadata_msg(), totalNumProgressUnits);
232  if (IndexMetadata.isMetadataFilePresent(caseDirPath)) {
233  try {
234  // metadata file exists, get list of existing Solr cores for this case
235  progressUnitsCompleted++;
236  progress.progress(Bundle.SolrSearch_findingIndexes_msg(), progressUnitsCompleted);
237  IndexMetadata indexMetadata = new IndexMetadata(caseDirPath);
238  indexes = indexMetadata.getIndexes();
239  } catch (IndexMetadata.TextIndexMetadataException ex) {
240  logger.log(Level.SEVERE, String.format("Unable to read text index metadata file"), ex);
241  throw new AutopsyServiceException("Unable to read text index metadata file", ex);
242  }
243  } else {
244  // metadata file doesn't exist.
245  // do case subdirectory search to look for Solr 4 Schema 1.8 indexes
246  progressUnitsCompleted++;
247  progress.progress(Bundle.SolrSearch_findingIndexes_msg(), progressUnitsCompleted);
248  Index oldIndex = IndexFinder.findOldIndexDir(theCase);
249  if (oldIndex != null) {
250  // add index to the list of indexes that exist for this case
251  indexes.add(oldIndex);
252  }
253  }
254 
255  if (context.cancelRequested()) {
256  return;
257  }
258 
259  // check if we found any existing indexes
260  Index currentVersionIndex = null;
261  if (indexes.isEmpty()) {
262  // new case that doesn't have an existing index. create new index folder
263  progressUnitsCompleted++;
264  progress.progress(Bundle.SolrSearch_creatingNewIndex_msg(), progressUnitsCompleted);
265  currentVersionIndex = IndexFinder.createLatestVersionIndexDir(theCase);
266  // add current index to the list of indexes that exist for this case
267  indexes.add(currentVersionIndex);
268  } else {
269  // check if one of the existing indexes is for latest Solr version and schema
270  progressUnitsCompleted++;
271  progress.progress(Bundle.SolrSearch_checkingForLatestIndex_msg(), progressUnitsCompleted);
272  currentVersionIndex = IndexFinder.findLatestVersionIndexDir(indexes);
273  if (currentVersionIndex == null) {
274  // found existing index(es) but none were for latest Solr version and schema version
275  progressUnitsCompleted++;
276  progress.progress(Bundle.SolrSearch_indentifyingIndex_msg(), progressUnitsCompleted);
277  Index indexToUse = IndexFinder.identifyIndexToUse(indexes);
278  if (indexToUse == null) {
279  // unable to find index that can be used
280  throw new AutopsyServiceException("Unable to find index that can be used for this case");
281  }
282 
283  if (context.cancelRequested()) {
284  return;
285  }
286 
287  double currentSolrVersion = NumberUtils.toDouble(IndexFinder.getCurrentSolrVersion());
288  double indexSolrVersion = NumberUtils.toDouble(indexToUse.getSolrVersion());
289  if (indexSolrVersion == currentSolrVersion) {
290  // latest Solr version but not latest schema. index should be used in read-only mode
292  // pop up a message box to indicate the read-only restrictions.
293  JOptionPane optionPane = new JOptionPane(
294  NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.msg"),
295  JOptionPane.WARNING_MESSAGE,
296  JOptionPane.DEFAULT_OPTION);
297  try {
298  SwingUtilities.invokeAndWait(() -> {
299  JDialog dialog = optionPane.createDialog(NbBundle.getMessage(this.getClass(), "SolrSearchService.IndexReadOnlyDialog.title"));
300  dialog.setVisible(true);
301  });
302  } catch (InterruptedException ex) {
303  // Cancelled
304  return;
305  } catch (InvocationTargetException ex) {
306  throw new AutopsyServiceException("Error displaying limited search features warning dialog", ex);
307  }
308  }
309  // proceed with case open
310  currentVersionIndex = indexToUse;
311  } else {
312  // index needs to be upgraded to latest supported version of Solr
313  throw new AutopsyServiceException("Unable to find index to use for Case open");
314  }
315  }
316  }
317 
318  try {
319  // update text index metadata file
320  if (!indexes.isEmpty()) {
321  IndexMetadata indexMetadata = new IndexMetadata(caseDirPath, indexes);
322  }
323  } catch (IndexMetadata.TextIndexMetadataException ex) {
324  throw new AutopsyServiceException("Failed to save Solr core info in text index metadata file", ex);
325  }
326 
327  // open core
328  try {
329  // check text index size to gauge estimated time to open/load the index
330  long indexSizeInBytes = FileUtils.sizeOfDirectory(new File(currentVersionIndex.getIndexPath()));
331  long sizeInGb = indexSizeInBytes / 1000000000;
332  if (sizeInGb < LARGE_INDEX_SIZE_GB) {
333  progress.progress(Bundle.SolrSearch_openCore_msg(), totalNumProgressUnits - 1);
334  } else if (sizeInGb >= LARGE_INDEX_SIZE_GB && sizeInGb < GIANT_INDEX_SIZE_GB) {
335  progress.switchToIndeterminate(Bundle.SolrSearch_openLargeCore_msg());
336  } else {
337  progress.switchToIndeterminate(Bundle.SolrSearch_openGiantCore_msg());
338  }
339 
340  KeywordSearch.getServer().openCoreForCase(theCase, currentVersionIndex);
341  } catch (KeywordSearchModuleException ex) {
342  throw new AutopsyServiceException(String.format("Failed to open or create core for %s", caseDirPath), ex);
343  }
344 
345  progress.progress(Bundle.SolrSearch_complete_msg(), totalNumProgressUnits);
346  }
347 
355  @Override
357  /*
358  * TODO (JIRA 2525): The following code KeywordSearch.CaseChangeListener
359  * gambles that any BlackboardResultWriters (SwingWorkers) will complete
360  * in less than roughly two seconds. This stuff should be reworked using
361  * an ExecutorService and tasks with Futures.
362  */
363  KeywordSearchResultFactory.BlackboardResultWriter.stopAllWriters();
364  try {
365  Thread.sleep(2000);
366  } catch (InterruptedException ex) {
367  logger.log(Level.SEVERE, "Unexpected interrupt while waiting for BlackboardResultWriters to terminate", ex);
368  }
369 
370  try {
371  KeywordSearch.getServer().closeCore();
372  } catch (KeywordSearchModuleException ex) {
373  throw new AutopsyServiceException(String.format("Failed to close core for %s", context.getCase().getCaseDirectory()), ex);
374  }
375  }
376 }
void start(String message, int totalWorkUnits)
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static boolean deleteDir(File dirPath)
Definition: FileUtil.java:47

Copyright © 2012-2016 Basis Technology. Generated on: Tue Feb 20 2018
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.