Autopsy  4.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
Server.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2016 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Path;
37 import java.nio.file.Paths;
38 import java.util.ArrayList;
39 import java.util.Collection;
40 import java.util.List;
41 import java.util.concurrent.locks.ReentrantReadWriteLock;
42 import java.util.logging.Level;
43 import javax.swing.AbstractAction;
44 import org.apache.solr.client.solrj.SolrQuery;
45 import org.apache.solr.client.solrj.SolrRequest;
46 import org.apache.solr.client.solrj.SolrServerException;
47 import org.apache.solr.client.solrj.impl.HttpSolrServer;
48 import org.apache.solr.client.solrj.impl.XMLResponseParser;
49 import org.apache.solr.client.solrj.request.CoreAdminRequest;
50 import org.apache.solr.client.solrj.response.CoreAdminResponse;
51 import org.apache.solr.client.solrj.response.QueryResponse;
52 import org.apache.solr.client.solrj.response.TermsResponse;
53 import org.apache.solr.common.SolrDocument;
54 import org.apache.solr.common.SolrDocumentList;
55 import org.apache.solr.common.SolrException;
56 import org.apache.solr.common.SolrInputDocument;
57 import org.apache.solr.common.util.NamedList;
58 import org.openide.modules.InstalledFileLocator;
59 import org.openide.modules.Places;
60 import org.openide.util.NbBundle;
68 import org.sleuthkit.datamodel.Content;
69 
74 public class Server {
75 
79  public static enum Schema {
80 
81  ID {
82  @Override
83  public String toString() {
84  return "id"; //NON-NLS
85  }
86  },
87  IMAGE_ID {
88  @Override
89  public String toString() {
90  return "image_id"; //NON-NLS
91  }
92  },
93  // This is not stored or index . it is copied to Text and Content_Ws
94  CONTENT {
95  @Override
96  public String toString() {
97  return "content"; //NON-NLS
98  }
99  },
100  TEXT {
101  @Override
102  public String toString() {
103  return "text"; //NON-NLS
104  }
105  },
106  CONTENT_WS {
107  @Override
108  public String toString() {
109  return "content_ws"; //NON-NLS
110  }
111  },
112  FILE_NAME {
113  @Override
114  public String toString() {
115  return "file_name"; //NON-NLS
116  }
117  },
118  // note that we no longer index this field
119  CTIME {
120  @Override
121  public String toString() {
122  return "ctime"; //NON-NLS
123  }
124  },
125  // note that we no longer index this field
126  ATIME {
127  @Override
128  public String toString() {
129  return "atime"; //NON-NLS
130  }
131  },
132  // note that we no longer index this field
133  MTIME {
134  @Override
135  public String toString() {
136  return "mtime"; //NON-NLS
137  }
138  },
139  // note that we no longer index this field
140  CRTIME {
141  @Override
142  public String toString() {
143  return "crtime"; //NON-NLS
144  }
145  },
146  NUM_CHUNKS {
147  @Override
148  public String toString() {
149  return "num_chunks"; //NON-NLS
150  }
151  },
152  };
153 
154  public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
155  //max content size we can send to Solr
156  public static final long MAX_CONTENT_SIZE = 1L * 1024 * 1024 * 1024;
157  private static final Logger logger = Logger.getLogger(Server.class.getName());
158  private static final String DEFAULT_CORE_NAME = "coreCase"; //NON-NLS
159  public static final String CORE_EVT = "CORE_EVT"; //NON-NLS
160  @Deprecated
161  public static final char ID_CHUNK_SEP = '_';
162  public static final String CHUNK_ID_SEPARATOR = "_";
163  private String javaPath = "java"; //NON-NLS
164  public static final Charset DEFAULT_INDEXED_TEXT_CHARSET = Charset.forName("UTF-8");
165  private static final int MAX_SOLR_MEM_MB = 512; //TODO set dynamically based on avail. system resources
166  private Process curSolrProcess = null;
167  static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
168  static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
169  static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
170  private static final String KEY = "jjk#09s"; //NON-NLS
171  static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
172  static final int DEFAULT_SOLR_SERVER_PORT = 23232;
173  static final int DEFAULT_SOLR_STOP_PORT = 34343;
174  private int currentSolrServerPort = 0;
175  private int currentSolrStopPort = 0;
176  private static final boolean DEBUG = false;//(Version.getBuildType() == Version.Type.DEVELOPMENT);
178  private static final String SOLR = "solr";
179  private static final String CORE_PROPERTIES = "core.properties";
180 
181  public enum CORE_EVT_STATES {
182 
183  STOPPED, STARTED
184  };
185 
186  // A reference to the locally running Solr instance.
187  private final HttpSolrServer localSolrServer;
188 
189  // A reference to the Solr server we are currently connected to for the Case.
190  // This could be a local or remote server.
191  private HttpSolrServer currentSolrServer;
192 
193  private Core currentCore;
194  private final ReentrantReadWriteLock currentCoreLock;
195 
196  private final File solrFolder;
197  private final ServerAction serverAction;
199 
204  Server() {
205  initSettings();
206 
207  this.localSolrServer = new HttpSolrServer("http://localhost:" + currentSolrServerPort + "/solr"); //NON-NLS
208  serverAction = new ServerAction();
209  solrFolder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS
210  javaPath = PlatformUtil.getJavaPath();
211 
212  currentCoreLock = new ReentrantReadWriteLock(true);
213  uncPathUtilities = new UNCPathUtilities();
214 
215  logger.log(Level.INFO, "Created Server instance"); //NON-NLS
216  }
217 
218  private void initSettings() {
219 
220  if (ModuleSettings.settingExists(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT)) {
221  try {
222  currentSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
223  } catch (NumberFormatException nfe) {
224  logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
225  currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
226  }
227  } else {
228  currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
229  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(currentSolrServerPort));
230  }
231 
232  if (ModuleSettings.settingExists(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT)) {
233  try {
234  currentSolrStopPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT));
235  } catch (NumberFormatException nfe) {
236  logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
237  currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
238  }
239  } else {
240  currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
241  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(currentSolrStopPort));
242  }
243  }
244 
245  @Override
246  public void finalize() throws java.lang.Throwable {
247  stop();
248  super.finalize();
249  }
250 
251  public void addServerActionListener(PropertyChangeListener l) {
252  serverAction.addPropertyChangeListener(l);
253  }
254 
255  int getCurrentSolrServerPort() {
256  return currentSolrServerPort;
257  }
258 
259  int getCurrentSolrStopPort() {
260  return currentSolrStopPort;
261  }
262 
266  private static class InputStreamPrinterThread extends Thread {
267 
268  InputStream stream;
269  OutputStream out;
270  volatile boolean doRun = true;
271 
272  InputStreamPrinterThread(InputStream stream, String type) {
273  this.stream = stream;
274  try {
275  final String log = Places.getUserDirectory().getAbsolutePath()
276  + File.separator + "var" + File.separator + "log" //NON-NLS
277  + File.separator + "solr.log." + type; //NON-NLS
278  File outputFile = new File(log.concat(".0"));
279  File first = new File(log.concat(".1"));
280  File second = new File(log.concat(".2"));
281  if (second.exists()) {
282  second.delete();
283  }
284  if (first.exists()) {
285  first.renameTo(second);
286  }
287  if (outputFile.exists()) {
288  outputFile.renameTo(first);
289  } else {
290  outputFile.createNewFile();
291  }
292  out = new FileOutputStream(outputFile);
293 
294  } catch (Exception ex) {
295  logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
296  }
297  }
298 
299  void stopRun() {
300  doRun = false;
301  }
302 
303  @Override
304  public void run() {
305 
306  try (InputStreamReader isr = new InputStreamReader(stream);
307  BufferedReader br = new BufferedReader(isr);
308  OutputStreamWriter osw = new OutputStreamWriter(out, PlatformUtil.getDefaultPlatformCharset());
309  BufferedWriter bw = new BufferedWriter(osw);) {
310 
311  String line = null;
312  while (doRun && (line = br.readLine()) != null) {
313  bw.write(line);
314  bw.newLine();
315  if (DEBUG) {
316  //flush buffers if dev version for debugging
317  bw.flush();
318  }
319  }
320  bw.flush();
321  } catch (IOException ex) {
322  logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
323  }
324  }
325  }
326 
332  List<Long> getSolrPIDs() {
333  List<Long> pids = new ArrayList<>();
334 
335  //NOTE: these needs to be in sync with process start string in start()
336  final String pidsQuery = "Args.4.eq=-DSTOP.KEY=" + KEY + ",Args.6.eq=start.jar"; //NON-NLS
337 
338  long[] pidsArr = PlatformUtil.getJavaPIDs(pidsQuery);
339  if (pidsArr != null) {
340  for (int i = 0; i < pidsArr.length; ++i) {
341  pids.add(pidsArr[i]);
342  }
343  }
344 
345  return pids;
346  }
347 
352  void killSolr() {
353  List<Long> solrPids = getSolrPIDs();
354  for (long pid : solrPids) {
355  logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
356  PlatformUtil.killProcess(pid);
357  }
358  }
359 
365  void start() throws KeywordSearchModuleException, SolrServerNoPortException {
366  if (isRunning()) {
367  // If a Solr server is running we stop it.
368  stop();
369  }
370 
371  if (!isPortAvailable(currentSolrServerPort)) {
372  // There is something already listening on our port. Let's see if
373  // this is from an earlier run that didn't successfully shut down
374  // and if so kill it.
375  final List<Long> pids = this.getSolrPIDs();
376 
377  // If the culprit listening on the port is not a Solr process
378  // we refuse to start.
379  if (pids.isEmpty()) {
380  throw new SolrServerNoPortException(currentSolrServerPort);
381  }
382 
383  // Ok, we've tried to stop it above but there still appears to be
384  // a Solr process listening on our port so we forcefully kill it.
385  killSolr();
386 
387  // If either of the ports are still in use after our attempt to kill
388  // previously running processes we give up and throw an exception.
389  if (!isPortAvailable(currentSolrServerPort)) {
390  throw new SolrServerNoPortException(currentSolrServerPort);
391  }
392  if (!isPortAvailable(currentSolrStopPort)) {
393  throw new SolrServerNoPortException(currentSolrStopPort);
394  }
395  }
396 
397  logger.log(Level.INFO, "Starting Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
398 
399  if (isPortAvailable(currentSolrServerPort)) {
400  logger.log(Level.INFO, "Port [{0}] available, starting Solr", currentSolrServerPort); //NON-NLS
401  try {
402  final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + Integer.toString(MAX_SOLR_MEM_MB) + "m"; //NON-NLS
403  List<String> commandLine = new ArrayList<>();
404  commandLine.add(javaPath);
405  commandLine.add(MAX_SOLR_MEM_MB_PAR);
406  commandLine.add("-DSTOP.PORT=" + currentSolrStopPort); //NON-NLS
407  commandLine.add("-Djetty.port=" + currentSolrServerPort); //NON-NLS
408  commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
409  commandLine.add("-jar"); //NON-NLS
410  commandLine.add("start.jar"); //NON-NLS
411 
412  ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
413  solrProcessBuilder.directory(solrFolder);
414 
415  // Redirect stdout and stderr to files to prevent blocking.
416  Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
417  solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
418 
419  Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
420  solrProcessBuilder.redirectError(solrStderrPath.toFile());
421 
422  logger.log(Level.INFO, "Starting Solr using: {0}", solrProcessBuilder.command()); //NON-NLS
423  curSolrProcess = solrProcessBuilder.start();
424  logger.log(Level.INFO, "Finished starting Solr"); //NON-NLS
425 
426  try {
427  //block for 10 seconds, give time to fully start the process
428  //so if it's restarted solr operations can be resumed seamlessly
429  Thread.sleep(10 * 1000);
430  } catch (InterruptedException ex) {
431  logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
432  }
433 
434  final List<Long> pids = this.getSolrPIDs();
435  logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
436  } catch (SecurityException ex) {
437  logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
438  throw new KeywordSearchModuleException(
439  NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
440  } catch (IOException ex) {
441  logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
442  throw new KeywordSearchModuleException(
443  NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
444  }
445  }
446  }
447 
453  static boolean isPortAvailable(int port) {
454  ServerSocket ss = null;
455  try {
456 
457  ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
458  if (ss.isBound()) {
459  ss.setReuseAddress(true);
460  ss.close();
461  return true;
462  }
463 
464  } catch (IOException e) {
465  } finally {
466  if (ss != null) {
467  try {
468  ss.close();
469  } catch (IOException e) {
470  /*
471  * should not be thrown
472  */
473  }
474  }
475  }
476  return false;
477  }
478 
484  void changeSolrServerPort(int port) {
485  currentSolrServerPort = port;
486  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
487  }
488 
494  void changeSolrStopPort(int port) {
495  currentSolrStopPort = port;
496  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
497  }
498 
504  synchronized void stop() {
505 
506  try {
507  // Close any open core before stopping server
508  closeCore();
509  } catch (KeywordSearchModuleException e) {
510  logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
511  }
512 
513  try {
514  logger.log(Level.INFO, "Stopping Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
515 
516  //try graceful shutdown
517  final String[] SOLR_STOP_CMD = {
518  javaPath,
519  "-DSTOP.PORT=" + currentSolrStopPort, //NON-NLS
520  "-DSTOP.KEY=" + KEY, //NON-NLS
521  "-jar", //NON-NLS
522  "start.jar", //NON-NLS
523  "--stop", //NON-NLS
524  };
525  Process stop = Runtime.getRuntime().exec(SOLR_STOP_CMD, null, solrFolder);
526  logger.log(Level.INFO, "Waiting for stopping Solr server"); //NON-NLS
527  stop.waitFor();
528 
529  //if still running, forcefully stop it
530  if (curSolrProcess != null) {
531  curSolrProcess.destroy();
532  curSolrProcess = null;
533  }
534 
535  } catch (InterruptedException | IOException ex) {
536  } finally {
537  //stop Solr stream -> log redirect threads
538  try {
539  if (errorRedirectThread != null) {
540  errorRedirectThread.stopRun();
541  errorRedirectThread = null;
542  }
543  } finally {
544  //if still running, kill it
545  killSolr();
546  }
547 
548  logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
549  }
550  }
551 
559  synchronized boolean isRunning() throws KeywordSearchModuleException {
560  try {
561 
562  if (isPortAvailable(currentSolrServerPort)) {
563  return false;
564  }
565 
566  if (curSolrProcess != null && !curSolrProcess.isAlive()) {
567  return false;
568  }
569 
570  // making a status request here instead of just doing solrServer.ping(), because
571  // that doesn't work when there are no cores
572  //TODO handle timeout in cases when some other type of server on that port
573  CoreAdminRequest.getStatus(null, localSolrServer);
574 
575  logger.log(Level.INFO, "Solr server is running"); //NON-NLS
576  } catch (SolrServerException ex) {
577 
578  Throwable cause = ex.getRootCause();
579 
580  // TODO: check if SocketExceptions should actually happen (is
581  // probably caused by starting a connection as the server finishes
582  // shutting down)
583  if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
584  logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
585  return false;
586  } else {
587  throw new KeywordSearchModuleException(
588  NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
589  }
590  } catch (SolrException ex) {
591  // Just log 404 errors for now...
592  logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
593  return false;
594  } catch (IOException ex) {
595  throw new KeywordSearchModuleException(
596  NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
597  }
598 
599  return true;
600  }
601 
602  /*
603  * ** Convenience methods for use while we only open one case at a time ***
604  */
614  void openCoreForCase(Case theCase) throws KeywordSearchModuleException {
615  currentCoreLock.writeLock().lock();
616  try {
617  currentCore = openCore(theCase);
618  serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
619  } finally {
620  currentCoreLock.writeLock().unlock();
621  }
622  }
623 
629  boolean coreIsOpen() {
630  currentCoreLock.readLock().lock();
631  try {
632  return (null != currentCore);
633  } finally {
634  currentCoreLock.readLock().unlock();
635  }
636  }
637 
638  void closeCore() throws KeywordSearchModuleException {
639  currentCoreLock.writeLock().lock();
640  try {
641  if (null != currentCore) {
642  currentCore.close();
643  currentCore = null;
644  serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
645  }
646  } finally {
647  currentCoreLock.writeLock().unlock();
648  }
649  }
650 
651  void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
652  currentCoreLock.readLock().lock();
653  try {
654  currentCore.addDocument(doc);
655  } finally {
656  currentCoreLock.readLock().unlock();
657  }
658  }
659 
667  String geCoreDataDirPath(Case theCase) {
668  String indexDir = theCase.getModuleDirectory() + File.separator + "keywordsearch" + File.separator + "data"; //NON-NLS
669  if (uncPathUtilities != null) {
670  // if we can check for UNC paths, do so, otherwise just return the indexDir
671  String result = uncPathUtilities.mappedDriveToUNC(indexDir);
672  if (result == null) {
673  uncPathUtilities.rescanDrives();
674  result = uncPathUtilities.mappedDriveToUNC(indexDir);
675  }
676  if (result == null) {
677  return indexDir;
678  }
679  return result;
680  }
681  return indexDir;
682  }
683 
697  private Core openCore(Case theCase) throws KeywordSearchModuleException {
698  try {
699  if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
700  currentSolrServer = this.localSolrServer;
701  } else {
702  String host = UserPreferences.getIndexingServerHost();
703  String port = UserPreferences.getIndexingServerPort();
704  currentSolrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
705  }
706  connectToSolrServer(currentSolrServer);
707 
708  } catch (SolrServerException | IOException ex) {
709  throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex);
710  }
711 
712  String dataDir = geCoreDataDirPath(theCase);
713  String coreName = theCase.getTextIndexName();
714  return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, new File(dataDir), theCase.getCaseType());
715  }
716 
722  void commit() throws SolrServerException, NoOpenCoreException {
723  currentCoreLock.readLock().lock();
724  try {
725  if (null == currentCore) {
726  throw new NoOpenCoreException();
727  }
728  currentCore.commit();
729  } finally {
730  currentCoreLock.readLock().unlock();
731  }
732  }
733 
734  NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
735  currentCoreLock.readLock().lock();
736  try {
737  if (null == currentCore) {
738  throw new NoOpenCoreException();
739  }
740  return currentCore.request(request);
741  } finally {
742  currentCoreLock.readLock().unlock();
743  }
744  }
745 
756  public int queryNumIndexedFiles() throws KeywordSearchModuleException, NoOpenCoreException {
757  currentCoreLock.readLock().lock();
758  try {
759  if (null == currentCore) {
760  throw new NoOpenCoreException();
761  }
762  try {
763  return currentCore.queryNumIndexedFiles();
764  } catch (SolrServerException ex) {
765  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
766  }
767  } finally {
768  currentCoreLock.readLock().unlock();
769  }
770  }
771 
781  public int queryNumIndexedChunks() throws KeywordSearchModuleException, NoOpenCoreException {
782  currentCoreLock.readLock().lock();
783  try {
784  if (null == currentCore) {
785  throw new NoOpenCoreException();
786  }
787  try {
788  return currentCore.queryNumIndexedChunks();
789  } catch (SolrServerException ex) {
790  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
791  }
792  } finally {
793  currentCoreLock.readLock().unlock();
794  }
795  }
796 
806  public int queryNumIndexedDocuments() throws KeywordSearchModuleException, NoOpenCoreException {
807  currentCoreLock.readLock().lock();
808  try {
809  if (null == currentCore) {
810  throw new NoOpenCoreException();
811  }
812  try {
813  return currentCore.queryNumIndexedDocuments();
814  } catch (SolrServerException ex) {
815  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
816  }
817  } finally {
818  currentCoreLock.readLock().unlock();
819  }
820  }
821 
832  public boolean queryIsIndexed(long contentID) throws KeywordSearchModuleException, NoOpenCoreException {
833  currentCoreLock.readLock().lock();
834  try {
835  if (null == currentCore) {
836  throw new NoOpenCoreException();
837  }
838  try {
839  return currentCore.queryIsIndexed(contentID);
840  } catch (SolrServerException ex) {
841  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
842  }
843 
844  } finally {
845  currentCoreLock.readLock().unlock();
846  }
847  }
848 
860  public int queryNumFileChunks(long fileID) throws KeywordSearchModuleException, NoOpenCoreException {
861  currentCoreLock.readLock().lock();
862  try {
863  if (null == currentCore) {
864  throw new NoOpenCoreException();
865  }
866  try {
867  return currentCore.queryNumFileChunks(fileID);
868  } catch (SolrServerException ex) {
869  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
870  }
871  } finally {
872  currentCoreLock.readLock().unlock();
873  }
874  }
875 
886  public QueryResponse query(SolrQuery sq) throws KeywordSearchModuleException, NoOpenCoreException {
887  currentCoreLock.readLock().lock();
888  try {
889  if (null == currentCore) {
890  throw new NoOpenCoreException();
891  }
892  try {
893  return currentCore.query(sq);
894  } catch (SolrServerException ex) {
895  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
896  }
897  } finally {
898  currentCoreLock.readLock().unlock();
899  }
900  }
901 
913  public QueryResponse query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
914  currentCoreLock.readLock().lock();
915  try {
916  if (null == currentCore) {
917  throw new NoOpenCoreException();
918  }
919  try {
920  return currentCore.query(sq, method);
921  } catch (SolrServerException ex) {
922  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
923  }
924  } finally {
925  currentCoreLock.readLock().unlock();
926  }
927  }
928 
939  public TermsResponse queryTerms(SolrQuery sq) throws KeywordSearchModuleException, NoOpenCoreException {
940  currentCoreLock.readLock().lock();
941  try {
942  if (null == currentCore) {
943  throw new NoOpenCoreException();
944  }
945  try {
946  return currentCore.queryTerms(sq);
947  } catch (SolrServerException ex) {
948  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
949  }
950  } finally {
951  currentCoreLock.readLock().unlock();
952  }
953  }
954 
964  public String getSolrContent(final Content content) throws NoOpenCoreException {
965  currentCoreLock.readLock().lock();
966  try {
967  if (null == currentCore) {
968  throw new NoOpenCoreException();
969  }
970  return currentCore.getSolrContent(content.getId(), 0);
971  } finally {
972  currentCoreLock.readLock().unlock();
973  }
974  }
975 
988  public String getSolrContent(final Content content, int chunkID) throws NoOpenCoreException {
989  currentCoreLock.readLock().lock();
990  try {
991  if (null == currentCore) {
992  throw new NoOpenCoreException();
993  }
994  return currentCore.getSolrContent(content.getId(), chunkID);
995  } finally {
996  currentCoreLock.readLock().unlock();
997  }
998  }
999 
1009  public String getSolrContent(final long objectID) throws NoOpenCoreException {
1010  currentCoreLock.readLock().lock();
1011  try {
1012  if (null == currentCore) {
1013  throw new NoOpenCoreException();
1014  }
1015  return currentCore.getSolrContent(objectID, 0);
1016  } finally {
1017  currentCoreLock.readLock().unlock();
1018  }
1019  }
1020 
1031  public String getSolrContent(final long objectID, final int chunkID) throws NoOpenCoreException {
1032  currentCoreLock.readLock().lock();
1033  try {
1034  if (null == currentCore) {
1035  throw new NoOpenCoreException();
1036  }
1037  return currentCore.getSolrContent(objectID, chunkID);
1038  } finally {
1039  currentCoreLock.readLock().unlock();
1040  }
1041  }
1042 
1048  public static Ingester getIngester() {
1049  return Ingester.getDefault();
1050  }
1051 
1061  public static String getChunkIdString(long parentID, int childID) {
1062  return Long.toString(parentID) + Server.CHUNK_ID_SEPARATOR + Integer.toString(childID);
1063  }
1064 
1078  private Core openCore(String coreName, File dataDir, CaseType caseType) throws KeywordSearchModuleException {
1079 
1080  try {
1081  if (!dataDir.exists()) {
1082  dataDir.mkdirs();
1083  }
1084 
1085  if (!this.isRunning()) {
1086  logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
1087  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
1088  }
1089 
1090  if (!coreIsLoaded(coreName)) {
1091  /*
1092  * The core either does not exist or it is not loaded. Make a
1093  * request that will cause the core to be created if it does not
1094  * exist or loaded if it already exists.
1095  */
1096 
1097  // In single user mode, if there is a core.properties file already,
1098  // we've hit a solr bug. Compensate by deleting it.
1099  if (caseType == CaseType.SINGLE_USER_CASE) {
1100  Path corePropertiesFile = Paths.get(solrFolder.toString(), SOLR, coreName, CORE_PROPERTIES);
1101  if (corePropertiesFile.toFile().exists()) {
1102  try {
1103  corePropertiesFile.toFile().delete();
1104  } catch (Exception ex) {
1105  logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
1106  }
1107  }
1108  }
1109 
1110  CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
1111  createCoreRequest.setDataDir(dataDir.getAbsolutePath());
1112  createCoreRequest.setCoreName(coreName);
1113  createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
1114  createCoreRequest.setIsLoadOnStartup(false);
1115  createCoreRequest.setIsTransient(true);
1116  currentSolrServer.request(createCoreRequest);
1117  }
1118 
1119  if (!coreIndexFolderExists(coreName)) {
1120  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
1121  }
1122 
1123  return new Core(coreName, caseType);
1124 
1125  } catch (SolrServerException | SolrException | IOException ex) {
1126  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
1127  }
1128  }
1129 
1138  void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
1139  CoreAdminRequest.getStatus(null, solrServer);
1140  }
1141 
1155  private boolean coreIsLoaded(String coreName) throws SolrServerException, IOException {
1156  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1157  return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1158  }
1159 
1170  private boolean coreIndexFolderExists(String coreName) throws SolrServerException, IOException {
1171  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1172  Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1173  if (null != dataDirPath) {
1174  File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1175  return indexDir.exists();
1176  } else {
1177  return false;
1178  }
1179  }
1180 
1181  class Core {
1182 
1183  // handle to the core in Solr
1184  private final String name;
1185 
1186  private final CaseType caseType;
1187 
1188  // the server to access a core needs to be built from a URL with the
1189  // core in it, and is only good for core-specific operations
1190  private final HttpSolrServer solrCore;
1191 
1192  private Core(String name, CaseType caseType) {
1193  this.name = name;
1194  this.caseType = caseType;
1195 
1196  this.solrCore = new HttpSolrServer(currentSolrServer.getBaseURL() + "/" + name);
1197 
1198  //TODO test these settings
1199  //solrCore.setSoTimeout(1000 * 60); // socket read timeout, make large enough so can index larger files
1200  //solrCore.setConnectionTimeout(1000);
1201  solrCore.setDefaultMaxConnectionsPerHost(2);
1202  solrCore.setMaxTotalConnections(5);
1203  solrCore.setFollowRedirects(false); // defaults to false
1204  // allowCompression defaults to false.
1205  // Server side must support gzip or deflate for this to have any effect.
1206  solrCore.setAllowCompression(true);
1207  solrCore.setMaxRetries(1); // defaults to 0. > 1 not recommended.
1208  solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
1209 
1210  }
1211 
1217  String getName() {
1218  return name;
1219  }
1220 
1221  private QueryResponse query(SolrQuery sq) throws SolrServerException {
1222  return solrCore.query(sq);
1223  }
1224 
1225  private NamedList<Object> request(SolrRequest request) throws SolrServerException {
1226  try {
1227  return solrCore.request(request);
1228  } catch (IOException e) {
1229  logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
1230  throw new SolrServerException(
1231  NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
1232  }
1233 
1234  }
1235 
1236  private QueryResponse query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException {
1237  return solrCore.query(sq, method);
1238  }
1239 
1240  private TermsResponse queryTerms(SolrQuery sq) throws SolrServerException {
1241  QueryResponse qres = solrCore.query(sq);
1242  return qres.getTermsResponse();
1243  }
1244 
1245  private void commit() throws SolrServerException {
1246  try {
1247  //commit and block
1248  solrCore.commit(true, true);
1249  } catch (IOException e) {
1250  logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1251  throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1252  }
1253  }
1254 
1255  void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1256  try {
1257  solrCore.add(doc);
1258  } catch (SolrServerException ex) {
1259  logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1260  throw new KeywordSearchModuleException(
1261  NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1262  } catch (IOException ex) {
1263  logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1264  throw new KeywordSearchModuleException(
1265  NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1266  }
1267  }
1268 
1277  private String getSolrContent(long contentID, int chunkID) {
1278  final SolrQuery q = new SolrQuery();
1279  q.setQuery("*:*");
1280  String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1281  if (chunkID != 0) {
1282  filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
1283  }
1284  q.addFilterQuery(filterQuery);
1285  q.setFields(Schema.TEXT.toString());
1286  try {
1287  // Get the first result.
1288  SolrDocumentList solrDocuments = solrCore.query(q).getResults();
1289 
1290  if (!solrDocuments.isEmpty()) {
1291  SolrDocument solrDocument = solrDocuments.get(0);
1292  if (solrDocument != null) {
1293  Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1294  if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
1295  {
1296  return fieldValues.toArray(new String[0])[0];
1297  } else // The indexed text for files has 2 values, the file name and the file content.
1298  // We return the file content value.
1299  {
1300  return fieldValues.toArray(new String[0])[1];
1301  }
1302  }
1303  }
1304  } catch (SolrServerException ex) {
1305  logger.log(Level.WARNING, "Error getting content from Solr", ex); //NON-NLS
1306  return null;
1307  }
1308 
1309  return null;
1310  }
1311 
1312  synchronized void close() throws KeywordSearchModuleException {
1313  // We only unload cores for "single-user" cases.
1314  if (this.caseType == CaseType.MULTI_USER_CASE) {
1315  return;
1316  }
1317 
1318  try {
1319  CoreAdminRequest.unloadCore(this.name, currentSolrServer);
1320  } catch (SolrServerException ex) {
1321  throw new KeywordSearchModuleException(
1322  NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1323  } catch (IOException ex) {
1324  throw new KeywordSearchModuleException(
1325  NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1326  }
1327  }
1328 
1338  private int queryNumIndexedFiles() throws SolrServerException {
1340  }
1341 
1350  private int queryNumIndexedChunks() throws SolrServerException {
1351  SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
1352  q.setRows(0);
1353  int numChunks = (int) query(q).getResults().getNumFound();
1354  return numChunks;
1355  }
1356 
1367  private int queryNumIndexedDocuments() throws SolrServerException {
1368  SolrQuery q = new SolrQuery("*:*");
1369  q.setRows(0);
1370  return (int) query(q).getResults().getNumFound();
1371  }
1372 
1382  private boolean queryIsIndexed(long contentID) throws SolrServerException {
1383  String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1384  SolrQuery q = new SolrQuery("*:*");
1385  q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1386  //q.setFields(Server.Schema.ID.toString());
1387  q.setRows(0);
1388  return (int) query(q).getResults().getNumFound() != 0;
1389  }
1390 
1402  private int queryNumFileChunks(long contentID) throws SolrServerException {
1403  String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1404  final SolrQuery q
1405  = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
1406  q.setRows(0);
1407  return (int) query(q).getResults().getNumFound();
1408  }
1409  }
1410 
1411  class ServerAction extends AbstractAction {
1412 
1413  private static final long serialVersionUID = 1L;
1414 
1415  @Override
1416  public void actionPerformed(ActionEvent e) {
1417  logger.log(Level.INFO, e.paramString().trim());
1418  }
1419  }
1420 
1424  class SolrServerNoPortException extends SocketException {
1425 
1426  private static final long serialVersionUID = 1L;
1427 
1431  private final int port;
1432 
1433  SolrServerNoPortException(int port) {
1434  super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1435  Server.PROPERTIES_CURRENT_SERVER_PORT));
1436  this.port = port;
1437  }
1438 
1439  int getPortNumber() {
1440  return port;
1441  }
1442  }
1443 }
String getSolrContent(final long objectID)
Definition: Server.java:1009
final ReentrantReadWriteLock currentCoreLock
Definition: Server.java:194
synchronized String mappedDriveToUNC(String inputPath)
boolean coreIsLoaded(String coreName)
Definition: Server.java:1155
void addServerActionListener(PropertyChangeListener l)
Definition: Server.java:251
static final String HL_ANALYZE_CHARS_UNLIMITED
Definition: Server.java:154
String getSolrContent(final Content content)
Definition: Server.java:964
boolean coreIndexFolderExists(String coreName)
Definition: Server.java:1170
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
Definition: Server.java:164
InputStreamPrinterThread errorRedirectThread
Definition: Server.java:198
QueryResponse query(SolrQuery sq)
Definition: Server.java:886
static String getConfigSetting(String moduleName, String settingName)
static synchronized String getJavaPath()
TermsResponse queryTerms(SolrQuery sq)
Definition: Server.java:939
boolean queryIsIndexed(long contentID)
Definition: Server.java:832
String getSolrContent(final Content content, int chunkID)
Definition: Server.java:988
String getSolrContent(final long objectID, final int chunkID)
Definition: Server.java:1031
synchronized static Logger getLogger(String name)
Definition: Logger.java:161
Core openCore(String coreName, File dataDir, CaseType caseType)
Definition: Server.java:1078
static synchronized long[] getJavaPIDs(String sigarSubQuery)
static String getChunkIdString(long parentID, int childID)
Definition: Server.java:1061
static boolean settingExists(String moduleName, String settingName)
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)
Definition: Server.java:913

Copyright © 2012-2016 Basis Technology. Generated on: Mon Jan 2 2017
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.