Autopsy  4.1
Graphical digital forensics platform for The Sleuth Kit and other tools.
Server.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2016 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import java.awt.event.ActionEvent;
22 import java.beans.PropertyChangeListener;
23 import java.io.BufferedReader;
24 import java.io.BufferedWriter;
25 import java.io.File;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.InputStream;
29 import java.io.InputStreamReader;
30 import java.io.OutputStream;
31 import java.io.OutputStreamWriter;
32 import java.net.ConnectException;
33 import java.net.ServerSocket;
34 import java.net.SocketException;
35 import java.nio.charset.Charset;
36 import java.nio.file.Files;
37 import java.nio.file.Path;
38 import java.nio.file.Paths;
39 import java.util.ArrayList;
40 import java.util.Arrays;
41 import java.util.Collection;
42 import java.util.List;
43 import java.util.concurrent.locks.ReentrantReadWriteLock;
44 import java.util.logging.Level;
45 import javax.swing.AbstractAction;
46 import org.apache.solr.client.solrj.SolrQuery;
47 import org.apache.solr.client.solrj.SolrRequest;
48 import org.apache.solr.client.solrj.SolrServerException;
49 import org.apache.solr.client.solrj.impl.HttpSolrServer;
50 import org.apache.solr.client.solrj.impl.XMLResponseParser;
51 import org.apache.solr.client.solrj.request.CoreAdminRequest;
52 import org.apache.solr.client.solrj.response.CoreAdminResponse;
53 import org.apache.solr.client.solrj.response.QueryResponse;
54 import org.apache.solr.client.solrj.response.TermsResponse;
55 import org.apache.solr.common.SolrDocument;
56 import org.apache.solr.common.SolrDocumentList;
57 import org.apache.solr.common.SolrException;
58 import org.apache.solr.common.SolrInputDocument;
59 import org.apache.solr.common.util.NamedList;
60 import org.openide.modules.InstalledFileLocator;
61 import org.openide.modules.Places;
62 import org.openide.util.NbBundle;
71 
76 public class Server {
77 
81  public static enum Schema {
82 
83  ID {
84  @Override
85  public String toString() {
86  return "id"; //NON-NLS
87  }
88  },
89  IMAGE_ID {
90  @Override
91  public String toString() {
92  return "image_id"; //NON-NLS
93  }
94  },
95  // This is not stored or index . it is copied to Text and Content_Ws
96  CONTENT {
97  @Override
98  public String toString() {
99  return "content"; //NON-NLS
100  }
101  },
102  CONTENT_STR {
103  @Override
104  public String toString() {
105  return "content_str"; //NON-NLS
106  }
107  },
108  TEXT {
109  @Override
110  public String toString() {
111  return "text"; //NON-NLS
112  }
113  },
114  CONTENT_WS {
115  @Override
116  public String toString() {
117  return "content_ws"; //NON-NLS
118  }
119  },
120  FILE_NAME {
121  @Override
122  public String toString() {
123  return "file_name"; //NON-NLS
124  }
125  },
126  // note that we no longer index this field
127  CTIME {
128  @Override
129  public String toString() {
130  return "ctime"; //NON-NLS
131  }
132  },
133  // note that we no longer index this field
134  ATIME {
135  @Override
136  public String toString() {
137  return "atime"; //NON-NLS
138  }
139  },
140  // note that we no longer index this field
141  MTIME {
142  @Override
143  public String toString() {
144  return "mtime"; //NON-NLS
145  }
146  },
147  // note that we no longer index this field
148  CRTIME {
149  @Override
150  public String toString() {
151  return "crtime"; //NON-NLS
152  }
153  },
154  NUM_CHUNKS {
155  @Override
156  public String toString() {
157  return "num_chunks"; //NON-NLS
158  }
159  },
160  CHUNK_SIZE {
161  @Override
162  public String toString() {
163  return "chunk_size"; //NON-NLS
164  }
165  }
166  };
167 
168  public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
169  //max content size we can send to Solr
170  public static final long MAX_CONTENT_SIZE = 1L * 31 * 1024 * 1024;
171  private static final Logger logger = Logger.getLogger(Server.class.getName());
172  public static final String CORE_EVT = "CORE_EVT"; //NON-NLS
173  @Deprecated
174  public static final char ID_CHUNK_SEP = '_';
175  public static final String CHUNK_ID_SEPARATOR = "_";
176  private String javaPath = "java";
177  public static final Charset DEFAULT_INDEXED_TEXT_CHARSET = Charset.forName("UTF-8");
178  private Process curSolrProcess = null;
179  private static final int MAX_SOLR_MEM_MB = 512; //TODO set dynamically based on avail. system resources
180  static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
181  static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
182  static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
183  private static final String KEY = "jjk#09s"; //NON-NLS
184  static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
185  static final int DEFAULT_SOLR_SERVER_PORT = 23232;
186  static final int DEFAULT_SOLR_STOP_PORT = 34343;
187  private int currentSolrServerPort = 0;
188  private int currentSolrStopPort = 0;
189  private static final boolean DEBUG = false;//(Version.getBuildType() == Version.Type.DEVELOPMENT);
190  private static final String SOLR = "solr";
191  private static final String CORE_PROPERTIES = "core.properties";
192 
193  public enum CORE_EVT_STATES {
194 
195  STOPPED, STARTED
196  };
197 
198  // A reference to the locally running Solr instance.
199  private final HttpSolrServer localSolrServer;
200 
201  // A reference to the Solr server we are currently connected to for the Case.
202  // This could be a local or remote server.
203  private HttpSolrServer currentSolrServer;
204 
205  private Core currentCore;
206  private final ReentrantReadWriteLock currentCoreLock;
207 
208  private final File solrFolder;
209  private Path solrHome;
210  private final ServerAction serverAction;
212 
217  Server() {
218  initSettings();
219 
220  this.localSolrServer = new HttpSolrServer("http://localhost:" + currentSolrServerPort + "/solr"); //NON-NLS
221  serverAction = new ServerAction();
222  solrFolder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS
223  javaPath = PlatformUtil.getJavaPath();
224 
225  solrHome = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "solr"); //NON-NLS
226  if (!solrHome.toFile().exists()) {
227  try {
228  Files.createDirectory(solrHome);
229  Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "solr.xml"), solrHome.resolve("solr.xml")); //NON-NLS
230  Files.copy(Paths.get(solrFolder.getAbsolutePath(), "solr", "zoo.cfg"), solrHome.resolve("zoo.cfg")); //NON-NLS
231  } catch (IOException ex) {
232  logger.log(Level.SEVERE, "Failed to create Solr home folder:", ex); //NON-NLS
233  }
234  }
235  currentCoreLock = new ReentrantReadWriteLock(true);
236 
237  logger.log(Level.INFO, "Created Server instance using Java at {0}", javaPath); //NON-NLS
238  }
239 
240  private void initSettings() {
241 
242  if (ModuleSettings.settingExists(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT)) {
243  try {
244  currentSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
245  } catch (NumberFormatException nfe) {
246  logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
247  currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
248  }
249  } else {
250  currentSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
251  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(currentSolrServerPort));
252  }
253 
254  if (ModuleSettings.settingExists(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT)) {
255  try {
256  currentSolrStopPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT));
257  } catch (NumberFormatException nfe) {
258  logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
259  currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
260  }
261  } else {
262  currentSolrStopPort = DEFAULT_SOLR_STOP_PORT;
263  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(currentSolrStopPort));
264  }
265  }
266 
267  @Override
268  public void finalize() throws java.lang.Throwable {
269  stop();
270  super.finalize();
271  }
272 
273  public void addServerActionListener(PropertyChangeListener l) {
274  serverAction.addPropertyChangeListener(l);
275  }
276 
277  int getCurrentSolrServerPort() {
278  return currentSolrServerPort;
279  }
280 
281  int getCurrentSolrStopPort() {
282  return currentSolrStopPort;
283  }
284 
288  private static class InputStreamPrinterThread extends Thread {
289 
290  InputStream stream;
291  OutputStream out;
292  volatile boolean doRun = true;
293 
294  InputStreamPrinterThread(InputStream stream, String type) {
295  this.stream = stream;
296  try {
297  final String log = Places.getUserDirectory().getAbsolutePath()
298  + File.separator + "var" + File.separator + "log" //NON-NLS
299  + File.separator + "solr.log." + type; //NON-NLS
300  File outputFile = new File(log.concat(".0"));
301  File first = new File(log.concat(".1"));
302  File second = new File(log.concat(".2"));
303  if (second.exists()) {
304  second.delete();
305  }
306  if (first.exists()) {
307  first.renameTo(second);
308  }
309  if (outputFile.exists()) {
310  outputFile.renameTo(first);
311  } else {
312  outputFile.createNewFile();
313  }
314  out = new FileOutputStream(outputFile);
315 
316  } catch (Exception ex) {
317  logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
318  }
319  }
320 
321  void stopRun() {
322  doRun = false;
323  }
324 
325  @Override
326  public void run() {
327 
328  try (InputStreamReader isr = new InputStreamReader(stream);
329  BufferedReader br = new BufferedReader(isr);
330  OutputStreamWriter osw = new OutputStreamWriter(out, PlatformUtil.getDefaultPlatformCharset());
331  BufferedWriter bw = new BufferedWriter(osw);) {
332 
333  String line = null;
334  while (doRun && (line = br.readLine()) != null) {
335  bw.write(line);
336  bw.newLine();
337  if (DEBUG) {
338  //flush buffers if dev version for debugging
339  bw.flush();
340  }
341  }
342  bw.flush();
343  } catch (IOException ex) {
344  logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
345  }
346  }
347  }
348 
358  private Process runSolrCommand(List<String> solrArguments) throws IOException {
359  final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + Integer.toString(MAX_SOLR_MEM_MB) + "m"; //NON-NLS
360  List<String> commandLine = new ArrayList<>();
361  commandLine.add(javaPath);
362  commandLine.add(MAX_SOLR_MEM_MB_PAR);
363  commandLine.add("-DSTOP.PORT=" + currentSolrStopPort); //NON-NLS
364  commandLine.add("-Djetty.port=" + currentSolrServerPort); //NON-NLS
365  commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
366  commandLine.add("-jar"); //NON-NLS
367  commandLine.add("start.jar"); //NON-NLS
368 
369  commandLine.addAll(solrArguments);
370 
371  ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
372  solrProcessBuilder.directory(solrFolder);
373 
374  // Redirect stdout and stderr to files to prevent blocking.
375  Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
376  solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
377 
378  Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
379  solrProcessBuilder.redirectError(solrStderrPath.toFile());
380 
381  logger.log(Level.INFO, "Running Solr command: {0}", solrProcessBuilder.command()); //NON-NLS
382  Process process = solrProcessBuilder.start();
383  logger.log(Level.INFO, "Finished running Solr command"); //NON-NLS
384  return process;
385  }
386 
392  List<Long> getSolrPIDs() {
393  List<Long> pids = new ArrayList<>();
394 
395  //NOTE: these needs to be in sync with process start string in start()
396  final String pidsQuery = "Args.*.eq=-DSTOP.KEY=" + KEY + ",Args.*.eq=start.jar"; //NON-NLS
397 
398  long[] pidsArr = PlatformUtil.getJavaPIDs(pidsQuery);
399  if (pidsArr != null) {
400  for (int i = 0; i < pidsArr.length; ++i) {
401  pids.add(pidsArr[i]);
402  }
403  }
404 
405  return pids;
406  }
407 
412  void killSolr() {
413  List<Long> solrPids = getSolrPIDs();
414  for (long pid : solrPids) {
415  logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
416  PlatformUtil.killProcess(pid);
417  }
418  }
419 
425  void start() throws KeywordSearchModuleException, SolrServerNoPortException {
426  if (isRunning()) {
427  // If a Solr server is running we stop it.
428  stop();
429  }
430 
431  if (!isPortAvailable(currentSolrServerPort)) {
432  // There is something already listening on our port. Let's see if
433  // this is from an earlier run that didn't successfully shut down
434  // and if so kill it.
435  final List<Long> pids = this.getSolrPIDs();
436 
437  // If the culprit listening on the port is not a Solr process
438  // we refuse to start.
439  if (pids.isEmpty()) {
440  throw new SolrServerNoPortException(currentSolrServerPort);
441  }
442 
443  // Ok, we've tried to stop it above but there still appears to be
444  // a Solr process listening on our port so we forcefully kill it.
445  killSolr();
446 
447  // If either of the ports are still in use after our attempt to kill
448  // previously running processes we give up and throw an exception.
449  if (!isPortAvailable(currentSolrServerPort)) {
450  throw new SolrServerNoPortException(currentSolrServerPort);
451  }
452  if (!isPortAvailable(currentSolrStopPort)) {
453  throw new SolrServerNoPortException(currentSolrStopPort);
454  }
455  }
456 
457  logger.log(Level.INFO, "Starting Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
458 
459  if (isPortAvailable(currentSolrServerPort)) {
460  logger.log(Level.INFO, "Port [{0}] available, starting Solr", currentSolrServerPort); //NON-NLS
461  try {
462  curSolrProcess = runSolrCommand(new ArrayList<>(
463  Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS
464  "-Dcollection.configName=AutopsyConfig"))); //NON-NLS
465 
466  try {
467  //block for 10 seconds, give time to fully start the process
468  //so if it's restarted solr operations can be resumed seamlessly
469  Thread.sleep(10 * 1000);
470  } catch (InterruptedException ex) {
471  logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
472  }
473 
474  final List<Long> pids = this.getSolrPIDs();
475  logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
476  } catch (SecurityException ex) {
477  logger.log(Level.SEVERE, "Could not start Solr process!", ex); //NON-NLS
478  throw new KeywordSearchModuleException(
479  NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
480  } catch (IOException ex) {
481  logger.log(Level.SEVERE, "Could not start Solr server process!", ex); //NON-NLS
482  throw new KeywordSearchModuleException(
483  NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
484  }
485  }
486  }
487 
493  static boolean isPortAvailable(int port) {
494  ServerSocket ss = null;
495  try {
496 
497  ss = new ServerSocket(port, 0, java.net.Inet4Address.getByName("localhost")); //NON-NLS
498  if (ss.isBound()) {
499  ss.setReuseAddress(true);
500  ss.close();
501  return true;
502  }
503 
504  } catch (IOException e) {
505  } finally {
506  if (ss != null) {
507  try {
508  ss.close();
509  } catch (IOException e) {
510  /*
511  * should not be thrown
512  */
513  }
514  }
515  }
516  return false;
517  }
518 
524  void changeSolrServerPort(int port) {
525  currentSolrServerPort = port;
526  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
527  }
528 
534  void changeSolrStopPort(int port) {
535  currentSolrStopPort = port;
536  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
537  }
538 
544  synchronized void stop() {
545 
546  try {
547  // Close any open core before stopping server
548  closeCore();
549  } catch (KeywordSearchModuleException e) {
550  logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
551  }
552 
553  try {
554  logger.log(Level.INFO, "Stopping Solr server from: {0}", solrFolder.getAbsolutePath()); //NON-NLS
555 
556  //try graceful shutdown
557  Process process = runSolrCommand(new ArrayList<>(Arrays.asList("--stop"))); //NON-NLS
558 
559  logger.log(Level.INFO, "Waiting for Solr server to stop"); //NON-NLS
560  process.waitFor();
561 
562  //if still running, forcefully stop it
563  if (curSolrProcess != null) {
564  curSolrProcess.destroy();
565  curSolrProcess = null;
566  }
567 
568  } catch (IOException | InterruptedException ex) {
569  logger.log(Level.WARNING, "Error while attempting to stop Solr server", ex);
570  } finally {
571  //stop Solr stream -> log redirect threads
572  try {
573  if (errorRedirectThread != null) {
574  errorRedirectThread.stopRun();
575  errorRedirectThread = null;
576  }
577  } finally {
578  //if still running, kill it
579  killSolr();
580  }
581 
582  logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
583  }
584  }
585 
593  synchronized boolean isRunning() throws KeywordSearchModuleException {
594  try {
595 
596  if (isPortAvailable(currentSolrServerPort)) {
597  return false;
598  }
599 
600  // making a status request here instead of just doing solrServer.ping(), because
601  // that doesn't work when there are no cores
602  //TODO handle timeout in cases when some other type of server on that port
603  CoreAdminRequest.getStatus(null, localSolrServer);
604 
605  logger.log(Level.INFO, "Solr server is running"); //NON-NLS
606  } catch (SolrServerException ex) {
607 
608  Throwable cause = ex.getRootCause();
609 
610  // TODO: check if SocketExceptions should actually happen (is
611  // probably caused by starting a connection as the server finishes
612  // shutting down)
613  if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
614  logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
615  return false;
616  } else {
617  throw new KeywordSearchModuleException(
618  NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
619  }
620  } catch (SolrException ex) {
621  // Just log 404 errors for now...
622  logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
623  return false;
624  } catch (IOException ex) {
625  throw new KeywordSearchModuleException(
626  NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
627  }
628 
629  return true;
630  }
631 
632  /*
633  * ** Convenience methods for use while we only open one case at a time ***
634  */
644  void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
645  currentCoreLock.writeLock().lock();
646  try {
647  currentCore = openCore(theCase, index);
648 
649  try {
650  // execute a test query. if it fails, an exception will be thrown
652  } catch (NoOpenCoreException ex) {
653  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
654  }
655 
656  serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
657  } finally {
658  currentCoreLock.writeLock().unlock();
659  }
660  }
661 
667  boolean coreIsOpen() {
668  currentCoreLock.readLock().lock();
669  try {
670  return (null != currentCore);
671  } finally {
672  currentCoreLock.readLock().unlock();
673  }
674  }
675 
676  Index getIndexInfo() throws NoOpenCoreException {
677  currentCoreLock.readLock().lock();
678  try {
679  if (null == currentCore) {
680  throw new NoOpenCoreException();
681  }
682  return currentCore.getIndexInfo();
683  } finally {
684  currentCoreLock.readLock().unlock();
685  }
686  }
687 
688  void closeCore() throws KeywordSearchModuleException {
689  currentCoreLock.writeLock().lock();
690  try {
691  if (null != currentCore) {
692  currentCore.close();
693  currentCore = null;
694  serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
695  }
696  } finally {
697  currentCoreLock.writeLock().unlock();
698  }
699  }
700 
701  void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException, NoOpenCoreException {
702  currentCoreLock.readLock().lock();
703  try {
704  if (null == currentCore) {
705  throw new NoOpenCoreException();
706  }
707  currentCore.addDocument(doc);
708  } finally {
709  currentCoreLock.readLock().unlock();
710  }
711  }
712 
722  @NbBundle.Messages({
723  "# {0} - core name", "Server.deleteCore.exception.msg=Failed to delete Solr core {0}",})
724  void deleteCore(String coreName) throws KeywordSearchServiceException {
725  /*
726  * Send a core unload request to the Solr server, with the parameters
727  * that request deleting the index and the instance directory
728  * (deleteInstanceDir removes everything related to the core, the index
729  * directory, the configuration files, etc.) set to true.
730  * NOTE: this method doesn't delete the actual Solr index directory. That is
731  * done as part of deleting case output directory.
732  */
733 
734  // check whether the core we are deleting is the currently open core
735  currentCoreLock.readLock().lock();
736  try {
737  if (null != currentCore) {
738  if (currentCore.getName().equals(coreName)) {
739  // close current core first
740  closeCore();
741  }
742  }
743  } catch (KeywordSearchModuleException ex) {
744  throw new KeywordSearchServiceException(NbBundle.getMessage(Server.class, "Server.close.exception.msg"), ex);
745  } finally {
746  currentCoreLock.readLock().unlock();
747  }
748 
749  try {
750  HttpSolrServer solrServer = new HttpSolrServer("http://" + UserPreferences.getIndexingServerHost() + ":" + UserPreferences.getIndexingServerPort() + "/solr"); //NON-NLS
751  connectToSolrServer(solrServer);
752  org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName, true, true, solrServer);
753  } catch (SolrServerException | IOException ex) {
754  throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
755  }
756  }
757 
758 
770  private Core openCore(Case theCase, Index index) throws KeywordSearchModuleException {
771 
772  try {
773  if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
774  currentSolrServer = this.localSolrServer;
775  } else {
776  String host = UserPreferences.getIndexingServerHost();
777  String port = UserPreferences.getIndexingServerPort();
778  currentSolrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
779  }
780  connectToSolrServer(currentSolrServer);
781 
782  } catch (SolrServerException | IOException ex) {
783  throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex);
784  }
785 
786  try {
787  File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
788  if (!dataDir.exists()) {
789  dataDir.mkdirs();
790  }
791 
792  if (!this.isRunning()) {
793  logger.log(Level.SEVERE, "Core create/open requested, but server not yet running"); //NON-NLS
794  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
795  }
796 
797  String coreName = index.getIndexName();
798  if (!coreIsLoaded(coreName)) {
799  /*
800  * The core either does not exist or it is not loaded. Make a
801  * request that will cause the core to be created if it does not
802  * exist or loaded if it already exists.
803  */
804 
805  // In single user mode, if there is a core.properties file already,
806  // we've hit a solr bug. Compensate by deleting it.
807  if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
808  Path corePropertiesFile = Paths.get(solrFolder.toString(), SOLR, coreName, CORE_PROPERTIES);
809  if (corePropertiesFile.toFile().exists()) {
810  try {
811  corePropertiesFile.toFile().delete();
812  } catch (Exception ex) {
813  logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
814  }
815  }
816  }
817 
818  CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
819  createCoreRequest.setDataDir(dataDir.getAbsolutePath());
820  createCoreRequest.setCoreName(coreName);
821  createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
822  createCoreRequest.setIsLoadOnStartup(false);
823  createCoreRequest.setIsTransient(true);
824  currentSolrServer.request(createCoreRequest);
825  }
826 
827  if (!coreIndexFolderExists(coreName)) {
828  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
829  }
830 
831  return new Core(coreName, theCase.getCaseType(), index);
832 
833  } catch (SolrServerException | SolrException | IOException ex) {
834  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
835  }
836  }
837 
843  void commit() throws SolrServerException, NoOpenCoreException {
844  currentCoreLock.readLock().lock();
845  try {
846  if (null == currentCore) {
847  throw new NoOpenCoreException();
848  }
849  currentCore.commit();
850  } finally {
851  currentCoreLock.readLock().unlock();
852  }
853  }
854 
855  NamedList<Object> request(SolrRequest request) throws SolrServerException, NoOpenCoreException {
856  currentCoreLock.readLock().lock();
857  try {
858  if (null == currentCore) {
859  throw new NoOpenCoreException();
860  }
861  return currentCore.request(request);
862  } finally {
863  currentCoreLock.readLock().unlock();
864  }
865  }
866 
877  public int queryNumIndexedFiles() throws KeywordSearchModuleException, NoOpenCoreException {
878  currentCoreLock.readLock().lock();
879  try {
880  if (null == currentCore) {
881  throw new NoOpenCoreException();
882  }
883  try {
884  return currentCore.queryNumIndexedFiles();
885  } catch (SolrServerException | IOException ex) {
886  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
887  }
888  } finally {
889  currentCoreLock.readLock().unlock();
890  }
891  }
892 
902  public int queryNumIndexedChunks() throws KeywordSearchModuleException, NoOpenCoreException {
903  currentCoreLock.readLock().lock();
904  try {
905  if (null == currentCore) {
906  throw new NoOpenCoreException();
907  }
908  try {
909  return currentCore.queryNumIndexedChunks();
910  } catch (SolrServerException | IOException ex) {
911  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
912  }
913  } finally {
914  currentCoreLock.readLock().unlock();
915  }
916  }
917 
927  public int queryNumIndexedDocuments() throws KeywordSearchModuleException, NoOpenCoreException {
928  currentCoreLock.readLock().lock();
929  try {
930  if (null == currentCore) {
931  throw new NoOpenCoreException();
932  }
933  try {
934  return currentCore.queryNumIndexedDocuments();
935  } catch (SolrServerException | IOException ex) {
936  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
937  }
938  } finally {
939  currentCoreLock.readLock().unlock();
940  }
941  }
942 
953  public boolean queryIsIndexed(long contentID) throws KeywordSearchModuleException, NoOpenCoreException {
954  currentCoreLock.readLock().lock();
955  try {
956  if (null == currentCore) {
957  throw new NoOpenCoreException();
958  }
959  try {
960  return currentCore.queryIsIndexed(contentID);
961  } catch (SolrServerException | IOException ex) {
962  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
963  }
964 
965  } finally {
966  currentCoreLock.readLock().unlock();
967  }
968  }
969 
982  public int queryNumFileChunks(long fileID) throws KeywordSearchModuleException, NoOpenCoreException {
983  currentCoreLock.readLock().lock();
984  try {
985  if (null == currentCore) {
986  throw new NoOpenCoreException();
987  }
988  try {
989  return currentCore.queryNumFileChunks(fileID);
990  } catch (SolrServerException | IOException ex) {
991  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
992  }
993  } finally {
994  currentCoreLock.readLock().unlock();
995  }
996  }
997 
1008  public QueryResponse query(SolrQuery sq) throws KeywordSearchModuleException, NoOpenCoreException, IOException {
1009  currentCoreLock.readLock().lock();
1010  try {
1011  if (null == currentCore) {
1012  throw new NoOpenCoreException();
1013  }
1014  try {
1015  return currentCore.query(sq);
1016  } catch (SolrServerException ex) {
1017  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
1018  }
1019  } finally {
1020  currentCoreLock.readLock().unlock();
1021  }
1022  }
1023 
1035  public QueryResponse query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
1036  currentCoreLock.readLock().lock();
1037  try {
1038  if (null == currentCore) {
1039  throw new NoOpenCoreException();
1040  }
1041  try {
1042  return currentCore.query(sq, method);
1043  } catch (SolrServerException | IOException ex) {
1044  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
1045  }
1046  } finally {
1047  currentCoreLock.readLock().unlock();
1048  }
1049  }
1050 
1061  public TermsResponse queryTerms(SolrQuery sq) throws KeywordSearchModuleException, NoOpenCoreException {
1062  currentCoreLock.readLock().lock();
1063  try {
1064  if (null == currentCore) {
1065  throw new NoOpenCoreException();
1066  }
1067  try {
1068  return currentCore.queryTerms(sq);
1069  } catch (SolrServerException | IOException ex) {
1070  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
1071  }
1072  } finally {
1073  currentCoreLock.readLock().unlock();
1074  }
1075  }
1076 
1086  public String getSolrContent(final Content content) throws NoOpenCoreException {
1087  currentCoreLock.readLock().lock();
1088  try {
1089  if (null == currentCore) {
1090  throw new NoOpenCoreException();
1091  }
1092  return currentCore.getSolrContent(content.getId(), 0);
1093  } finally {
1094  currentCoreLock.readLock().unlock();
1095  }
1096  }
1097 
1110  public String getSolrContent(final Content content, int chunkID) throws NoOpenCoreException {
1111  currentCoreLock.readLock().lock();
1112  try {
1113  if (null == currentCore) {
1114  throw new NoOpenCoreException();
1115  }
1116  return currentCore.getSolrContent(content.getId(), chunkID);
1117  } finally {
1118  currentCoreLock.readLock().unlock();
1119  }
1120  }
1121 
1131  public String getSolrContent(final long objectID) throws NoOpenCoreException {
1132  currentCoreLock.readLock().lock();
1133  try {
1134  if (null == currentCore) {
1135  throw new NoOpenCoreException();
1136  }
1137  return currentCore.getSolrContent(objectID, 0);
1138  } finally {
1139  currentCoreLock.readLock().unlock();
1140  }
1141  }
1142 
1153  public String getSolrContent(final long objectID, final int chunkID) throws NoOpenCoreException {
1154  currentCoreLock.readLock().lock();
1155  try {
1156  if (null == currentCore) {
1157  throw new NoOpenCoreException();
1158  }
1159  return currentCore.getSolrContent(objectID, chunkID);
1160  } finally {
1161  currentCoreLock.readLock().unlock();
1162  }
1163  }
1164 
1174  public static String getChunkIdString(long parentID, int childID) {
1175  return Long.toString(parentID) + Server.CHUNK_ID_SEPARATOR + Integer.toString(childID);
1176  }
1177 
1186  void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
1187  CoreAdminRequest.getStatus(null, solrServer);
1188  }
1189 
1203  private boolean coreIsLoaded(String coreName) throws SolrServerException, IOException {
1204  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1205  return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1206  }
1207 
1219  private boolean coreIndexFolderExists(String coreName) throws SolrServerException, IOException {
1220  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, currentSolrServer);
1221  Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1222  if (null != dataDirPath) {
1223  File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1224  return indexDir.exists();
1225  } else {
1226  return false;
1227  }
1228  }
1229 
1230  class Core {
1231 
1232  // handle to the core in Solr
1233  private final String name;
1234 
1235  private final CaseType caseType;
1236 
1237  private final Index textIndex;
1238 
1239  // the server to access a core needs to be built from a URL with the
1240  // core in it, and is only good for core-specific operations
1241  private final HttpSolrServer solrCore;
1242 
1243  private Core(String name, CaseType caseType, Index index) {
1244  this.name = name;
1245  this.caseType = caseType;
1246  this.textIndex = index;
1247 
1248  this.solrCore = new HttpSolrServer(currentSolrServer.getBaseURL() + "/" + name); //NON-NLS
1249 
1250  //TODO test these settings
1251  //solrCore.setSoTimeout(1000 * 60); // socket read timeout, make large enough so can index larger files
1252  //solrCore.setConnectionTimeout(1000);
1253  solrCore.setDefaultMaxConnectionsPerHost(2);
1254  solrCore.setMaxTotalConnections(5);
1255  solrCore.setFollowRedirects(false); // defaults to false
1256  // allowCompression defaults to false.
1257  // Server side must support gzip or deflate for this to have any effect.
1258  solrCore.setAllowCompression(true);
1259  solrCore.setParser(new XMLResponseParser()); // binary parser is used by default
1260 
1261  }
1262 
1268  String getName() {
1269  return name;
1270  }
1271 
1272  private Index getIndexInfo() {
1273  return this.textIndex;
1274  }
1275 
1276  private QueryResponse query(SolrQuery sq) throws SolrServerException, IOException {
1277  return solrCore.query(sq);
1278  }
1279 
1280  private NamedList<Object> request(SolrRequest request) throws SolrServerException {
1281  try {
1282  return solrCore.request(request);
1283  } catch (IOException e) {
1284  logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
1285  throw new SolrServerException(
1286  NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
1287  }
1288 
1289  }
1290 
1291  private QueryResponse query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException, IOException {
1292  return solrCore.query(sq, method);
1293  }
1294 
1295  private TermsResponse queryTerms(SolrQuery sq) throws SolrServerException, IOException {
1296  QueryResponse qres = solrCore.query(sq);
1297  return qres.getTermsResponse();
1298  }
1299 
1300  private void commit() throws SolrServerException {
1301  try {
1302  //commit and block
1303  solrCore.commit(true, true);
1304  } catch (IOException e) {
1305  logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
1306  throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
1307  }
1308  }
1309 
1310  void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
1311  try {
1312  solrCore.add(doc);
1313  } catch (SolrServerException ex) {
1314  logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1315  throw new KeywordSearchModuleException(
1316  NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg", doc.getField("id")), ex); //NON-NLS
1317  } catch (IOException ex) {
1318  logger.log(Level.SEVERE, "Could not add document to index via update handler: " + doc.getField("id"), ex); //NON-NLS
1319  throw new KeywordSearchModuleException(
1320  NbBundle.getMessage(this.getClass(), "Server.addDoc.exception.msg2", doc.getField("id")), ex); //NON-NLS
1321  }
1322  }
1323 
1332  private String getSolrContent(long contentID, int chunkID) {
1333  final SolrQuery q = new SolrQuery();
1334  q.setQuery("*:*");
1335  String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1336  if (chunkID != 0) {
1337  filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
1338  }
1339  q.addFilterQuery(filterQuery);
1340  q.setFields(Schema.TEXT.toString());
1341  try {
1342  // Get the first result.
1343  SolrDocumentList solrDocuments = solrCore.query(q).getResults();
1344 
1345  if (!solrDocuments.isEmpty()) {
1346  SolrDocument solrDocument = solrDocuments.get(0);
1347  if (solrDocument != null) {
1348  Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
1349  if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
1350  {
1351  return fieldValues.toArray(new String[0])[0];
1352  } else // The indexed text for files has 2 values, the file name and the file content.
1353  // We return the file content value.
1354  {
1355  return fieldValues.toArray(new String[0])[1];
1356  }
1357  }
1358  }
1359  } catch (SolrServerException ex) {
1360  logger.log(Level.WARNING, "Error getting content from Solr", ex); //NON-NLS
1361  return null;
1362  }
1363 
1364  return null;
1365  }
1366 
1367  synchronized void close() throws KeywordSearchModuleException {
1368  // We only unload cores for "single-user" cases.
1369  if (this.caseType == CaseType.MULTI_USER_CASE) {
1370  return;
1371  }
1372 
1373  try {
1374  CoreAdminRequest.unloadCore(this.name, currentSolrServer);
1375  } catch (SolrServerException ex) {
1376  throw new KeywordSearchModuleException(
1377  NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
1378  } catch (IOException ex) {
1379  throw new KeywordSearchModuleException(
1380  NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
1381  }
1382  }
1383 
1393  private int queryNumIndexedFiles() throws SolrServerException, IOException {
1395  }
1396 
1406  private int queryNumIndexedChunks() throws SolrServerException, IOException {
1407  SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
1408  q.setRows(0);
1409  int numChunks = (int) query(q).getResults().getNumFound();
1410  return numChunks;
1411  }
1412 
1423  private int queryNumIndexedDocuments() throws SolrServerException, IOException {
1424  SolrQuery q = new SolrQuery("*:*");
1425  q.setRows(0);
1426  return (int) query(q).getResults().getNumFound();
1427  }
1428 
1438  private boolean queryIsIndexed(long contentID) throws SolrServerException, IOException {
1439  String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1440  SolrQuery q = new SolrQuery("*:*");
1441  q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
1442  //q.setFields(Server.Schema.ID.toString());
1443  q.setRows(0);
1444  return (int) query(q).getResults().getNumFound() != 0;
1445  }
1446 
1458  private int queryNumFileChunks(long contentID) throws SolrServerException, IOException {
1459  String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
1460  final SolrQuery q
1461  = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
1462  q.setRows(0);
1463  return (int) query(q).getResults().getNumFound();
1464  }
1465  }
1466 
1467  class ServerAction extends AbstractAction {
1468 
1469  private static final long serialVersionUID = 1L;
1470 
1471  @Override
1472  public void actionPerformed(ActionEvent e) {
1473  logger.log(Level.INFO, e.paramString().trim());
1474  }
1475  }
1476 
1480  class SolrServerNoPortException extends SocketException {
1481 
1482  private static final long serialVersionUID = 1L;
1483 
1487  private final int port;
1488 
1489  SolrServerNoPortException(int port) {
1490  super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
1491  Server.PROPERTIES_CURRENT_SERVER_PORT));
1492  this.port = port;
1493  }
1494 
1495  int getPortNumber() {
1496  return port;
1497  }
1498  }
1499 }
String getSolrContent(final long objectID)
Definition: Server.java:1131
final ReentrantReadWriteLock currentCoreLock
Definition: Server.java:206
boolean coreIsLoaded(String coreName)
Definition: Server.java:1203
void addServerActionListener(PropertyChangeListener l)
Definition: Server.java:273
static final String HL_ANALYZE_CHARS_UNLIMITED
Definition: Server.java:168
Process runSolrCommand(List< String > solrArguments)
Definition: Server.java:358
String getSolrContent(final Content content)
Definition: Server.java:1086
boolean coreIndexFolderExists(String coreName)
Definition: Server.java:1219
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
Definition: Server.java:177
InputStreamPrinterThread errorRedirectThread
Definition: Server.java:211
QueryResponse query(SolrQuery sq)
Definition: Server.java:1008
static String getConfigSetting(String moduleName, String settingName)
static synchronized String getJavaPath()
TermsResponse queryTerms(SolrQuery sq)
Definition: Server.java:1061
boolean queryIsIndexed(long contentID)
Definition: Server.java:953
String getSolrContent(final Content content, int chunkID)
Definition: Server.java:1110
String getSolrContent(final long objectID, final int chunkID)
Definition: Server.java:1153
synchronized static Logger getLogger(String name)
Definition: Logger.java:161
static synchronized long[] getJavaPIDs(String sigarSubQuery)
Core openCore(Case theCase, Index index)
Definition: Server.java:770
static String getChunkIdString(long parentID, int childID)
Definition: Server.java:1174
static boolean settingExists(String moduleName, String settingName)
String toString(boolean preserveState)
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)
Definition: Server.java:1035

Copyright © 2012-2016 Basis Technology. Generated on: Mon Apr 24 2017
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.