Autopsy  4.19.0
Graphical digital forensics platform for The Sleuth Kit and other tools.
Server.java
Go to the documentation of this file.
1 /*
2  * Autopsy Forensic Browser
3  *
4  * Copyright 2011-2021 Basis Technology Corp.
5  * Contact: carrier <at> sleuthkit <dot> org
6  *
7  * Licensed under the Apache License, Version 2.0 (the "License");
8  * you may not use this file except in compliance with the License.
9  * You may obtain a copy of the License at
10  *
11  * http://www.apache.org/licenses/LICENSE-2.0
12  *
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  */
19 package org.sleuthkit.autopsy.keywordsearch;
20 
21 import com.google.common.util.concurrent.ThreadFactoryBuilder;
22 import java.awt.event.ActionEvent;
23 import java.beans.PropertyChangeListener;
24 import java.io.BufferedReader;
25 import java.io.BufferedWriter;
26 import java.io.File;
27 import java.io.FileOutputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.InputStreamReader;
31 import java.io.OutputStream;
32 import java.io.OutputStreamWriter;
33 import java.net.ConnectException;
34 import java.net.DatagramSocket;
35 import java.net.ServerSocket;
36 import java.net.SocketException;
37 import java.nio.charset.Charset;
38 import java.nio.file.Files;
39 import java.nio.file.OpenOption;
40 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
41 import java.nio.file.Path;
42 import java.nio.file.Paths;
43 import java.util.ArrayList;
44 import java.util.Arrays;
45 import java.util.Collections;
46 import java.util.Iterator;
47 import java.util.List;
48 import java.util.Random;
49 import java.util.concurrent.ScheduledThreadPoolExecutor;
50 import java.util.concurrent.TimeUnit;
51 import java.util.concurrent.locks.ReentrantReadWriteLock;
52 import java.util.logging.Level;
53 import javax.swing.AbstractAction;
54 import org.apache.commons.io.FileUtils;
55 import java.util.concurrent.TimeoutException;
56 import java.util.concurrent.atomic.AtomicBoolean;
57 import java.util.concurrent.atomic.AtomicInteger;
58 import java.util.stream.Collectors;
59 import static java.util.stream.Collectors.toList;
60 import org.apache.solr.client.solrj.SolrQuery;
61 import org.apache.solr.client.solrj.SolrRequest;
62 import org.apache.solr.client.solrj.SolrServerException;
63 import org.apache.solr.client.solrj.SolrClient;
64 import org.apache.solr.client.solrj.impl.HttpSolrClient;
65 import org.apache.solr.client.solrj.impl.CloudSolrClient;
66 import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
67 import org.apache.solr.client.solrj.impl.XMLResponseParser;
68 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
69 import org.apache.solr.client.solrj.response.CollectionAdminResponse;
70 import org.apache.solr.client.solrj.request.CoreAdminRequest;
71 import org.apache.solr.client.solrj.response.CoreAdminResponse;
72 import org.apache.solr.client.solrj.impl.BaseHttpSolrClient.RemoteSolrException;
73 import org.apache.solr.client.solrj.request.QueryRequest;
74 import org.apache.solr.client.solrj.response.QueryResponse;
75 import org.apache.solr.client.solrj.response.TermsResponse;
76 import org.apache.solr.client.solrj.response.TermsResponse.Term;
77 import org.apache.solr.common.SolrDocument;
78 import org.apache.solr.common.SolrDocumentList;
79 import org.apache.solr.common.SolrException;
80 import org.apache.solr.common.SolrInputDocument;
81 import org.apache.solr.common.util.NamedList;
82 import org.openide.modules.InstalledFileLocator;
83 import org.openide.modules.Places;
84 import org.openide.util.NbBundle;
85 import org.openide.windows.WindowManager;
102 import org.sleuthkit.datamodel.Content;
103 
108 public class Server {
109 
113  public static enum Schema {
114 
115  ID {
116  @Override
117  public String toString() {
118  return "id"; //NON-NLS
119  }
120  },
121  IMAGE_ID {
122  @Override
123  public String toString() {
124  return "image_id"; //NON-NLS
125  }
126  },
127  // This is not stored or indexed. it is copied to text by the schema
128  CONTENT {
129  @Override
130  public String toString() {
131  return "content"; //NON-NLS
132  }
133  },
134  // String representation for regular expression searching
135  CONTENT_STR {
136  @Override
137  public String toString() {
138  return "content_str"; //NON-NLS
139  }
140  },
141  // default search field. Populated by schema
142  TEXT {
143  @Override
144  public String toString() {
145  return "text"; //NON-NLS
146  }
147  },
148  // no longer populated. Was used for regular expression searching.
149  // Should not be used.
150  CONTENT_WS {
151  @Override
152  public String toString() {
153  return "content_ws"; //NON-NLS
154  }
155  },
156  CONTENT_JA {
157  @Override
158  public String toString() {
159  return "content_ja"; //NON-NLS
160  }
161  },
162  LANGUAGE {
163  @Override
164  public String toString() {
165  return "language"; //NON-NLS
166  }
167  },
168  FILE_NAME {
169  @Override
170  public String toString() {
171  return "file_name"; //NON-NLS
172  }
173  },
174  // note that we no longer store or index this field
175  CTIME {
176  @Override
177  public String toString() {
178  return "ctime"; //NON-NLS
179  }
180  },
181  // note that we no longer store or index this field
182  ATIME {
183  @Override
184  public String toString() {
185  return "atime"; //NON-NLS
186  }
187  },
188  // note that we no longer store or index this field
189  MTIME {
190  @Override
191  public String toString() {
192  return "mtime"; //NON-NLS
193  }
194  },
195  // note that we no longer store or index this field
196  CRTIME {
197  @Override
198  public String toString() {
199  return "crtime"; //NON-NLS
200  }
201  },
202  NUM_CHUNKS {
203  @Override
204  public String toString() {
205  return "num_chunks"; //NON-NLS
206  }
207  },
208  CHUNK_SIZE {
209  @Override
210  public String toString() {
211  return "chunk_size"; //NON-NLS
212  }
213  },
219  TERMFREQ {
220  @Override
221  public String toString() {
222  return "termfreq"; //NON-NLS
223  }
224  }
225  };
226 
227  public static final String HL_ANALYZE_CHARS_UNLIMITED = "500000"; //max 1MB in a chunk. use -1 for unlimited, but -1 option may not be supported (not documented)
228  //max content size we can send to Solr
229  public static final long MAX_CONTENT_SIZE = 1L * 31 * 1024 * 1024;
230  private static final Logger logger = Logger.getLogger(Server.class.getName());
231  public static final String CORE_EVT = "CORE_EVT"; //NON-NLS
232  @Deprecated
233  public static final char ID_CHUNK_SEP = '_';
234  public static final String CHUNK_ID_SEPARATOR = "_";
235  private String javaPath = "java";
236  public static final Charset DEFAULT_INDEXED_TEXT_CHARSET = Charset.forName("UTF-8");
237  private Process curSolrProcess = null;
238  static final String PROPERTIES_FILE = KeywordSearchSettings.MODULE_NAME;
239  static final String PROPERTIES_CURRENT_SERVER_PORT = "IndexingServerPort"; //NON-NLS
240  static final String PROPERTIES_CURRENT_STOP_PORT = "IndexingServerStopPort"; //NON-NLS
241  private static final String KEY = "jjk#09s"; //NON-NLS
242  static final String DEFAULT_SOLR_SERVER_HOST = "localhost"; //NON-NLS
243  static final int DEFAULT_SOLR_SERVER_PORT = 23232;
244  static final int DEFAULT_SOLR_STOP_PORT = 34343;
245  private int localSolrServerPort = 0;
246  private int localSolrStopPort = 0;
247  private File localSolrFolder;
248  private static final String SOLR = "solr";
249  private static final String CORE_PROPERTIES = "core.properties";
250  private static final boolean DEBUG = false;//(Version.getBuildType() == Version.Type.DEVELOPMENT);
251  private static final int NUM_COLLECTION_CREATION_RETRIES = 5;
252  private static final int NUM_EMBEDDED_SERVER_RETRIES = 12; // attempt to connect to embedded Solr server for 1 minute
253  private static final int EMBEDDED_SERVER_RETRY_WAIT_SEC = 5;
254 
255  public enum CORE_EVT_STATES {
256 
257  STOPPED, STARTED
258  };
259 
260  private enum SOLR_VERSION {
261 
262  SOLR8, SOLR4
263  };
264 
265  // A reference to the locally running Solr instance.
266  private HttpSolrClient localSolrServer = null;
267  private SOLR_VERSION localServerVersion = SOLR_VERSION.SOLR8; // start local Solr 8 by default
268 
269  // A reference to the remote/network running Solr instance.
270  private HttpSolrClient remoteSolrServer;
271 
272  private Collection currentCollection;
273  private final ReentrantReadWriteLock currentCoreLock;
274 
275  private final ServerAction serverAction;
277 
282  Server() {
283  initSettings();
284 
285  localSolrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr");
286 
287  serverAction = new ServerAction();
288  File solr8Folder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS
289  File solr4Folder = InstalledFileLocator.getDefault().locate("solr4", Server.class.getPackage().getName(), false); //NON-NLS
290 
291  // Figure out where Java is located. The Java home location
292  // will be passed as the SOLR_JAVA_HOME environment
293  // variable to the Solr script but it can be overridden by the user in
294  // either autopsy-solr.cmd or autopsy-solr-in.cmd.
295  javaPath = PlatformUtil.getJavaPath();
296 
297  Path solr8Home = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "solr"); //NON-NLS
298  try {
299  // Always copy the config files, as they may have changed. Otherwise potentially stale Solr configuration is being used.
300  if (!solr8Home.toFile().exists()) {
301  Files.createDirectory(solr8Home);
302  } else {
303  // delete the configsets directory as the Autopsy configset could have changed
304  FileUtil.deleteDir(solr8Home.resolve("configsets").toFile());
305  }
306  Files.copy(Paths.get(solr8Folder.getAbsolutePath(), "server", "solr", "solr.xml"), solr8Home.resolve("solr.xml"), REPLACE_EXISTING); //NON-NLS
307  Files.copy(Paths.get(solr8Folder.getAbsolutePath(), "server", "solr", "zoo.cfg"), solr8Home.resolve("zoo.cfg"), REPLACE_EXISTING); //NON-NLS
308  FileUtils.copyDirectory(Paths.get(solr8Folder.getAbsolutePath(), "server", "solr", "configsets").toFile(), solr8Home.resolve("configsets").toFile()); //NON-NLS
309  } catch (IOException ex) {
310  logger.log(Level.SEVERE, "Failed to create Solr 8 home folder:", ex); //NON-NLS
311  }
312 
313  Path solr4Home = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "solr4"); //NON-NLS
314  try {
315  // Always copy the config files, as they may have changed. Otherwise potentially stale Solr configuration is being used.
316  if (!solr4Home.toFile().exists()) {
317  Files.createDirectory(solr4Home);
318  }
319  Files.copy(Paths.get(solr4Folder.getAbsolutePath(), "solr", "solr.xml"), solr4Home.resolve("solr.xml"), REPLACE_EXISTING); //NON-NLS
320  Files.copy(Paths.get(solr4Folder.getAbsolutePath(), "solr", "zoo.cfg"), solr4Home.resolve("zoo.cfg"), REPLACE_EXISTING); //NON-NLS
321  } catch (IOException ex) {
322  logger.log(Level.SEVERE, "Failed to create Solr 4 home folder:", ex); //NON-NLS
323  }
324 
325  currentCoreLock = new ReentrantReadWriteLock(true);
326 
327  logger.log(Level.INFO, "Created Server instance using Java at {0}", javaPath); //NON-NLS
328  }
329 
330  private void initSettings() {
331 
332  if (ModuleSettings.settingExists(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT)) {
333  try {
334  localSolrServerPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT));
335  } catch (NumberFormatException nfe) {
336  logger.log(Level.WARNING, "Could not decode indexing server port, value was not a valid port number, using the default. ", nfe); //NON-NLS
337  localSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
338  }
339  } else {
340  localSolrServerPort = DEFAULT_SOLR_SERVER_PORT;
341  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(localSolrServerPort));
342  }
343 
344  if (ModuleSettings.settingExists(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT)) {
345  try {
346  localSolrStopPort = Integer.decode(ModuleSettings.getConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT));
347  } catch (NumberFormatException nfe) {
348  logger.log(Level.WARNING, "Could not decode indexing server stop port, value was not a valid port number, using default", nfe); //NON-NLS
349  localSolrStopPort = DEFAULT_SOLR_STOP_PORT;
350  }
351  } else {
352  localSolrStopPort = DEFAULT_SOLR_STOP_PORT;
353  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(localSolrStopPort));
354  }
355  }
356 
357  private HttpSolrClient getSolrClient(String solrUrl) {
358  int connectionTimeoutMs = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getConnectionTimeout();
359  return new HttpSolrClient.Builder(solrUrl)
360  .withSocketTimeout(connectionTimeoutMs)
361  .withConnectionTimeout(connectionTimeoutMs)
362  .withResponseParser(new XMLResponseParser())
363  .build();
364  }
365 
366  private ConcurrentUpdateSolrClient getConcurrentClient(String solrUrl) {
367  int numThreads = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getNumThreads();
368  int numDocs = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getDocumentsQueueSize();
369  int connectionTimeoutMs = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getConnectionTimeout();
370  logger.log(Level.INFO, "Creating new ConcurrentUpdateSolrClient: {0}", solrUrl); //NON-NLS
371  logger.log(Level.INFO, "Queue size = {0}, Number of threads = {1}, Connection Timeout (ms) = {2}", new Object[]{numDocs, numThreads, connectionTimeoutMs}); //NON-NLS
372  ConcurrentUpdateSolrClient client = new ConcurrentUpdateSolrClient.Builder(solrUrl)
373  .withQueueSize(numDocs)
374  .withThreadCount(numThreads)
375  .withSocketTimeout(connectionTimeoutMs)
376  .withConnectionTimeout(connectionTimeoutMs)
377  .withResponseParser(new XMLResponseParser())
378  .build();
379 
380  return client;
381  }
382 
383  private CloudSolrClient getCloudSolrClient(String host, String port, String defaultCollectionName) throws KeywordSearchModuleException {
384  List<String> solrServerList = getSolrServerList(host, port);
385  List<String> solrUrls = new ArrayList<>();
386  for (String server : solrServerList) {
387  solrUrls.add("http://" + server + "/solr");
388  logger.log(Level.INFO, "Using Solr server: {0}", server);
389  }
390 
391  logger.log(Level.INFO, "Creating new CloudSolrClient"); //NON-NLS
392  int connectionTimeoutMs = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getConnectionTimeout();
393  CloudSolrClient client = new CloudSolrClient.Builder(solrUrls)
394  .withConnectionTimeout(connectionTimeoutMs)
395  .withSocketTimeout(connectionTimeoutMs)
396  .withResponseParser(new XMLResponseParser())
397  .build();
398  if (!defaultCollectionName.isEmpty()) {
399  client.setDefaultCollection(defaultCollectionName);
400  }
401  client.connect();
402  return client;
403  }
404 
405  @Override
406  public void finalize() throws java.lang.Throwable {
407  stop();
408  super.finalize();
409  }
410 
411  public void addServerActionListener(PropertyChangeListener l) {
412  serverAction.addPropertyChangeListener(l);
413  }
414 
415  int getLocalSolrServerPort() {
416  return localSolrServerPort;
417  }
418 
419  int getLocalSolrStopPort() {
420  return localSolrStopPort;
421  }
422 
426  private static class InputStreamPrinterThread extends Thread {
427 
428  InputStream stream;
429  OutputStream out;
430  volatile boolean doRun = true;
431 
432  InputStreamPrinterThread(InputStream stream, String type) {
433  this.stream = stream;
434  try {
435  final String log = Places.getUserDirectory().getAbsolutePath()
436  + File.separator + "var" + File.separator + "log" //NON-NLS
437  + File.separator + "solr.log." + type; //NON-NLS
438  File outputFile = new File(log.concat(".0"));
439  File first = new File(log.concat(".1"));
440  File second = new File(log.concat(".2"));
441  if (second.exists()) {
442  second.delete();
443  }
444  if (first.exists()) {
445  first.renameTo(second);
446  }
447  if (outputFile.exists()) {
448  outputFile.renameTo(first);
449  } else {
450  outputFile.createNewFile();
451  }
452  out = new FileOutputStream(outputFile);
453 
454  } catch (Exception ex) {
455  logger.log(Level.WARNING, "Failed to create solr log file", ex); //NON-NLS
456  }
457  }
458 
459  void stopRun() {
460  doRun = false;
461  }
462 
463  @Override
464  public void run() {
465 
466  try (InputStreamReader isr = new InputStreamReader(stream);
467  BufferedReader br = new BufferedReader(isr);
468  OutputStreamWriter osw = new OutputStreamWriter(out, PlatformUtil.getDefaultPlatformCharset());
469  BufferedWriter bw = new BufferedWriter(osw);) {
470 
471  String line = null;
472  while (doRun && (line = br.readLine()) != null) {
473  bw.write(line);
474  bw.newLine();
475  if (DEBUG) {
476  //flush buffers if dev version for debugging
477  bw.flush();
478  }
479  }
480  bw.flush();
481  } catch (IOException ex) {
482  logger.log(Level.SEVERE, "Error redirecting Solr output stream", ex); //NON-NLS
483  }
484  }
485  }
486 
496  private Process runLocalSolr8ControlCommand(List<String> solrArguments) throws IOException {
497  final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + UserPreferences.getMaxSolrVMSize() + "m"; //NON-NLS
498 
499  // This is our customized version of the Solr batch script to start/stop Solr.
500  File solr8Folder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS
501  Path solr8CmdPath;
503  solr8CmdPath = Paths.get(solr8Folder.getAbsolutePath(), "bin", "autopsy-solr.cmd"); //NON-NLS
504  } else {
505  solr8CmdPath = Paths.get(solr8Folder.getAbsolutePath(), "bin", "autopsy-solr"); //NON-NLS
506  }
507  Path solr8Home = Paths.get(PlatformUtil.getUserDirectory().getAbsolutePath(), "solr"); //NON-NLS
508 
509  List<String> commandLine = new ArrayList<>();
510  commandLine.add(solr8CmdPath.toString());
511  commandLine.addAll(solrArguments);
512 
513  ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
514  solrProcessBuilder.directory(solr8Folder);
515 
516  // Redirect stdout and stderr to files to prevent blocking.
517  Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
518  solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
519 
520  Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
521  solrProcessBuilder.redirectError(solrStderrPath.toFile());
522 
523  // get the path to the JRE folder. That's what Solr needs as SOLR_JAVA_HOME
524  String jreFolderPath = new File(javaPath).getParentFile().getParentFile().getAbsolutePath();
525 
526  solrProcessBuilder.environment().put("SOLR_JAVA_HOME", jreFolderPath); // NON-NLS
527  solrProcessBuilder.environment().put("SOLR_HOME", solr8Home.toString()); // NON-NLS
528  solrProcessBuilder.environment().put("STOP_KEY", KEY); // NON-NLS
529  solrProcessBuilder.environment().put("SOLR_JAVA_MEM", MAX_SOLR_MEM_MB_PAR); // NON-NLS
530  logger.log(Level.INFO, "Setting Solr 8 directory: {0}", solr8Folder.toString()); //NON-NLS
531  logger.log(Level.INFO, "Running Solr 8 command: {0} from {1}", new Object[]{solrProcessBuilder.command(), solr8Folder.toString()}); //NON-NLS
532  Process process = solrProcessBuilder.start();
533  logger.log(Level.INFO, "Finished running Solr 8 command"); //NON-NLS
534  return process;
535  }
536 
546  private Process runLocalSolr4ControlCommand(List<String> solrArguments) throws IOException {
547  final String MAX_SOLR_MEM_MB_PAR = "-Xmx" + UserPreferences.getMaxSolrVMSize() + "m"; //NON-NLS
548  File solr4Folder = InstalledFileLocator.getDefault().locate("solr4", Server.class.getPackage().getName(), false); //NON-NLS
549 
550  List<String> commandLine = new ArrayList<>();
551  commandLine.add(javaPath);
552  commandLine.add(MAX_SOLR_MEM_MB_PAR);
553  commandLine.add("-DSTOP.PORT=" + localSolrStopPort); //NON-NLS
554  commandLine.add("-Djetty.port=" + localSolrServerPort); //NON-NLS
555  commandLine.add("-DSTOP.KEY=" + KEY); //NON-NLS
556  commandLine.add("-jar"); //NON-NLS
557  commandLine.add("start.jar"); //NON-NLS
558 
559  commandLine.addAll(solrArguments);
560 
561  ProcessBuilder solrProcessBuilder = new ProcessBuilder(commandLine);
562  solrProcessBuilder.directory(solr4Folder);
563 
564  // Redirect stdout and stderr to files to prevent blocking.
565  Path solrStdoutPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stdout"); //NON-NLS
566  solrProcessBuilder.redirectOutput(solrStdoutPath.toFile());
567 
568  Path solrStderrPath = Paths.get(Places.getUserDirectory().getAbsolutePath(), "var", "log", "solr.log.stderr"); //NON-NLS
569  solrProcessBuilder.redirectError(solrStderrPath.toFile());
570 
571  logger.log(Level.INFO, "Running Solr 4 command: {0}", solrProcessBuilder.command()); //NON-NLS
572  Process process = solrProcessBuilder.start();
573  logger.log(Level.INFO, "Finished running Solr 4 command"); //NON-NLS
574  return process;
575  }
576 
582  List<Long> getSolrPIDs() {
583  List<Long> pids = new ArrayList<>();
584 
585  //NOTE: these needs to be in sync with process start string in start()
586  final String pidsQuery = "Args.*.eq=-DSTOP.KEY=" + KEY + ",Args.*.eq=start.jar"; //NON-NLS
587 
588  long[] pidsArr = PlatformUtil.getJavaPIDs(pidsQuery);
589  if (pidsArr != null) {
590  for (int i = 0; i < pidsArr.length; ++i) {
591  pids.add(pidsArr[i]);
592  }
593  }
594 
595  return pids;
596  }
597 
602  void killSolr() {
603  List<Long> solrPids = getSolrPIDs();
604  for (long pid : solrPids) {
605  logger.log(Level.INFO, "Trying to kill old Solr process, PID: {0}", pid); //NON-NLS
606  PlatformUtil.killProcess(pid);
607  }
608  }
609 
610  void start() throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException {
611  startLocalSolr(SOLR_VERSION.SOLR8);
612  }
613 
614  private void configureSolrConnection(Case theCase, Index index) throws KeywordSearchModuleException, SolrServerNoPortException {
615 
616  try {
617  if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
618 
619  // makes sure the proper local Solr server is running
620  if (IndexFinder.getCurrentSolrVersion().equals(index.getSolrVersion())) {
621  startLocalSolr(SOLR_VERSION.SOLR8);
622  } else {
623  startLocalSolr(SOLR_VERSION.SOLR4);
624  }
625 
626  // check if the local Solr server is running
627  if (!this.isLocalSolrRunning()) {
628  logger.log(Level.SEVERE, "Local Solr server is not running"); //NON-NLS
629  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
630  }
631  } else {
632  // create SolrJ client to connect to remore Solr server
633  remoteSolrServer = configureMultiUserConnection(theCase, index, "");
634 
635  // test the connection
636  connectToSolrServer(remoteSolrServer);
637  }
638  } catch (SolrServerException | IOException ex) {
639  throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg", ex.getLocalizedMessage()), ex);
640  }
641  }
642 
657  private HttpSolrClient configureMultiUserConnection(Case theCase, Index index, String name) throws KeywordSearchModuleException {
658 
659  // read Solr connection info from user preferences, unless "solrserver.txt" is present
660  IndexingServerProperties properties = getMultiUserServerProperties(theCase.getCaseDirectory());
661  if (properties.host.isEmpty() || properties.port.isEmpty()) {
662  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.connectionInfoMissing.exception.msg", index.getSolrVersion()));
663  }
664  String solrUrl = "http://" + properties.host + ":" + properties.port + "/solr";
665 
666  if (!name.isEmpty()) {
667  solrUrl = solrUrl + "/" + name;
668  }
669 
670  // create SolrJ client to connect to remore Solr server
671  return getSolrClient(solrUrl);
672  }
673 
679  @NbBundle.Messages({
680  "Server.status.failed.msg=Local Solr server did not respond to status request. This may be because the server failed to start or is taking too long to initialize.",})
681  synchronized void startLocalSolr(SOLR_VERSION version) throws KeywordSearchModuleException, SolrServerNoPortException, SolrServerException {
682 
683  logger.log(Level.INFO, "Starting local Solr " + version + " server"); //NON-NLS
684  if (version == SOLR_VERSION.SOLR8) {
685  localSolrFolder = InstalledFileLocator.getDefault().locate("solr", Server.class.getPackage().getName(), false); //NON-NLS
686  } else {
687  // solr4
688  localSolrFolder = InstalledFileLocator.getDefault().locate("solr4", Server.class.getPackage().getName(), false); //NON-NLS
689  }
690 
691  if (isLocalSolrRunning()) {
692  if (localServerVersion.equals(version)) {
693  // this version of local server is already running
694  logger.log(Level.INFO, "Local Solr " + version + " server is already running"); //NON-NLS
695  return;
696  } else {
697  // wrong version of local server is running, stop it
698  stop();
699  }
700  }
701 
702  // set which version of local server is currently running
703  localServerVersion = version;
704 
705  if (!isPortAvailable(localSolrServerPort)) {
706  // There is something already listening on our port. Let's see if
707  // this is from an earlier run that didn't successfully shut down
708  // and if so kill it.
709  final List<Long> pids = this.getSolrPIDs();
710 
711  // If the culprit listening on the port is not a Solr process
712  // we refuse to start.
713  if (pids.isEmpty()) {
714  throw new SolrServerNoPortException(localSolrServerPort);
715  }
716 
717  // Ok, we've tried to stop it above but there still appears to be
718  // a Solr process listening on our port so we forcefully kill it.
719  killSolr();
720 
721  // If either of the ports are still in use after our attempt to kill
722  // previously running processes we give up and throw an exception.
723  if (!isPortAvailable(localSolrServerPort)) {
724  throw new SolrServerNoPortException(localSolrServerPort);
725  }
726  if (!isPortAvailable(localSolrStopPort)) {
727  throw new SolrServerNoPortException(localSolrStopPort);
728  }
729  }
730 
731  if (isPortAvailable(localSolrServerPort)) {
732  logger.log(Level.INFO, "Port [{0}] available, starting Solr", localSolrServerPort); //NON-NLS
733  try {
734  if (version == SOLR_VERSION.SOLR8) {
735  logger.log(Level.INFO, "Starting Solr 8 server"); //NON-NLS
736  curSolrProcess = runLocalSolr8ControlCommand(new ArrayList<>(Arrays.asList("start", "-p", //NON-NLS
737  Integer.toString(localSolrServerPort)))); //NON-NLS
738  } else {
739  // solr4
740  logger.log(Level.INFO, "Starting Solr 4 server"); //NON-NLS
741  curSolrProcess = runLocalSolr4ControlCommand(new ArrayList<>(
742  Arrays.asList("-Dbootstrap_confdir=../solr/configsets/AutopsyConfig/conf", //NON-NLS
743  "-Dcollection.configName=AutopsyConfig"))); //NON-NLS
744  }
745 
746  // Wait for the Solr server to start and respond to a statusRequest request.
747  for (int numRetries = 0; numRetries < NUM_EMBEDDED_SERVER_RETRIES; numRetries++) {
748  if (isLocalSolrRunning()) {
749  final List<Long> pids = this.getSolrPIDs();
750  logger.log(Level.INFO, "New Solr process PID: {0}", pids); //NON-NLS
751  return;
752  }
753 
754  // Local Solr server did not respond so we sleep for
755  // 5 seconds before trying again.
756  try {
757  TimeUnit.SECONDS.sleep(EMBEDDED_SERVER_RETRY_WAIT_SEC);
758  } catch (InterruptedException ex) {
759  logger.log(Level.WARNING, "Timer interrupted"); //NON-NLS
760  }
761  }
762 
763  // If we get here the Solr server has not responded to connection
764  // attempts in a timely fashion.
765  logger.log(Level.WARNING, "Local Solr server failed to respond to status requests.");
766  WindowManager.getDefault().invokeWhenUIReady(new Runnable() {
767  @Override
768  public void run() {
769  MessageNotifyUtil.Notify.error(
770  NbBundle.getMessage(this.getClass(), "Installer.errorInitKsmMsg"),
771  Bundle.Server_status_failed_msg());
772  }
773  });
774  } catch (SecurityException ex) {
775  throw new KeywordSearchModuleException(
776  NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg"), ex);
777  } catch (IOException ex) {
778  throw new KeywordSearchModuleException(
779  NbBundle.getMessage(this.getClass(), "Server.start.exception.cantStartSolr.msg2"), ex);
780  }
781  }
782  }
783 
790  static boolean isPortAvailable(int port) {
791  // implementation taken from https://stackoverflow.com/a/435579
792  if (port < 1 || port > 65535) {
793  throw new IllegalArgumentException("Invalid start port: " + port);
794  }
795 
796  ServerSocket ss = null;
797  DatagramSocket ds = null;
798  try {
799  ss = new ServerSocket(port);
800  ss.setReuseAddress(true);
801  ds = new DatagramSocket(port);
802  ds.setReuseAddress(true);
803  return true;
804  } catch (IOException e) {
805  } finally {
806  if (ds != null) {
807  ds.close();
808  }
809 
810  if (ss != null) {
811  try {
812  ss.close();
813  } catch (IOException e) {
814  /* should not be thrown */
815  }
816  }
817  }
818 
819  return false;
820  }
821 
822 
828  void changeSolrServerPort(int port) {
829  localSolrServerPort = port;
830  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_SERVER_PORT, String.valueOf(port));
831  }
832 
838  void changeSolrStopPort(int port) {
839  localSolrStopPort = port;
840  ModuleSettings.setConfigSetting(PROPERTIES_FILE, PROPERTIES_CURRENT_STOP_PORT, String.valueOf(port));
841  }
842 
848  synchronized void stop() {
849 
850  try {
851  // Close any open core before stopping server
852  closeCore();
853  } catch (KeywordSearchModuleException e) {
854  logger.log(Level.WARNING, "Failed to close core: ", e); //NON-NLS
855  }
856 
857  stopLocalSolr();
858  }
859 
863  private void stopLocalSolr() {
864  try {
865  //try graceful shutdown
866  Process process;
867  if (localServerVersion == SOLR_VERSION.SOLR8) {
868  logger.log(Level.INFO, "Stopping Solr 8 server"); //NON-NLS
869  process = runLocalSolr8ControlCommand(new ArrayList<>(Arrays.asList("stop", "-k", KEY, "-p", Integer.toString(localSolrServerPort)))); //NON-NLS
870  } else {
871  // solr 4
872  logger.log(Level.INFO, "Stopping Solr 4 server"); //NON-NLS
873  process = runLocalSolr4ControlCommand(new ArrayList<>(Arrays.asList("--stop"))); //NON-NLS
874  }
875 
876  logger.log(Level.INFO, "Waiting for Solr server to stop"); //NON-NLS
877  process.waitFor();
878 
879  //if still running, forcefully stop it
880  if (curSolrProcess != null) {
881  curSolrProcess.destroy();
882  curSolrProcess = null;
883  }
884 
885  } catch (IOException | InterruptedException ex) {
886  logger.log(Level.WARNING, "Error while attempting to stop Solr server", ex);
887  } finally {
888  //stop Solr stream -> log redirect threads
889  try {
890  if (errorRedirectThread != null) {
891  errorRedirectThread.stopRun();
892  errorRedirectThread = null;
893  }
894  } finally {
895  //if still running, kill it
896  killSolr();
897  }
898 
899  logger.log(Level.INFO, "Finished stopping Solr server"); //NON-NLS
900  }
901  }
902 
910  synchronized boolean isLocalSolrRunning() throws KeywordSearchModuleException {
911  try {
912 
913  if (isPortAvailable(localSolrServerPort)) {
914  return false;
915  }
916 
917  // making a statusRequest request here instead of just doing solrServer.ping(), because
918  // that doesn't work when there are no cores
919  //TODO handle timeout in cases when some other type of server on that port
921 
922  logger.log(Level.INFO, "Solr server is running"); //NON-NLS
923  } catch (SolrServerException ex) {
924 
925  Throwable cause = ex.getRootCause();
926 
927  // TODO: check if SocketExceptions should actually happen (is
928  // probably caused by starting a connection as the server finishes
929  // shutting down)
930  if (cause instanceof ConnectException || cause instanceof SocketException) { //|| cause instanceof NoHttpResponseException) {
931  logger.log(Level.INFO, "Solr server is not running, cause: {0}", cause.getMessage()); //NON-NLS
932  return false;
933  } else {
934  throw new KeywordSearchModuleException(
935  NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg"), ex);
936  }
937  } catch (SolrException ex) {
938  // Just log 404 errors for now...
939  logger.log(Level.INFO, "Solr server is not running", ex); //NON-NLS
940  return false;
941  } catch (IOException ex) {
942  throw new KeywordSearchModuleException(
943  NbBundle.getMessage(this.getClass(), "Server.isRunning.exception.errCheckSolrRunning.msg2"), ex);
944  }
945 
946  return true;
947  }
948 
949  /*
950  * ** Convenience methods for use while we only open one case at a time ***
951  */
961  void openCoreForCase(Case theCase, Index index) throws KeywordSearchModuleException {
962  currentCoreLock.writeLock().lock();
963  try {
964  currentCollection = openCore(theCase, index);
965 
966  try {
967  // execute a test query. if it fails, an exception will be thrown
969  } catch (NoOpenCoreException ex) {
970  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
971  }
972 
973  serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STARTED);
974  } finally {
975  currentCoreLock.writeLock().unlock();
976  }
977  }
978 
984  boolean coreIsOpen() {
985  currentCoreLock.readLock().lock();
986  try {
987  return (null != currentCollection);
988  } finally {
989  currentCoreLock.readLock().unlock();
990  }
991  }
992 
993  Index getIndexInfo() throws NoOpenCoreException {
994  currentCoreLock.readLock().lock();
995  try {
996  if (null == currentCollection) {
997  throw new NoOpenCoreException();
998  }
999  return currentCollection.getIndexInfo();
1000  } finally {
1001  currentCoreLock.readLock().unlock();
1002  }
1003  }
1004 
1005  void closeCore() throws KeywordSearchModuleException {
1006  currentCoreLock.writeLock().lock();
1007  try {
1008  if (null != currentCollection) {
1009  currentCollection.close();
1010  serverAction.putValue(CORE_EVT, CORE_EVT_STATES.STOPPED);
1011  }
1012  } finally {
1013  currentCollection = null;
1014  currentCoreLock.writeLock().unlock();
1015  }
1016  }
1017 
1018  void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException, NoOpenCoreException {
1019  currentCoreLock.readLock().lock();
1020  try {
1021  if (null == currentCollection) {
1022  throw new NoOpenCoreException();
1023  }
1024  TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Index chunk");
1025  currentCollection.addDocument(doc);
1026  HealthMonitor.submitTimingMetric(metric);
1027  } finally {
1028  currentCoreLock.readLock().unlock();
1029  }
1030  }
1031 
1040  @NbBundle.Messages({
1041  "# {0} - colelction name", "Server.deleteCore.exception.msg=Failed to delete Solr colelction {0}",})
1042  void deleteCollection(String coreName, CaseMetadata metadata) throws KeywordSearchServiceException, KeywordSearchModuleException {
1043  try {
1044  HttpSolrClient solrServer;
1045  if (metadata.getCaseType() == CaseType.SINGLE_USER_CASE) {
1046  solrServer = getSolrClient("http://localhost:" + localSolrServerPort + "/solr"); //NON-NLS
1047  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, solrServer);
1048  if (null != response.getCoreStatus(coreName).get("instanceDir")) { //NON-NLS
1049  /*
1050  * Send a core unload request to the Solr server, with the
1051  * parameter set that request deleting the index and the
1052  * instance directory (deleteInstanceDir = true). Note that
1053  * this removes everything related to the core on the server
1054  * (the index directory, the configuration files, etc.), but
1055  * does not delete the actual Solr text index because it is
1056  * currently stored in the case directory.
1057  */
1058  org.apache.solr.client.solrj.request.CoreAdminRequest.unloadCore(coreName, true, true, solrServer);
1059  }
1060  } else {
1061  IndexingServerProperties properties = getMultiUserServerProperties(metadata.getCaseDirectory());
1062  solrServer = getSolrClient("http://" + properties.getHost() + ":" + properties.getPort() + "/solr");
1063  connectToSolrServer(solrServer);
1064 
1065  CollectionAdminRequest.Delete deleteCollectionRequest = CollectionAdminRequest.deleteCollection(coreName);
1066  CollectionAdminResponse response = deleteCollectionRequest.process(solrServer);
1067  if (response.isSuccess()) {
1068  logger.log(Level.INFO, "Deleted collection {0}", coreName); //NON-NLS
1069  } else {
1070  logger.log(Level.WARNING, "Unable to delete collection {0}", coreName); //NON-NLS
1071  }
1072  }
1073  } catch (SolrServerException | IOException ex) {
1074  // We will get a RemoteSolrException with cause == null and detailsMessage
1075  // == "Already closed" if the core is not loaded. This is not an error in this scenario.
1076  if (!ex.getMessage().equals("Already closed")) { // NON-NLS
1077  throw new KeywordSearchServiceException(Bundle.Server_deleteCore_exception_msg(coreName), ex);
1078  }
1079  }
1080  }
1081 
1093  @NbBundle.Messages({
1094  "Server.exceptionMessage.unableToCreateCollection=Unable to create Solr collection",
1095  "Server.exceptionMessage.unableToBackupCollection=Unable to backup Solr collection",
1096  "Server.exceptionMessage.unableToRestoreCollection=Unable to restore Solr collection",
1097  })
1098  private Collection openCore(Case theCase, Index index) throws KeywordSearchModuleException {
1099 
1100  int numShardsToUse = 1;
1101  try {
1102  // connect to proper Solr server
1103  configureSolrConnection(theCase, index);
1104 
1105  if (theCase.getCaseType() == CaseType.MULTI_USER_CASE) {
1106  // select number of shards to use
1107  numShardsToUse = getNumShardsToUse();
1108  }
1109  } catch (Exception ex) {
1110  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1111  throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg", ex.getLocalizedMessage()), ex);
1112  }
1113 
1114  try {
1115  String collectionName = index.getIndexName();
1116 
1117  if (theCase.getCaseType() == CaseType.MULTI_USER_CASE) {
1118  if (!collectionExists(collectionName)) {
1119  /*
1120  * The collection does not exist. Make a request that will cause the colelction to be created.
1121  */
1122  boolean doRetry = false;
1123  for (int reTryAttempt = 0; reTryAttempt < NUM_COLLECTION_CREATION_RETRIES; reTryAttempt++) {
1124  try {
1125  doRetry = false;
1126  createMultiUserCollection(collectionName, numShardsToUse);
1127  } catch (Exception ex) {
1128  if (reTryAttempt >= NUM_COLLECTION_CREATION_RETRIES) {
1129  logger.log(Level.SEVERE, "Unable to create Solr collection " + collectionName, ex); //NON-NLS
1130  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
1131  } else {
1132  logger.log(Level.SEVERE, "Unable to create Solr collection " + collectionName + ". Re-trying...", ex); //NON-NLS
1133  Thread.sleep(1000L);
1134  doRetry = true;
1135  }
1136  }
1137  if (!doRetry) {
1138  break;
1139  }
1140  }
1141  }
1142  } else {
1143  if (!coreIsLoaded(collectionName)) {
1144  // In single user mode, the index is stored in case output directory
1145  File dataDir = new File(new File(index.getIndexPath()).getParent()); // "data dir" is the parent of the index directory
1146  if (!dataDir.exists()) {
1147  dataDir.mkdirs();
1148  }
1149 
1150  // In single user mode, if there is a core.properties file already,
1151  // we've hit a solr bug. Compensate by deleting it.
1152  if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
1153  Path corePropertiesFile = Paths.get(localSolrFolder.toString(), SOLR, collectionName, CORE_PROPERTIES);
1154  if (corePropertiesFile.toFile().exists()) {
1155  try {
1156  corePropertiesFile.toFile().delete();
1157  } catch (Exception ex) {
1158  logger.log(Level.INFO, "Could not delete pre-existing core.properties prior to opening the core."); //NON-NLS
1159  }
1160  }
1161  }
1162 
1163  // for single user cases, we unload the core when we close the case. So we have to load the core again.
1164  CoreAdminRequest.Create createCoreRequest = new CoreAdminRequest.Create();
1165  createCoreRequest.setDataDir(dataDir.getAbsolutePath());
1166  createCoreRequest.setCoreName(collectionName);
1167  createCoreRequest.setConfigSet("AutopsyConfig"); //NON-NLS
1168  createCoreRequest.setIsLoadOnStartup(false);
1169  createCoreRequest.setIsTransient(true);
1170  localSolrServer.request(createCoreRequest);
1171 
1172  if (!coreIndexFolderExists(collectionName)) {
1173  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
1174  }
1175  }
1176  }
1177 
1178  return new Collection(collectionName, theCase, index);
1179 
1180  } catch (Exception ex) {
1181  logger.log(Level.SEVERE, "Exception during Solr collection creation.", ex); //NON-NLS
1182  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.cantOpen.msg"), ex);
1183  }
1184  }
1185 
1186  private int getNumShardsToUse() throws KeywordSearchModuleException {
1187 
1188  // if we want to use a specific sharding strategy, use that
1189  if (org.sleuthkit.autopsy.keywordsearch.UserPreferences.getMaxNumShards() > 0) {
1190  return org.sleuthkit.autopsy.keywordsearch.UserPreferences.getMaxNumShards();
1191  }
1192 
1193  // otherwise get list of all live Solr servers in the cluster
1194  List<String> solrServerList = getSolrServerList(remoteSolrServer);
1195  // shard across all available servers
1196  return solrServerList.size();
1197  }
1198 
1199  /*
1200  * Poll the remote Solr server for list of existing collections, and check if
1201  * the collection of interest exists.
1202  *
1203  * @param collectionName The name of the collection.
1204  *
1205  * @return True if the collection exists, false otherwise.
1206  *
1207  * @throws SolrServerException If there is a problem communicating with the
1208  * Solr server.
1209  * @throws IOException If there is a problem communicating with the Solr
1210  * server.
1211  */
1212  private boolean collectionExists(String collectionName) throws SolrServerException, IOException {
1213  CollectionAdminRequest.List req = new CollectionAdminRequest.List();
1214  CollectionAdminResponse response = req.process(remoteSolrServer);
1215  List<?> existingCollections = (List<?>) response.getResponse().get("collections");
1216  if (existingCollections == null) {
1217  existingCollections = new ArrayList<>();
1218  }
1219  return existingCollections.contains(collectionName);
1220  }
1221 
1222  /* NOTE: Keeping this code for reference, since it works.
1223  private boolean collectionExists(String collectionName) throws SolrServerException, IOException {
1224 
1225  // TODO we could potentially use this API. Currently set exception "Solr instance is not running in SolrCloud mode"
1226  // List<String> list = CollectionAdminRequest.listCollections(localSolrServer);
1227 
1228  CollectionAdminRequest.ClusterStatus statusRequest = CollectionAdminRequest.getClusterStatus().setCollectionName(collectionName);
1229  CollectionAdminResponse statusResponse;
1230  try {
1231  statusResponse = statusRequest.process(remoteSolrServer);
1232  } catch (RemoteSolrException ex) {
1233  // collection doesn't exist
1234  return false;
1235  }
1236 
1237  if (statusResponse == null) {
1238  return false;
1239  }
1240 
1241  NamedList error = (NamedList) statusResponse.getResponse().get("error");
1242  if (error != null) {
1243  return false;
1244  }
1245 
1246  // For some reason this returns info about all collections even though it's supposed to only return about the one we are requesting
1247  NamedList cluster = (NamedList) statusResponse.getResponse().get("cluster");
1248  NamedList collections = (NamedList) cluster.get("collections");
1249  if (collections != null) {
1250  Object collection = collections.get(collectionName);
1251  return (collection != null);
1252  } else {
1253  return false;
1254  }
1255  }*/
1256 
1257  private void createMultiUserCollection(String collectionName, int numShardsToUse) throws KeywordSearchModuleException, SolrServerException, IOException {
1258  /*
1259  * The core either does not exist or it is not loaded. Make a
1260  * request that will cause the core to be created if it does not
1261  * exist or loaded if it already exists.
1262  */
1263 
1264  Integer numShards = numShardsToUse;
1265  logger.log(Level.INFO, "numShardsToUse: {0}", numShardsToUse);
1266  Integer numNrtReplicas = 1;
1267  Integer numTlogReplicas = 0;
1268  Integer numPullReplicas = 0;
1269  CollectionAdminRequest.Create createCollectionRequest = CollectionAdminRequest.createCollection(collectionName, "AutopsyConfig", numShards, numNrtReplicas, numTlogReplicas, numPullReplicas);
1270 
1271  CollectionAdminResponse createResponse = createCollectionRequest.process(remoteSolrServer);
1272  if (createResponse.isSuccess()) {
1273  logger.log(Level.INFO, "Collection {0} successfully created.", collectionName);
1274  } else {
1275  logger.log(Level.SEVERE, "Unable to create Solr collection {0}", collectionName); //NON-NLS
1276  throw new KeywordSearchModuleException(Bundle.Server_exceptionMessage_unableToCreateCollection());
1277  }
1278 
1279  /* If we need core name:
1280  Map<String, NamedList<Integer>> status = createResponse.getCollectionCoresStatus();
1281  existingCoreName = status.keySet().iterator().next();*/
1282  if (!collectionExists(collectionName)) {
1283  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.openCore.exception.noIndexDir.msg"));
1284  }
1285  }
1286 
1287  private void backupCollection(String collectionName, String backupName, String pathToBackupLocation) throws SolrServerException, IOException, KeywordSearchModuleException {
1288  CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(collectionName, backupName)
1289  .setLocation(pathToBackupLocation);
1290 
1291  CollectionAdminResponse backupResponse = backup.process(remoteSolrServer);
1292  if (backupResponse.isSuccess()) {
1293  logger.log(Level.INFO, "Collection {0} successfully backep up.", collectionName);
1294  } else {
1295  logger.log(Level.SEVERE, "Unable to back up Solr collection {0}", collectionName); //NON-NLS
1296  throw new KeywordSearchModuleException(Bundle.Server_exceptionMessage_unableToBackupCollection());
1297  }
1298  }
1299 
1300  private void restoreCollection(String backupName, String restoreCollectionName, String pathToBackupLocation) throws SolrServerException, IOException, KeywordSearchModuleException {
1301 
1302  CollectionAdminRequest.Restore restore = CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
1303  .setLocation(pathToBackupLocation);
1304 
1305  CollectionAdminResponse restoreResponse = restore.process(remoteSolrServer);
1306  if (restoreResponse.isSuccess()) {
1307  logger.log(Level.INFO, "Collection {0} successfully resored.", restoreCollectionName);
1308  } else {
1309  logger.log(Level.SEVERE, "Unable to restore Solr collection {0}", restoreCollectionName); //NON-NLS
1310  throw new KeywordSearchModuleException(Bundle.Server_exceptionMessage_unableToRestoreCollection());
1311  }
1312  }
1313 
1328  private boolean coreIsLoaded(String coreName) throws SolrServerException, IOException {
1329  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, localSolrServer);
1330  return response.getCoreStatus(coreName).get("instanceDir") != null; //NON-NLS
1331  }
1332 
1345  private boolean coreIndexFolderExists(String coreName) throws SolrServerException, IOException {
1346  CoreAdminResponse response = CoreAdminRequest.getStatus(coreName, localSolrServer);
1347  Object dataDirPath = response.getCoreStatus(coreName).get("dataDir"); //NON-NLS
1348  if (null != dataDirPath) {
1349  File indexDir = Paths.get((String) dataDirPath, "index").toFile(); //NON-NLS
1350  return indexDir.exists();
1351  } else {
1352  return false;
1353  }
1354  }
1355 
1367  public static IndexingServerProperties getMultiUserServerProperties(String caseDirectory) {
1368 
1369  // if "solrserver.txt" is present, use those connection settings
1370  Path serverFilePath = Paths.get(caseDirectory, "solrserver.txt"); //NON-NLS
1371  if (serverFilePath.toFile().exists()) {
1372  try {
1373  List<String> lines = Files.readAllLines(serverFilePath);
1374  if (lines.isEmpty()) {
1375  logger.log(Level.SEVERE, "solrserver.txt file does not contain any data"); //NON-NLS
1376  } else if (!lines.get(0).contains(",")) {
1377  logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0)); //NON-NLS
1378  } else {
1379  String[] parts = lines.get(0).split(",");
1380  if (parts.length != 2) {
1381  logger.log(Level.SEVERE, "solrserver.txt file is corrupt - could not read host/port from " + lines.get(0)); //NON-NLS
1382  } else {
1383  return new IndexingServerProperties(parts[0], parts[1]);
1384  }
1385  }
1386  } catch (IOException ex) {
1387  logger.log(Level.SEVERE, "solrserver.txt file could not be read", ex); //NON-NLS
1388  }
1389  }
1390 
1391  // otherwise (or if an error occurred) determine Solr version of the current case
1392  List<Index> indexes = new ArrayList<>();
1393  try {
1394  IndexMetadata indexMetadata = new IndexMetadata(caseDirectory);
1395  indexes = indexMetadata.getIndexes();
1396  } catch (IndexMetadata.TextIndexMetadataException ex) {
1397  logger.log(Level.SEVERE, "Unable to read text index metadata file: " + caseDirectory, ex);
1398 
1399  // default to using the latest Solr version settings
1400  String host = UserPreferences.getIndexingServerHost();
1401  String port = UserPreferences.getIndexingServerPort();
1402  return new IndexingServerProperties(host, port);
1403  }
1404 
1405  // select which index to use. In practice, all cases always have only one
1406  // index but there is support for having multiple indexes.
1407  Index indexToUse = IndexFinder.identifyIndexToUse(indexes);
1408  if (indexToUse == null) {
1409  // unable to find index that can be used
1410  logger.log(Level.SEVERE, "Unable to find index that can be used for case: {0}", caseDirectory);
1411 
1412  // default to using the latest Solr version settings
1413  String host = UserPreferences.getIndexingServerHost();
1414  String port = UserPreferences.getIndexingServerPort();
1415  return new IndexingServerProperties(host, port);
1416  }
1417 
1418  // return connection info for the Solr version of the current index
1419  if (IndexFinder.getCurrentSolrVersion().equals(indexToUse.getSolrVersion())) {
1420  // Solr 8
1421  String host = UserPreferences.getIndexingServerHost();
1422  String port = UserPreferences.getIndexingServerPort();
1423  return new IndexingServerProperties(host, port);
1424  } else {
1425  // Solr 4
1426  String host = UserPreferences.getSolr4ServerHost().trim();
1427  String port = UserPreferences.getSolr4ServerPort().trim();
1428  return new IndexingServerProperties(host, port);
1429  }
1430  }
1431 
1443  public static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath) throws KeywordSearchModuleException {
1444  // Look for the solr server list file
1445  String serverListName = "solrServerList.txt"; //NON-NLS
1446  Path serverListPath = Paths.get(rootOutputDirectory.toString(), serverListName);
1447  if (serverListPath.toFile().exists()) {
1448 
1449  // Read the list of solr servers
1450  List<String> lines;
1451  try {
1452  lines = Files.readAllLines(serverListPath);
1453  } catch (IOException ex) {
1454  throw new KeywordSearchModuleException(serverListName + " could not be read", ex); //NON-NLS
1455  }
1456 
1457  // Remove any lines that don't contain a comma (these are likely just whitespace)
1458  for (Iterator<String> iterator = lines.iterator(); iterator.hasNext();) {
1459  String line = iterator.next();
1460  if (!line.contains(",")) {
1461  // Remove the current element from the iterator and the list.
1462  iterator.remove();
1463  }
1464  }
1465  if (lines.isEmpty()) {
1466  throw new KeywordSearchModuleException(serverListName + " had no valid server information"); //NON-NLS
1467  }
1468 
1469  // Choose which server to use
1470  int rnd = new Random().nextInt(lines.size());
1471  String[] parts = lines.get(rnd).split(",");
1472  if (parts.length != 2) {
1473  throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd)); //NON-NLS
1474  }
1475 
1476  // Split it up just to do a sanity check on the data
1477  String host = parts[0];
1478  String port = parts[1];
1479  if (host.isEmpty() || port.isEmpty()) {
1480  throw new KeywordSearchModuleException("Invalid server data: " + lines.get(rnd)); //NON-NLS
1481  }
1482 
1483  // Write the server data to a file
1484  Path serverFile = Paths.get(caseDirectoryPath.toString(), "solrserver.txt"); //NON-NLS
1485  try {
1486  caseDirectoryPath.toFile().mkdirs();
1487  if (!caseDirectoryPath.toFile().exists()) {
1488  throw new KeywordSearchModuleException("Case directory " + caseDirectoryPath.toString() + " does not exist"); //NON-NLS
1489  }
1490  Files.write(serverFile, lines.get(rnd).getBytes());
1491  } catch (IOException ex) {
1492  throw new KeywordSearchModuleException(serverFile.toString() + " could not be written", ex); //NON-NLS
1493  }
1494  }
1495  }
1496 
1500  public static class IndexingServerProperties {
1501 
1502  private final String host;
1503  private final String port;
1504 
1505  IndexingServerProperties(String host, String port) {
1506  this.host = host;
1507  this.port = port;
1508  }
1509 
1515  public String getHost() {
1516  return host;
1517  }
1518 
1524  public String getPort() {
1525  return port;
1526  }
1527  }
1528 
1534  void commit() throws SolrServerException, NoOpenCoreException {
1535  currentCoreLock.readLock().lock();
1536  try {
1537  if (null == currentCollection) {
1538  throw new NoOpenCoreException();
1539  }
1540  currentCollection.commit();
1541  } finally {
1542  currentCoreLock.readLock().unlock();
1543  }
1544  }
1545 
1546  NamedList<Object> request(SolrRequest<?> request) throws SolrServerException, RemoteSolrException, NoOpenCoreException {
1547  currentCoreLock.readLock().lock();
1548  try {
1549  if (null == currentCollection) {
1550  throw new NoOpenCoreException();
1551  }
1552  return currentCollection.request(request);
1553  } finally {
1554  currentCoreLock.readLock().unlock();
1555  }
1556  }
1557 
1568  public int queryNumIndexedFiles() throws KeywordSearchModuleException, NoOpenCoreException {
1569  currentCoreLock.readLock().lock();
1570  try {
1571  if (null == currentCollection) {
1572  throw new NoOpenCoreException();
1573  }
1574  try {
1575  return currentCollection.queryNumIndexedFiles();
1576  } catch (Exception ex) {
1577  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1578  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxFiles.exception.msg"), ex);
1579  }
1580  } finally {
1581  currentCoreLock.readLock().unlock();
1582  }
1583  }
1584 
1594  public int queryNumIndexedChunks() throws KeywordSearchModuleException, NoOpenCoreException {
1595  currentCoreLock.readLock().lock();
1596  try {
1597  if (null == currentCollection) {
1598  throw new NoOpenCoreException();
1599  }
1600  try {
1601  return currentCollection.queryNumIndexedChunks();
1602  } catch (Exception ex) {
1603  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1604  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxChunks.exception.msg"), ex);
1605  }
1606  } finally {
1607  currentCoreLock.readLock().unlock();
1608  }
1609  }
1610 
1620  public int queryNumIndexedDocuments() throws KeywordSearchModuleException, NoOpenCoreException {
1621  currentCoreLock.readLock().lock();
1622  try {
1623  if (null == currentCollection) {
1624  throw new NoOpenCoreException();
1625  }
1626  try {
1627  return currentCollection.queryNumIndexedDocuments();
1628  } catch (Exception ex) {
1629  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1630  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumIdxDocs.exception.msg"), ex);
1631  }
1632  } finally {
1633  currentCoreLock.readLock().unlock();
1634  }
1635  }
1636 
1647  public boolean queryIsIndexed(long contentID) throws KeywordSearchModuleException, NoOpenCoreException {
1648  currentCoreLock.readLock().lock();
1649  try {
1650  if (null == currentCollection) {
1651  throw new NoOpenCoreException();
1652  }
1653  try {
1654  return currentCollection.queryIsIndexed(contentID);
1655  } catch (Exception ex) {
1656  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1657  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryIsIdxd.exception.msg"), ex);
1658  }
1659 
1660  } finally {
1661  currentCoreLock.readLock().unlock();
1662  }
1663  }
1664 
1676  public int queryNumFileChunks(long fileID) throws KeywordSearchModuleException, NoOpenCoreException {
1677  currentCoreLock.readLock().lock();
1678  try {
1679  if (null == currentCollection) {
1680  throw new NoOpenCoreException();
1681  }
1682  try {
1683  return currentCollection.queryNumFileChunks(fileID);
1684  } catch (Exception ex) {
1685  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1686  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryNumFileChunks.exception.msg"), ex);
1687  }
1688  } finally {
1689  currentCoreLock.readLock().unlock();
1690  }
1691  }
1692 
1703  public QueryResponse query(SolrQuery sq) throws KeywordSearchModuleException, NoOpenCoreException, IOException {
1704  currentCoreLock.readLock().lock();
1705  try {
1706  if (null == currentCollection) {
1707  throw new NoOpenCoreException();
1708  }
1709  try {
1710  return currentCollection.query(sq);
1711  } catch (Exception ex) {
1712  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1713  logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1714  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query.exception.msg", sq.getQuery()), ex);
1715  }
1716  } finally {
1717  currentCoreLock.readLock().unlock();
1718  }
1719  }
1720 
1732  public QueryResponse query(SolrQuery sq, SolrRequest.METHOD method) throws KeywordSearchModuleException, NoOpenCoreException {
1733  currentCoreLock.readLock().lock();
1734  try {
1735  if (null == currentCollection) {
1736  throw new NoOpenCoreException();
1737  }
1738  try {
1739  return currentCollection.query(sq, method);
1740  } catch (Exception ex) {
1741  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1742  logger.log(Level.SEVERE, "Solr query failed: " + sq.getQuery(), ex); //NON-NLS
1743  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.query2.exception.msg", sq.getQuery()), ex);
1744  }
1745  } finally {
1746  currentCoreLock.readLock().unlock();
1747  }
1748  }
1749 
1760  public TermsResponse queryTerms(SolrQuery sq) throws KeywordSearchModuleException, NoOpenCoreException {
1761  currentCoreLock.readLock().lock();
1762  try {
1763  if (null == currentCollection) {
1764  throw new NoOpenCoreException();
1765  }
1766  try {
1767  return currentCollection.queryTerms(sq);
1768  } catch (Exception ex) {
1769  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1770  logger.log(Level.SEVERE, "Solr terms query failed: " + sq.getQuery(), ex); //NON-NLS
1771  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.queryTerms.exception.msg", sq.getQuery()), ex);
1772  }
1773  } finally {
1774  currentCoreLock.readLock().unlock();
1775  }
1776  }
1777 
1785  void deleteDataSource(Long dataSourceId) throws IOException, KeywordSearchModuleException, NoOpenCoreException, SolrServerException {
1786  try {
1787  currentCoreLock.writeLock().lock();
1788  if (null == currentCollection) {
1789  throw new NoOpenCoreException();
1790  }
1791  currentCollection.deleteDataSource(dataSourceId);
1792  currentCollection.commit();
1793  } finally {
1794  currentCoreLock.writeLock().unlock();
1795  }
1796  }
1797 
1806  @NbBundle.Messages({
1807  "Server.getAllTerms.error=Extraction of all unique Solr terms failed:"})
1808  void extractAllTermsForDataSource(Path outputFile, ReportProgressPanel progressPanel) throws KeywordSearchModuleException, NoOpenCoreException {
1809  try {
1810  currentCoreLock.writeLock().lock();
1811  if (null == currentCollection) {
1812  throw new NoOpenCoreException();
1813  }
1814  try {
1815  currentCollection.extractAllTermsForDataSource(outputFile, progressPanel);
1816  } catch (Exception ex) {
1817  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
1818  logger.log(Level.SEVERE, "Extraction of all unique Solr terms failed: ", ex); //NON-NLS
1819  throw new KeywordSearchModuleException(Bundle.Server_getAllTerms_error(), ex);
1820  }
1821  } finally {
1822  currentCoreLock.writeLock().unlock();
1823  }
1824  }
1825 
1835  public String getSolrContent(final Content content) throws NoOpenCoreException {
1836  currentCoreLock.readLock().lock();
1837  try {
1838  if (null == currentCollection) {
1839  throw new NoOpenCoreException();
1840  }
1841  return currentCollection.getSolrContent(content.getId(), 0);
1842  } finally {
1843  currentCoreLock.readLock().unlock();
1844  }
1845  }
1846 
1859  public String getSolrContent(final Content content, int chunkID) throws NoOpenCoreException {
1860  currentCoreLock.readLock().lock();
1861  try {
1862  if (null == currentCollection) {
1863  throw new NoOpenCoreException();
1864  }
1865  return currentCollection.getSolrContent(content.getId(), chunkID);
1866  } finally {
1867  currentCoreLock.readLock().unlock();
1868  }
1869  }
1870 
1880  public String getSolrContent(final long objectID) throws NoOpenCoreException {
1881  currentCoreLock.readLock().lock();
1882  try {
1883  if (null == currentCollection) {
1884  throw new NoOpenCoreException();
1885  }
1886  return currentCollection.getSolrContent(objectID, 0);
1887  } finally {
1888  currentCoreLock.readLock().unlock();
1889  }
1890  }
1891 
1902  public String getSolrContent(final long objectID, final int chunkID) throws NoOpenCoreException {
1903  currentCoreLock.readLock().lock();
1904  try {
1905  if (null == currentCollection) {
1906  throw new NoOpenCoreException();
1907  }
1908  return currentCollection.getSolrContent(objectID, chunkID);
1909  } finally {
1910  currentCoreLock.readLock().unlock();
1911  }
1912  }
1913 
1923  public static String getChunkIdString(long parentID, int childID) {
1924  return Long.toString(parentID) + Server.CHUNK_ID_SEPARATOR + Integer.toString(childID);
1925  }
1926 
1933  private void connectToEmbeddedSolrServer() throws SolrServerException, IOException {
1934  TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check");
1935  CoreAdminRequest.getStatus(null, localSolrServer);
1937  }
1938 
1950  void connectToSolrServer(String host, String port) throws SolrServerException, IOException {
1951  try (HttpSolrClient solrServer = getSolrClient("http://" + host + ":" + port + "/solr")) {
1952  connectToSolrServer(solrServer);
1953  }
1954  }
1955 
1965  private void connectToSolrServer(HttpSolrClient solrServer) throws SolrServerException, IOException {
1966  TimingMetric metric = HealthMonitor.getTimingMetric("Solr: Connectivity check");
1967  CollectionAdminRequest.ClusterStatus statusRequest = CollectionAdminRequest.getClusterStatus();
1968  CollectionAdminResponse statusResponse = statusRequest.process(solrServer);
1969  int statusCode = Integer.valueOf(((NamedList) statusResponse.getResponse().get("responseHeader")).get("status").toString());
1970  if (statusCode != 0) {
1971  logger.log(Level.WARNING, "Could not connect to Solr server "); //NON-NLS
1972  } else {
1973  logger.log(Level.INFO, "Connected to Solr server "); //NON-NLS
1974  }
1976  }
1977 
1978  private List<String> getSolrServerList(String host, String port) throws KeywordSearchModuleException {
1979  HttpSolrClient solrServer = getSolrClient("http://" + host + ":" + port + "/solr");
1980  return getSolrServerList(solrServer);
1981  }
1982 
1983  private List<String> getSolrServerList(HttpSolrClient solrServer) throws KeywordSearchModuleException {
1984 
1985  try {
1986  CollectionAdminRequest.ClusterStatus statusRequest = CollectionAdminRequest.getClusterStatus();
1987  CollectionAdminResponse statusResponse;
1988  try {
1989  statusResponse = statusRequest.process(solrServer);
1990  } catch (RemoteSolrException ex) {
1991  // collection doesn't exist
1992  return Collections.emptyList();
1993  }
1994 
1995  if (statusResponse == null) {
1996  return Collections.emptyList();
1997  }
1998 
1999  NamedList<?> error = (NamedList) statusResponse.getResponse().get("error");
2000  if (error != null) {
2001  return Collections.emptyList();
2002  }
2003 
2004  NamedList<?> cluster = (NamedList) statusResponse.getResponse().get("cluster");
2005  @SuppressWarnings("unchecked")
2006  ArrayList<String> liveNodes = (ArrayList) cluster.get("live_nodes");
2007  return liveNodes;
2008  } catch (Exception ex) {
2009  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2010  throw new KeywordSearchModuleException(
2011  NbBundle.getMessage(this.getClass(), "Server.serverList.exception.msg", solrServer.getBaseURL()));
2012  }
2013  }
2014 
2015  class Collection {
2016 
2017  // handle to the collection in Solr
2018  private final String name;
2019 
2020  private final CaseType caseType;
2021 
2022  private final Index textIndex;
2023 
2024  // We use different Solr clients for different operations. HttpSolrClient is geared towards query performance.
2025  // ConcurrentUpdateSolrClient is geared towards batching solr documents for better indexing throughput. We
2026  // have implemented our own batching algorithm so we will probably not use ConcurrentUpdateSolrClient.
2027  // CloudSolrClient is geared towards SolrCloud deployments. These are only good for collection-specific operations.
2028  private HttpSolrClient queryClient;
2029  private SolrClient indexingClient;
2030 
2031  private final int maxBufferSize;
2032  private final List<SolrInputDocument> buffer;
2033  private final Object bufferLock;
2034 
2035  /* (JIRA-7521) Sometimes we get into a situation where Solr server is no longer able to index new data.
2036  * Typically main reason for this is Solr running out of memory. In this case we will stop trying to send new
2037  * data to Solr (for this collection) after certain number of consecutive batches have failed. */
2038  private static final int MAX_NUM_CONSECUTIVE_FAILURES = 5;
2039  private AtomicInteger numConsecutiveFailures = new AtomicInteger(0);
2040  private AtomicBoolean skipIndexing = new AtomicBoolean(false);
2041 
2042  private final ScheduledThreadPoolExecutor periodicTasksExecutor;
2043  private static final long PERIODIC_BATCH_SEND_INTERVAL_MINUTES = 10;
2044  private static final int NUM_BATCH_UPDATE_RETRIES = 10;
2045  private static final long SLEEP_BETWEEN_RETRIES_MS = 10000; // 10 seconds
2046 
2047  private Collection(String name, Case theCase, Index index) throws TimeoutException, InterruptedException, KeywordSearchModuleException {
2048  this.name = name;
2049  this.caseType = theCase.getCaseType();
2050  this.textIndex = index;
2051  bufferLock = new Object();
2052 
2053  if (caseType == CaseType.SINGLE_USER_CASE) {
2054  // get SolrJ client
2055  queryClient = getSolrClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // HttpClient
2056  indexingClient = getSolrClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // HttpClient
2057  } else {
2058  // read Solr connection info from user preferences, unless "solrserver.txt" is present
2059  queryClient = configureMultiUserConnection(theCase, index, name);
2060 
2061  // for MU cases, use CloudSolrClient for indexing. Indexing is only supported for Solr 8.
2062  if (IndexFinder.getCurrentSolrVersion().equals(index.getSolrVersion())) {
2063  IndexingServerProperties properties = getMultiUserServerProperties(theCase.getCaseDirectory());
2064  indexingClient = getCloudSolrClient(properties.getHost(), properties.getPort(), name); // CloudClient
2065  } else {
2066  indexingClient = configureMultiUserConnection(theCase, index, name); // HttpClient
2067  }
2068  }
2069 
2070  // document batching
2071  maxBufferSize = org.sleuthkit.autopsy.keywordsearch.UserPreferences.getDocumentsQueueSize();
2072  logger.log(Level.INFO, "Using Solr document queue size = {0}", maxBufferSize); //NON-NLS
2073  buffer = new ArrayList<>(maxBufferSize);
2074  periodicTasksExecutor = new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder().setNameFormat("periodic-batched-document-task-%d").build()); //NON-NLS
2075  periodicTasksExecutor.scheduleWithFixedDelay(new SendBatchedDocumentsTask(), PERIODIC_BATCH_SEND_INTERVAL_MINUTES, PERIODIC_BATCH_SEND_INTERVAL_MINUTES, TimeUnit.MINUTES);
2076  }
2077 
2084  private final class SendBatchedDocumentsTask implements Runnable {
2085 
2086  @Override
2087  public void run() {
2088 
2089  if (skipIndexing.get()) {
2090  return;
2091  }
2092 
2093  List<SolrInputDocument> clone;
2094  synchronized (bufferLock) {
2095 
2096  if (buffer.isEmpty()) {
2097  return;
2098  }
2099 
2100  // Buffer is full. Make a clone and release the lock, so that we don't
2101  // hold other ingest threads
2102  clone = buffer.stream().collect(toList());
2103  buffer.clear();
2104  }
2105 
2106  try {
2107  // send the cloned list to Solr
2108  sendBufferedDocs(clone);
2109  } catch (KeywordSearchModuleException ex) {
2110  logger.log(Level.SEVERE, "Periodic batched document update failed", ex); //NON-NLS
2111  }
2112  }
2113  }
2114 
2120  String getName() {
2121  return name;
2122  }
2123 
2124  private Index getIndexInfo() {
2125  return this.textIndex;
2126  }
2127 
2128  private QueryResponse query(SolrQuery sq) throws SolrServerException, IOException {
2129  return queryClient.query(sq);
2130  }
2131 
2132  private NamedList<Object> request(SolrRequest<?> request) throws SolrServerException, RemoteSolrException {
2133  try {
2134  return queryClient.request(request);
2135  } catch (Exception e) {
2136  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2137  logger.log(Level.WARNING, "Could not issue Solr request. ", e); //NON-NLS
2138  throw new SolrServerException(
2139  NbBundle.getMessage(this.getClass(), "Server.request.exception.exception.msg"), e);
2140  }
2141 
2142  }
2143 
2144  private QueryResponse query(SolrQuery sq, SolrRequest.METHOD method) throws SolrServerException, IOException {
2145  return queryClient.query(sq, method);
2146  }
2147 
2148  private TermsResponse queryTerms(SolrQuery sq) throws SolrServerException, IOException {
2149  QueryResponse qres = queryClient.query(sq);
2150  return qres.getTermsResponse();
2151  }
2152 
2153  private void commit() throws SolrServerException {
2154  List<SolrInputDocument> clone;
2155  synchronized (bufferLock) {
2156  // Make a clone and release the lock, so that we don't
2157  // hold other ingest threads
2158  clone = buffer.stream().collect(toList());
2159  buffer.clear();
2160  }
2161 
2162  try {
2163  sendBufferedDocs(clone);
2164  } catch (KeywordSearchModuleException ex) {
2165  throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), ex);
2166  }
2167 
2168  try {
2169  //commit and block
2170  indexingClient.commit(true, true);
2171  } catch (Exception e) {
2172  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2173  logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
2174  throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
2175  }
2176  }
2177 
2178  private void deleteDataSource(Long dsObjId) throws IOException, SolrServerException {
2179  String dataSourceId = Long.toString(dsObjId);
2180  String deleteQuery = "image_id:" + dataSourceId;
2181 
2182  queryClient.deleteByQuery(deleteQuery);
2183  }
2184 
2196  @NbBundle.Messages({
2197  "# {0} - Number of extracted terms",
2198  "ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms..."
2199  })
2200  private void extractAllTermsForDataSource(Path outputFile, ReportProgressPanel progressPanel) throws IOException, SolrServerException, NoCurrentCaseException, KeywordSearchModuleException {
2201 
2202  Files.deleteIfExists(outputFile);
2203  OpenOption[] options = new OpenOption[] { java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND };
2204 
2205  // step through the terms
2206  int termStep = 1000;
2207  long numExtractedTerms = 0;
2208  String firstTerm = "";
2209  while (true) {
2210  SolrQuery query = new SolrQuery();
2211  query.setRequestHandler("/terms");
2212  query.setTerms(true);
2213  query.setTermsLimit(termStep);
2214  query.setTermsLower(firstTerm);
2215  query.setTermsLowerInclusive(false);
2216 
2217  // Returned terms sorted by "index" order, which is the fastest way. Per Solr documentation:
2218  // "Retrieving terms in index order is very fast since the implementation directly uses Lucene’s TermEnum to iterate over the term dictionary."
2219  // All other sort criteria return very inconsistent and overlapping resuts.
2220  query.setTermsSortString("index");
2221 
2222  // "text" field is the schema field that we populate with (lowercased) terms
2223  query.addTermsField(Server.Schema.TEXT.toString());
2224  query.setTermsMinCount(0);
2225 
2226  // Unfortunatelly Solr "terms queries" do not support any filtering so we can't filter by data source this way.
2227  // query.addFilterQuery(Server.Schema.IMAGE_ID.toString() + ":" + dataSourceId);
2228 
2229  QueryRequest request = new QueryRequest(query);
2230  TermsResponse response = request.process(queryClient).getTermsResponse();
2231  List<Term> terms = response.getTerms(Server.Schema.TEXT.toString());
2232 
2233  if (terms == null || terms.isEmpty()) {
2234  numExtractedTerms += terms.size();
2235  progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_numberExtractedTerms(numExtractedTerms));
2236  break;
2237  }
2238 
2239  // set the first term for the next query
2240  firstTerm = terms.get(terms.size()-1).getTerm();
2241 
2242  List<String> listTerms = terms.stream().map(Term::getTerm).collect(Collectors.toList());
2243  Files.write(outputFile, listTerms, options);
2244 
2245  numExtractedTerms += termStep;
2246  progressPanel.updateStatusLabel(Bundle.ExtractAllTermsReport_numberExtractedTerms(numExtractedTerms));
2247  }
2248  }
2249 
2258  void addDocument(SolrInputDocument doc) throws KeywordSearchModuleException {
2259 
2260  if (skipIndexing.get()) {
2261  return;
2262  }
2263 
2264  List<SolrInputDocument> clone;
2265  synchronized (bufferLock) {
2266  buffer.add(doc);
2267  // buffer documents if the buffer is not full
2268  if (buffer.size() < maxBufferSize) {
2269  return;
2270  }
2271 
2272  // Buffer is full. Make a clone and release the lock, so that we don't
2273  // hold other ingest threads
2274  clone = buffer.stream().collect(toList());
2275  buffer.clear();
2276  }
2277 
2278  // send the cloned list to Solr
2279  sendBufferedDocs(clone);
2280  }
2281 
2289  @NbBundle.Messages({
2290  "Collection.unableToIndexData.error=Unable to add data to text index. All future text indexing for the current case will be skipped.",
2291 
2292  })
2293  private void sendBufferedDocs(List<SolrInputDocument> docBuffer) throws KeywordSearchModuleException {
2294 
2295  if (docBuffer.isEmpty()) {
2296  return;
2297  }
2298 
2299  try {
2300  boolean success = true;
2301  for (int reTryAttempt = 0; reTryAttempt < NUM_BATCH_UPDATE_RETRIES; reTryAttempt++) {
2302  try {
2303  success = true;
2304  indexingClient.add(docBuffer);
2305  } catch (Exception ex) {
2306  success = false;
2307  if (reTryAttempt < NUM_BATCH_UPDATE_RETRIES - 1) {
2308  logger.log(Level.WARNING, "Unable to send document batch to Solr. Re-trying...", ex); //NON-NLS
2309  try {
2310  Thread.sleep(SLEEP_BETWEEN_RETRIES_MS);
2311  } catch (InterruptedException ignore) {
2312  throw new KeywordSearchModuleException(
2313  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex); //NON-NLS
2314  }
2315  }
2316  }
2317  if (success) {
2318  numConsecutiveFailures.set(0);
2319  if (reTryAttempt > 0) {
2320  logger.log(Level.INFO, "Batch update suceeded after {0} re-try", reTryAttempt); //NON-NLS
2321  }
2322  return;
2323  }
2324  }
2325  // if we are here, it means all re-try attempts failed
2326  logger.log(Level.SEVERE, "Unable to send document batch to Solr. All re-try attempts failed!"); //NON-NLS
2327  throw new KeywordSearchModuleException(NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg")); //NON-NLS
2328  } catch (Exception ex) {
2329  // Solr throws a lot of unexpected exception types
2330  numConsecutiveFailures.incrementAndGet();
2331  logger.log(Level.SEVERE, "Could not add batched documents to index", ex); //NON-NLS
2332 
2333  // display message to user that that a document batch is missing from the index
2334  MessageNotifyUtil.Notify.error(
2335  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"),
2336  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"));
2337  throw new KeywordSearchModuleException(
2338  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex); //NON-NLS
2339  } finally {
2340  if (numConsecutiveFailures.get() >= MAX_NUM_CONSECUTIVE_FAILURES) {
2341  // skip all future indexing
2342  skipIndexing.set(true);
2343  logger.log(Level.SEVERE, "Unable to add data to text index. All future text indexing for the current case will be skipped!"); //NON-NLS
2344 
2345  // display message to user that no more data will be added to the index
2346  MessageNotifyUtil.Notify.error(
2347  NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"),
2348  Bundle.Collection_unableToIndexData_error());
2349  if (RuntimeProperties.runningWithGUI()) {
2350  MessageNotifyUtil.Message.error(Bundle.Collection_unableToIndexData_error());
2351  }
2352  }
2353  docBuffer.clear();
2354  }
2355  }
2356 
2367  private String getSolrContent(long contentID, int chunkID) {
2368  final SolrQuery q = new SolrQuery();
2369  q.setQuery("*:*");
2370  String filterQuery = Schema.ID.toString() + ":" + KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
2371  if (chunkID != 0) {
2372  filterQuery = filterQuery + Server.CHUNK_ID_SEPARATOR + chunkID;
2373  }
2374  q.addFilterQuery(filterQuery);
2375  q.setFields(Schema.TEXT.toString());
2376  try {
2377  // Get the first result.
2378  SolrDocumentList solrDocuments = queryClient.query(q).getResults();
2379 
2380  if (!solrDocuments.isEmpty()) {
2381  SolrDocument solrDocument = solrDocuments.get(0);
2382  if (solrDocument != null) {
2383  java.util.Collection<Object> fieldValues = solrDocument.getFieldValues(Schema.TEXT.toString());
2384  if (fieldValues.size() == 1) // The indexed text field for artifacts will only have a single value.
2385  {
2386  return fieldValues.toArray(new String[0])[0];
2387  } else // The indexed text for files has 2 values, the file name and the file content.
2388  // We return the file content value.
2389  {
2390  return fieldValues.toArray(new String[0])[1];
2391  }
2392  }
2393  }
2394  } catch (Exception ex) {
2395  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2396  logger.log(Level.SEVERE, "Error getting content from Solr. Solr document id " + contentID + ", chunk id " + chunkID + ", query: " + filterQuery, ex); //NON-NLS
2397  return null;
2398  }
2399 
2400  return null;
2401  }
2402 
2403  synchronized void close() throws KeywordSearchModuleException {
2404  try {
2405 
2406  // stop the periodic batch update task. If the task is already running,
2407  // allow it to finish.
2408  ThreadUtils.shutDownTaskExecutor(periodicTasksExecutor);
2409 
2410  // We only unload cores for "single-user" cases.
2411  if (this.caseType == CaseType.MULTI_USER_CASE) {
2412  return;
2413  }
2414 
2415  CoreAdminRequest.unloadCore(this.name, localSolrServer);
2416  } catch (Exception ex) {
2417  // intentional "catch all" as Solr is known to throw all kinds of Runtime exceptions
2418  throw new KeywordSearchModuleException(
2419  NbBundle.getMessage(this.getClass(), "Server.close.exception.msg"), ex);
2420  } finally {
2421  try {
2422  queryClient.close();
2423  queryClient = null;
2424  indexingClient.close();
2425  indexingClient = null;
2426  } catch (IOException ex) {
2427  throw new KeywordSearchModuleException(
2428  NbBundle.getMessage(this.getClass(), "Server.close.exception.msg2"), ex);
2429  }
2430  }
2431  }
2432 
2442  private int queryNumIndexedFiles() throws SolrServerException, IOException {
2444  }
2445 
2455  private int queryNumIndexedChunks() throws SolrServerException, IOException {
2456  SolrQuery q = new SolrQuery(Server.Schema.ID + ":*" + Server.CHUNK_ID_SEPARATOR + "*");
2457  q.setRows(0);
2458  int numChunks = (int) query(q).getResults().getNumFound();
2459  return numChunks;
2460  }
2461 
2472  private int queryNumIndexedDocuments() throws SolrServerException, IOException {
2473  SolrQuery q = new SolrQuery("*:*");
2474  q.setRows(0);
2475  return (int) query(q).getResults().getNumFound();
2476  }
2477 
2487  private boolean queryIsIndexed(long contentID) throws SolrServerException, IOException {
2488  String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
2489  SolrQuery q = new SolrQuery("*:*");
2490  q.addFilterQuery(Server.Schema.ID.toString() + ":" + id);
2491  //q.setFields(Server.Schema.ID.toString());
2492  q.setRows(0);
2493  return (int) query(q).getResults().getNumFound() != 0;
2494  }
2495 
2507  private int queryNumFileChunks(long contentID) throws SolrServerException, IOException {
2508  String id = KeywordSearchUtil.escapeLuceneQuery(Long.toString(contentID));
2509  final SolrQuery q
2510  = new SolrQuery(Server.Schema.ID + ":" + id + Server.CHUNK_ID_SEPARATOR + "*");
2511  q.setRows(0);
2512  return (int) query(q).getResults().getNumFound();
2513  }
2514  }
2515 
2516  class ServerAction extends AbstractAction {
2517 
2518  private static final long serialVersionUID = 1L;
2519 
2520  @Override
2521  public void actionPerformed(ActionEvent e) {
2522  logger.log(Level.INFO, e.paramString().trim());
2523  }
2524  }
2525 
2529  class SolrServerNoPortException extends SocketException {
2530 
2531  private static final long serialVersionUID = 1L;
2532 
2536  private final int port;
2537 
2538  SolrServerNoPortException(int port) {
2539  super(NbBundle.getMessage(Server.class, "Server.solrServerNoPortException.msg", port,
2540  Server.PROPERTIES_CURRENT_SERVER_PORT));
2541  this.port = port;
2542  }
2543 
2544  int getPortNumber() {
2545  return port;
2546  }
2547  }
2548 }
static synchronized String getConfigSetting(String moduleName, String settingName)
String getSolrContent(final long objectID)
Definition: Server.java:1880
final ReentrantReadWriteLock currentCoreLock
Definition: Server.java:273
ConcurrentUpdateSolrClient getConcurrentClient(String solrUrl)
Definition: Server.java:366
static IndexingServerProperties getMultiUserServerProperties(String caseDirectory)
Definition: Server.java:1367
Collection openCore(Case theCase, Index index)
Definition: Server.java:1098
void connectToSolrServer(HttpSolrClient solrServer)
Definition: Server.java:1965
boolean coreIsLoaded(String coreName)
Definition: Server.java:1328
Process runLocalSolr8ControlCommand(List< String > solrArguments)
Definition: Server.java:496
static final int EMBEDDED_SERVER_RETRY_WAIT_SEC
Definition: Server.java:253
List< String > getSolrServerList(String host, String port)
Definition: Server.java:1978
static final int NUM_COLLECTION_CREATION_RETRIES
Definition: Server.java:251
void backupCollection(String collectionName, String backupName, String pathToBackupLocation)
Definition: Server.java:1287
void configureSolrConnection(Case theCase, Index index)
Definition: Server.java:614
CloudSolrClient getCloudSolrClient(String host, String port, String defaultCollectionName)
Definition: Server.java:383
boolean collectionExists(String collectionName)
Definition: Server.java:1212
static TimingMetric getTimingMetric(String name)
static void selectSolrServerForCase(Path rootOutputDirectory, Path caseDirectoryPath)
Definition: Server.java:1443
void addServerActionListener(PropertyChangeListener l)
Definition: Server.java:411
static synchronized boolean settingExists(String moduleName, String settingName)
static final String HL_ANALYZE_CHARS_UNLIMITED
Definition: Server.java:227
String getSolrContent(final Content content)
Definition: Server.java:1835
boolean coreIndexFolderExists(String coreName)
Definition: Server.java:1345
void restoreCollection(String backupName, String restoreCollectionName, String pathToBackupLocation)
Definition: Server.java:1300
static synchronized void setConfigSetting(String moduleName, String settingName, String settingVal)
HttpSolrClient configureMultiUserConnection(Case theCase, Index index, String name)
Definition: Server.java:657
static final Charset DEFAULT_INDEXED_TEXT_CHARSET
default Charset to index text as
Definition: Server.java:236
InputStreamPrinterThread errorRedirectThread
Definition: Server.java:276
Process runLocalSolr4ControlCommand(List< String > solrArguments)
Definition: Server.java:546
List< String > getSolrServerList(HttpSolrClient solrServer)
Definition: Server.java:1983
QueryResponse query(SolrQuery sq)
Definition: Server.java:1703
static synchronized String getJavaPath()
static void submitTimingMetric(TimingMetric metric)
TermsResponse queryTerms(SolrQuery sq)
Definition: Server.java:1760
boolean queryIsIndexed(long contentID)
Definition: Server.java:1647
void createMultiUserCollection(String collectionName, int numShardsToUse)
Definition: Server.java:1257
String getSolrContent(final Content content, int chunkID)
Definition: Server.java:1859
String getSolrContent(final long objectID, final int chunkID)
Definition: Server.java:1902
HttpSolrClient getSolrClient(String solrUrl)
Definition: Server.java:357
synchronized static Logger getLogger(String name)
Definition: Logger.java:124
static synchronized long[] getJavaPIDs(String sigarSubQuery)
static String getChunkIdString(long parentID, int childID)
Definition: Server.java:1923
static boolean deleteDir(File dirPath)
Definition: FileUtil.java:47
QueryResponse query(SolrQuery sq, SolrRequest.METHOD method)
Definition: Server.java:1732

Copyright © 2012-2021 Basis Technology. Generated on: Fri Aug 6 2021
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.