/*******************************************************************************
* Copyright (c) 2013, Perforce Software
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
******************************************************************************/
package com.perforce.search.manager.impl;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.SimpleOrderedMap;
import com.perforce.p4java.extension.exception.ServerConnectionException;
import com.perforce.p4java.extension.path.PerforcePath;
import com.perforce.p4java.extension.path.PerforcePathFactory;
import com.perforce.search.configuration.Configuration;
import com.perforce.search.manager.ContentStreamWrapper;
import com.perforce.search.manager.SchemaKey;
import com.perforce.search.manager.SearchIndexManager;
import com.perforce.search.manager.SolrSchemaField;
import com.perforce.search.model.impl.SearchModelImpl;
import com.perforce.search.server.ServerConnectionPoolFacade;
public class SearchIndexManagerImpl implements SearchIndexManager {
private Logger log = Logger.getLogger(SearchIndexManagerImpl.class);
public static final String LITERALS_PREFIX = "literal_";
private Configuration configuration;
private final String searchUrl;
private SolrServer solr;
private final String collectionName;
private boolean indexAllRevisions;
private final ContentStream emptyStream;
public SearchIndexManagerImpl(final Configuration config,
final ServerConnectionPoolFacade pool) {
this.configuration = config;
this.searchUrl = config.getSearchURL();
this.collectionName = config.getCollectionName();
this.indexAllRevisions = config.isIndexAllRevisions();
solr = new HttpSolrServer(searchUrl);
this.emptyStream = new ContentStreamWrapper(new ByteArrayInputStream(
(new String("")).getBytes()), "");
}
private String[] toStringArray(final PerforcePath path) {
// TODO: need to manage escaped "/"'s?
String[] split = path.asRevisionlessString(null).split("/");
// first 2 will be empty (//depot/...) and the last one is the filename
List<String> ret = new ArrayList<String>();
for (int i = 2; i < split.length - 1; ++i) {
ret.add(split[i]);
}
return ret.toArray(new String[ret.size()]);
}
@Override
public boolean add(final PerforcePath path, final int rev,
final boolean headRev, final String modifiedUser,
final Date modifiedDate, final long size,
final Map<String, String> fstats) throws ServerConnectionException {
try {
SolrInputDocument doc = new SolrInputDocument();
doc.addField(SchemaKey.ID,
(indexAllRevisions) ? PerforcePathFactory
.getPathWithNewRevision(path, rev).asString(null)
: path.asRevisionlessString(null));
doc.addField(SchemaKey.REVISION,
path.getRevisionString().replace("#", ""));
doc.addField(SchemaKey.FILENAME, path.getFileName());
doc.addField(SchemaKey.HEADREV, headRev);
doc.addField(SchemaKey.MODIFIED_BY, modifiedUser);
doc.addField(SchemaKey.MODIFIED_TIME, modifiedDate);
doc.addField(SchemaKey.SIZE, size);
for (String p : toStringArray(path)) {
doc.addField(SchemaKey.PATH, p);
}
if (fstats != null) {
for (Map.Entry<String, String> e : fstats.entrySet()) {
doc.addField(e.getKey(), e.getValue());
}
}
solr.add(doc);
return true;
} catch (SolrServerException e) {
log.error("Solr Exception for " + path.asString(null) + ", "
+ e.getLocalizedMessage());
throw new ServerConnectionException(e);
} catch (IOException e) {
log.error("IOException for " + path.asString(null) + ", "
+ e.getLocalizedMessage());
} catch (Exception e) {
log.error("Generic exception for " + path.asString(null) + ", "
+ e.getLocalizedMessage());
}
return false;
}
private String literal(final String key) {
return "literal." + key;
}
@Override
public boolean upload(final PerforcePath path, final ContentStream cs,
final boolean headRev, final String modifiedUser,
final Date modifiedDate, final long size,
final Map<String, String> fstats) throws ServerConnectionException {
try {
ContentStreamUpdateRequest up = new ContentStreamUpdateRequest(
"/update/extract");
up.addContentStream(cs);
// add an empty content stream to force multipart which keeps the
// URL nice and short
up.addContentStream(emptyStream);
up.setParam(
literal(SchemaKey.ID),
(indexAllRevisions) ? path.asString(null) : path
.asRevisionlessString(null));
up.setParam(literal(SchemaKey.REVISION), path.getRevisionString()
.replace("#", ""));
up.setParam(literal(SchemaKey.FILENAME), path.getFileName());
up.setParam(literal(SchemaKey.HEADREV), String.valueOf(headRev));
up.setParam(literal(SchemaKey.MODIFIED_BY), modifiedUser);
up.setParam(literal(SchemaKey.MODIFIED_TIME),
modifiedDate.toString());
up.setParam(literal(SchemaKey.SIZE), String.valueOf(size));
if (fstats != null) {
for (Map.Entry<String, String> e : fstats.entrySet()) {
up.setParam(literal(e.getKey()), e.getValue());
}
}
ModifiableSolrParams params = up.getParams();
params.add(literal(SchemaKey.PATH), toStringArray(path));
up.setParams(params);
solr.request(up);
return true;
} catch (SolrServerException e) {
log.error("SolrServerException on " + path.asString(null) + ", "
+ e.getLocalizedMessage());
throw new ServerConnectionException(e);
} catch (IOException e) {
log.error("IOException on " + path.asString(null) + ", "
+ e.getLocalizedMessage());
} catch (NoClassDefFoundError e) {
log.error("NoClassDefException on " + path.asString(null) + ", "
+ e.getLocalizedMessage());
} catch (Exception e) {
log.error(
"Exception on " + path.asString(null) + ", "
+ e.getLocalizedMessage(), e);
}
return false;
}
@Override
public void commit() {
try {
solr.commit();
} catch (SolrServerException e) {
log.error("Caught Solr exception: , " + e.getLocalizedMessage());
} catch (IOException e) {
log.error("Caught IO exception: , " + e.getLocalizedMessage());
}
}
@Override
public List<String> search(final String query) {
QueryResponse rsp = null;
try {
log.debug("Searching for: " + query);
SolrQuery solrQuery = new SolrQuery().setQuery(query);
solrQuery.set("collectionName", this.collectionName);
solrQuery.setRows(configuration.getMaxSearchResults());
rsp = solr.query(solrQuery);
SolrDocumentList docs = rsp.getResults();
List<String> searchDepotPaths = new ArrayList<String>();
for (SolrDocument doc : docs) {
searchDepotPaths.add((String) doc.get(SchemaKey.ID));
}
return searchDepotPaths;
} catch (SolrServerException e) {
log.error("Error running search, " + e.getLocalizedMessage());
}
return null;
}
@Override
public QueryResponse searchForFiles(final String query, final int count,
final int startRow) {
try {
log.debug("Searching for: " + query);
SolrQuery solrQuery = new SolrQuery().setQuery(query);
solrQuery.set("collectionName", this.collectionName);
solrQuery.setRows(count);
solrQuery.setStart(startRow);
return solr.query(solrQuery);
} catch (SolrException e) {
log.error("Error running search: " + e.getLocalizedMessage());
} catch (SolrServerException e) {
log.error("Error running search: " + e.getLocalizedMessage());
}
return null;
}
private List<SolrSchemaField> processFieldType(
final List<SimpleOrderedMap<Object>> fields) {
List<SolrSchemaField> retList = new ArrayList<SolrSchemaField>();
for (SimpleOrderedMap<Object> field : fields) {
Object indexed = field.get("indexed");
// only put indexed fields in here
if (indexed == null || !((Boolean) indexed)) {
continue;
}
retList.add(new SolrSchemaFieldImpl(field.get("name"), field
.get("type"), field.get("indexed"), field
.get("multivalued"), field.get("stored")));
}
return retList;
}
@SuppressWarnings("unchecked")
@Override
public List<SolrSchemaField> getSearchFields() {
try {
SolrServer solr = new HttpSolrServer(this.searchUrl + "/"
+ this.collectionName);
SolrQuery query = new SolrQuery();
query.setRequestHandler("/schema");
QueryResponse response = solr.query(query);
// complicated: the query returns the schema as a list of maps
// each element in the list is a field, each field has a map of NVP
SimpleOrderedMap<Object> schema = (SimpleOrderedMap<Object>) response
.getResponse().get("schema");
// get fields and dynamic fields
List<SolrSchemaField> retList = processFieldType((List<SimpleOrderedMap<Object>>) schema
.get("fields"));
retList.addAll(processFieldType((List<SimpleOrderedMap<Object>>) schema
.get("dynamicFields")));
return retList;
} catch (SolrServerException e) {
log.error("Expection running getSearchFields(), "
+ e.getLocalizedMessage());
}
return null;
}
@Override
public List<SolrDocument> getRectifyHeadRevision(final PerforcePath path) {
// do a query for path# and head=true, it should turn up one revision
// (the latest)
// id:(escaped path)# and headrevision:true
// only when indexing all revisions
if (!indexAllRevisions) {
return Collections.emptyList();
}
final int headUpperBound = 100; // limit (not too much) possible
// results)
QueryResponse qr = searchForFiles(String.format(
"%s:%s* AND %s:true",
SchemaKey.ID,
SearchModelImpl.escapeSolrString(path
.asRevisionlessString(null) + "#"), SchemaKey.HEADREV),
headUpperBound, 0);
if (qr == null) {
return Collections.emptyList();
}
SolrDocumentList l = qr.getResults();
if (l == null || l.isEmpty() || l.size() == 1) {
return Collections.emptyList();
}
// loop through 2x, once to get the max rev, the second *reindex*
// yes, reindex, solr update is remove/add, and if you don't store every
// field you can't just partial update
SolrDocument maxRevDoc = null;
int maxRev = 0;
for (Iterator<SolrDocument> i = l.iterator(); i.hasNext();) {
SolrDocument doc = i.next();
int rev = Integer.valueOf((String) doc
.getFieldValue(SchemaKey.REVISION));
if (rev > maxRev) {
maxRev = rev;
maxRevDoc = doc;
}
}
List<SolrDocument> paths = new ArrayList<SolrDocument>();
for (Iterator<SolrDocument> i = l.iterator(); i.hasNext();) {
SolrDocument doc = i.next();
if (doc == maxRevDoc) {
continue;
}
paths.add(doc);
}
return paths;
}
}