elasticlunr

Index

constructor
elasticlunr.Index()

elasticlunr.Index is object that manages a search index. It contains the indexes
and stores all the tokens and document lookups. It also provides the main
user facing API for the library.

elasticlunr.Index = function () {
  this._fields = [];
  this._ref = 'id';
  this.pipeline = new elasticlunr.Pipeline;
  this.documentStore = new elasticlunr.DocumentStore;
  this.index = {};
  this.eventEmitter =  new elasticlunr.EventEmitter;

  this._idfCache = {};

  this.on('add', 'remove', 'update', (function () {
    this._idfCache = {};
  }).bind(this));
};

on

method
elasticlunr.Index.prototype.on()

Option name Type Description
[eventName] String

The name(s) of events to bind the function to.

fn Function

The serialised set to load.

Bind a handler to events being emitted by the index.

The handler can be bound to many events at the same time.

elasticlunr.Index.prototype.on = function () {
  var args = Array.prototype.slice.call(arguments);
  return this.eventEmitter.addListener.apply(this.eventEmitter, args);
};

off

method
elasticlunr.Index.prototype.off()

Option name Type Description
eventName String

The name of events to remove the function from.

fn Function

The serialised set to load.

Removes a handler from an event being emitted by the index.

elasticlunr.Index.prototype.off = function (name, fn) {
  return this.eventEmitter.removeListener(name, fn);
};

load

method
elasticlunr.Index.load()

Option name Type Description
serialisedData Object

The serialised set to load.

return elasticlunr.Index

Loads a previously serialised index.

Issues a warning if the index being imported was serialised
by a different version of elasticlunr.

elasticlunr.Index.load = function (serialisedData) {
  if (serialisedData.version !== elasticlunr.version) {
    elasticlunr.utils.warn('version mismatch: current ' 
                    + elasticlunr.version + ' importing ' + serialisedData.version);
  }

  var idx = new this;

  idx._fields = serialisedData.fields;
  idx._ref = serialisedData.ref;
  idx.documentStore = elasticlunr.DocumentStore.load(serialisedData.documentStore);
  idx.pipeline = elasticlunr.Pipeline.load(serialisedData.pipeline);
  idx.index = {};
  for (var field in serialisedData.index) {
    idx.index[field] = elasticlunr.InvertedIndex.load(serialisedData.index[field]);
  }

  return idx;
};

addField

method
elasticlunr.Index.prototype.addField()

Option name Type Description
fieldName String

The name of the field within the document that should be indexed

return elasticlunr.Index

Adds a field to the list of fields that will be searchable within documents in the index.

Remember that inner index is build based on field, which means each field has one inverted index.

Fields should be added before any documents are added to the index, fields
that are added after documents are added to the index will only apply to new
documents added to the index.

elasticlunr.Index.prototype.addField = function (fieldName) {
  this._fields.push(fieldName);
  this.index[fieldName] = new elasticlunr.InvertedIndex;
  return this;
};

setRef

method
elasticlunr.Index.prototype.setRef()

Option name Type Description
refName String

The property to use to uniquely identify the documents in the index.

emitEvent Boolean

Whether to emit add events, defaults to true

return elasticlunr.Index

Sets the property used to uniquely identify documents added to the index,
by default this property is 'id'.

This should only be changed before adding documents to the index, changing
the ref property without resetting the index can lead to unexpected results.

elasticlunr.Index.prototype.setRef = function (refName) {
  this._ref = refName;
  return this;
};

addDoc

method
elasticlunr.Index.prototype.addDoc()

Option name Type Description
doc Object

The JSON format document to add to the index.

emitEvent Boolean

Whether or not to emit events, default true.

Add a JSON format document to the index.

This is the way new documents enter the index, this function will run the
fields from the document through the index's pipeline and then add it to
the index, it will then show up in search results.

An 'add' event is emitted with the document that has been added and the index
the document has been added to. This event can be silenced by passing false
as the second argument to add.

elasticlunr.Index.prototype.addDoc = function (doc, emitEvent) {
  if (!doc) return;
  var emitEvent = emitEvent === undefined ? true : emitEvent;

  var docRef = doc[this._ref];
  this.documentStore.addDoc(docRef, doc);
  this._fields.forEach(function (field) {
    var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
    var fieldLength = fieldTokens.length;
    var tokenCount = {};
    fieldTokens.forEach(function (token) {
      if (token in tokenCount) tokenCount[token] += 1;
      else tokenCount[token] = 1;
    }, this);

    for (var token in tokenCount) {
      var tf = tokenCount[token];
      tf = Math.sqrt(tf);
      this.index[field].addToken(token, { ref: docRef, tf: tf });
    }
  }, this);

  if (emitEvent) this.eventEmitter.emit('add', doc, this);
};

removeDoc

method
elasticlunr.Index.prototype.removeDoc()

Option name Type Description
docRef Object

The document to remove from the index.

emitEvent Boolean

Whether to emit remove events, defaults to true

Removes a document from the index.

To make sure documents no longer show up in search results they can be
removed from the index using this method.

A 'remove' event is emitted with the document that has been removed and the index
the document has been removed from. This event can be silenced by passing false
as the second argument to remove.

elasticlunr.Index.prototype.removeDoc = function (doc, emitEvent) {
  if (!doc) return;
  var emitEvent = emitEvent === undefined ? true : emitEvent;

  var docRef = doc[this._ref];
  if (!this.documentStore.hasDoc(docRef)) return;

  this.documentStore.removeDoc(docRef);

  this._fields.forEach(function (field) {
    var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
    for (var token in fieldTokens) {
      this.index[field].removeToken(token);
    }
  }, this);

  if (emitEvent) this.eventEmitter.emit('remove', doc, this);
};

update

method
elasticlunr.Index.prototype.update()

Option name Type Description
doc Object

The document to update in the index.

emitEvent Boolean

Whether to emit update events, defaults to true

Updates a document in the index.

When a document contained within the index gets updated, fields changed,
added or removed, to make sure it correctly matched against search queries,
it should be updated in the index.

This method is just a wrapper around remove and add

An 'update' event is emitted with the document that has been updated and the index.
This event can be silenced by passing false as the second argument to update. Only
an update event will be fired, the 'add' and 'remove' events of the underlying calls
are silenced.

elasticlunr.Index.prototype.update = function (doc, emitEvent) {
  var emitEvent = emitEvent === undefined ? true : emitEvent;

  this.removeDoc(doc[this._ref], false);
  this.addDoc(doc, false);

  if (emitEvent) this.eventEmitter.emit('update', doc, this);
};
Option name Type Description
query String

The query to search the index with.

config Object

The query config, JSON format.

return Object

Searches the index using the passed query.
Queries should be a string, multiple words are allowed.

If config is null, will search all fields defaultly, and lead to AND based query.
If config is specified, will search specified with query time boosting.

All query tokens are passed through the same pipeline that document tokens
are passed through, so any language processing involved will be run on every
query term.

Each query term is expanded, so that the term 'he' might be expanded to
'hello' and 'help' if those terms were already included in the index.

Matching documents are returned as an array of objects, each object contains
the matching document ref, as set for this index, and the similarity score
for this document against the query.

elasticlunr.Index.prototype.search = function (query, config) {
  if (!query) return [];
  var queryTokens = this.pipeline.run(elasticlunr.tokenizer(query));

  var searchFields = {};
  if (config == null || config['fields'] == null) {
    this._fields.forEach(function (field) {
      searchFields[field] = {boost: 1};
    }, this);
  } else {
    searchFields = config['fields'];
  }

  var queryResults = {};
  var squaredWeight = this.computeSquaredWeight(queryTokens, searchFields);

  for (var field in searchFields) {
    var fieldSearchResults = this.fieldSearch(queryTokens, field, config);
    var fieldSearchScores = {};
    var fieldBoost = searchFields[field].boost || 1;
    var queryNorm = 1 / Math.sqrt(1 / (fieldBoost * fieldBoost) * squaredWeight);
    console.log('queryNorm: ' + queryNorm +' in field:' + field);

    for (var docRef in fieldSearchResults) {
      var score = this.computeScore(queryTokens, docRef, field);
      score = queryNorm * score;
      fieldSearchScores[docRef] = score;
    }

    for (var docRef in fieldSearchScores) {
      if (docRef in queryResults) {
        queryResults[docRef] += fieldSearchScores[docRef];
      } else {
        queryResults[docRef] = fieldSearchScores[docRef];
      }
    }
  }

  var results = [];
  for (var docRef in queryResults) {
    results.push({ref: docRef, score: queryResults[docRef]});
  }

  results.sort(function (a, b) { return b.score - a.score });
  return results;
};

computeScore

method
elasticlunr.Index.prototype.computeScore()

Option name Type Description
queryTokens Array
docRef String,Integer
field String

compute the score of a documents in given field.

elasticlunr.Index.prototype.computeScore = function (queryTokens, docRef, field) {
  var doc = this.documentStore.getDoc(docRef);

  var coord = 0;
  var fieldTokens = this.pipeline.run(elasticlunr.tokenizer(doc[field]));
  queryTokens.forEach(function (token) {
    if (fieldTokens.indexOf(token) != -1) coord += 1;
  }, this);

  coord = coord / queryTokens.length;

  var score = 0.0;
  queryTokens.forEach(function name(token) {
    var tf = this.index[field].getTF(token, docRef);
    var idf = this.idf(token, field);
    var norm = 1 / Math.sqrt(fieldTokens.length);
    score += tf * idf * norm;
  }, this);

  score *= coord;
  return score;
};

computeSquaredWeight

method
elasticlunr.Index.prototype.computeSquaredWeight()

Option name Type Description
queryTokens Array

query tokens.

searchFields Object

fields to search.

return Float

compute squared weight of query tokens.

elasticlunr.Index.prototype.computeSquaredWeight = function (queryTokens, searchFields) {
  var weight = 0.0;
  queryTokens.forEach(function (token) {
    var fieldWeight = 0.0;
    for (var field in searchFields) {
      var fieldBoost = searchFields[field].boost || 1;
      var idf = this.idf(token, field);
      fieldWeight += idf * idf * fieldBoost * fieldBoost;
    }
    weight += fieldWeight;
  }, this);

  return weight;
};

fieldSearch

method
elasticlunr.Index.prototype.fieldSearch()

Option name Type Description
queryTokens Array

The query tokens to query in this field.

field String

Field to query in.

config Object

search config.

return Object

search queryTokens in specified field.

elasticlunr.Index.prototype.fieldSearch = function (queryTokens, fieldName, config) {
  var booleanType = (config == null || config['boolean'] == null) ? 'OR' : config['boolean'];
  var tokenResults = {};

  queryTokens.forEach(function (token) {
    var docs = this.index[fieldName].getDocs(token);
    tokenResults[token] = docs; 
  }, this);

  var res = {};
  if (booleanType == 'OR') {
    res = this.union(tokenResults);
  } else if (booleanType == 'AND') {
    res = this.intersect(tokenResults, queryTokens);
  }
  
  return res;
};

union

method
elasticlunr.Index.prototype.union()

Option name Type Description
tokenResults Object

results of all query tokens

return Object

union tokenResults among each token results.

elasticlunr.Index.prototype.union = function (tokenResults) {
  var res = {};
  for (var token in tokenResults) {
    for (var docRef in tokenResults[token]) {
      res[docRef] = true;
    }
  }

  return res;
};

intersect

method
elasticlunr.Index.prototype.intersect()

Option name Type Description
results Object

first results

docs Object

field search results of a token

return Object

intersect the two results

elasticlunr.Index.prototype.intersect = function (tokenResults, queryTokens) {
  var res = {};
  var token = queryTokens[0];
  for (var docRef in tokenResults[token]) {
    var flag = true;
    for (var key in tokenResults) {
      if (!(docRef in tokenResults[key])) {
        flag = false;
        break;
      }
    }

    if (flag == true) {
      res[docRef] = true;
    }
  }

  return res;
};

toJSON

method
elasticlunr.Index.prototype.toJSON()

Returns a representation of the index ready for serialisation.

elasticlunr.Index.prototype.toJSON = function () {
  var indexJson = {};
  for (var idx in this._fields) {
    var fieldName = this._fields[idx];
    indexJson[fieldName] = this.index[fieldName].toJSON();
  }

  return {
    version: elasticlunr.version,
    fields: this._fields,
    ref: this._ref,
    documentStore: this.documentStore.toJSON(),
    index: indexJson,
    pipeline: this.pipeline.toJSON()
  };
};

use

method
elasticlunr.Index.prototype.use()

Option name Type Description
plugin Function

The plugin to apply.

Applies a plugin to the current index.

A plugin is a function that is called with the index as its context.
Plugins can be used to customise or extend the behaviour the index
in some way. A plugin is just a function, that encapsulated the custom
behaviour that should be applied to the index.

The plugin function will be called with the index as its argument, additional
arguments can also be passed when calling use. The function will be called
with the index as its context.

Example:

var myPlugin = function (idx, arg1, arg2) {
  // `this` is the index to be extended
  // apply any extensions etc here.
}

var idx = lunr(function () {
  this.use(myPlugin, 'arg1', 'arg2')
})
elasticlunr.Index.prototype.use = function (plugin) {
  var args = Array.prototype.slice.call(arguments, 1)
  args.unshift(this)
  plugin.apply(this, args)
}