d057542001-06-04Johan Schön void index_document(Search.Database.Base db,
4779302001-06-05Per Hedbor  string|Standards.URI uri, void|string language, mapping fields, mapping uri_anchors)
90f5642001-05-17Johan Schön {
57ab6d2001-05-26Per Hedbor  db->remove_document( uri, language );
d057542001-06-04Johan Schön  foreach(indices(fields), string field)
57ab6d2001-05-26Per Hedbor  {
d057542001-06-04Johan Schön  string f; if( strlen(f = fields[field] ) )
57ab6d2001-05-26Per Hedbor  {
4779302001-06-05Per Hedbor  array words=Search.Utils.tokenize_and_normalize( f );
d057542001-06-04Johan Schön  db->insert_words(uri, language, field, words );
57ab6d2001-05-26Per Hedbor  } }
90f5642001-05-17Johan Schön  // Tokenize any anchor fields
d057542001-06-04Johan Schön  int source_hash=hash((string)uri)&0xf; foreach(indices(uri_anchors|| ({ })), string link_uri) { array(string) words=
4779302001-06-05Per Hedbor  Search.Utils.tokenize_and_normalize(uri_anchors[link_uri]);
57ab6d2001-05-26Per Hedbor  db->insert_words(link_uri, 0, "anchor", words, source_hash);
90f5642001-05-17Johan Schön  } }
d057542001-06-04Johan Schön  array(Standards.URI) filter_and_extract_links(Search.Database.Base db, string|Standards.URI uri, void|string language, string|Stdio.File data, string content_type)
90f5642001-05-17Johan Schön { Search.Filter.Base filter=Search.get_filter(content_type); if(!filter) throw("No indexer for content type "+content_type); Search.Filter.Base.Output filteroutput=filter->filter(uri, data, content_type);
d057542001-06-04Johan Schön  index_document(db, uri, language, filteroutput->fields, filteroutput->uri_anchors);
90f5642001-05-17Johan Schön  return filteroutput->links; }
eac31a2001-05-31Johan Schön void remove_document(Search.Database.Base db,
90f5642001-05-17Johan Schön  string|Standards.URI|string uri, void|string language) { db->remove_document(uri, language); }