<?php

/*
** DGS Search
** $Id: indexer.php,v 1.2 2006/01/24 07:08:09 sella Exp $
** Copyright (c) 2000-2001 Digital Genesis Software, LLC. All Rights Reserved.
** Released under the GPL Version 2 License.
** http://www.digitalgenesis.com
*/

function mergeFileIntoIndexFile($file, $indexFile, $fpIn, $fpOut) {
	global $config;
	
	if (!getFileIndex($fileIndex, $indexFile)) {
		if ($config['debug']) {
			printf('<B>mergeFileIntoIndexFile:</B> Failed to create list of file names from index file (<B>%s</B>).<BR>', $indexFile);
		}
		return false;
	} else if (!getNextOffset($keywordIndexOffset, $fpIn, 0)) {
		if ($config['debug']) {
			printf('<B>mergeFileIntoIndexFile:</B> Failed to get keyword index from master index file (<B>%s</B>).<BR>', $indexFile);
		}
		return false;
	} else if (!getFilenameOffset($fileIndexOffset, $fpIn)) {
		if ($config['debug']) {
			printf('<B>mergeFileIntoIndexFile:</B> Failed to get file name index from master index file (<B>%s</B>).<BR>', $indexFile);
		}
		return false;
	} else if (!readIndexFile($keywordIndex, getIndexFileName($file))) {
		if ($config['debug']) {
			printf('<B>mergeFileIntoIndexFile:</B> Could not read index file for (<B>%s</B>); it will not be indexed.<BR>', $file);
		}
		return false;
	} else if (!isset($fileIndex[$file])) {
		if ($config['debug']) {
			printf('<B>mergeFileIntoIndexFile:</B> File (<B>%s</B>) has never been indexed; performing initial indexing.<BR>', $file);
		}
		$fileIndex[$file] = end($fileIndex) + strlen(key($fileIndex)) + 1;
	}

	if (!mergeKeywordIndex($file, $indexFile, $fpIn, $fpOut)) {
		if ($config['debug']) {
			printf('<B>mergeFileIntoIndexFile:</B> Could not merge keyword index for (<B>%s</B>); it will not be indexed.<BR>', $file);
		}
		return false;
	} else if (!mergeFileIndex()) {
		if ($config['debug']) {
			printf('<B>mergeFileIntoIndexFile:</B> Could not merge file index for (<B>%s</B>); it will not be indexed.<BR>', $file);
		}
		return false;
	} else if (!mergeQuickIndex()) {
		if ($config['debug']) {
			printf('<B>mergeFileIntoIndexFile:</B> Could not merge quick index for (<B>%s</B>); it will not be indexed.<BR>', $file);
		}
		return false;
	}
	return true;
}

function mergeFileIndex() {
	return true;
}

function mergeQuickIndex() {
	return true;
}

function copyKeywordRecord($fpOut, $fpIn, $keyword) {
	global $config;

	if (!getKeywordOffset($offset, $fpIn, $keyword)) {
		if ($config['debug']) {
			printf('Failed to get offset of <B>%s</B>.<BR>', $keyword);
		}
		return false;
	} else if (!getNextOffset($nextOffset, $fpIn, $offset)) {
		if ($config['debug']) {
			printf('Failed to get next offset after <B>%s</B>.<BR>', $keyword);
		}
		return false;
	} else if (!getKeywordFilenameCount($fileCount, $fpIn, $keyword)) {
		if ($config['debug']) {
			printf('Failed to get number of files <B>%s</B> appeared in from.<BR>', $keyword);
		}
		return false;
	} else if (!getKeywordTotal($occurances, $fpIn, $keyword)) {
		if ($config['debug']) {
			printf('Failed to get total number of occuraces of <B>%s</B> from source.<BR>', $masterKeyword, $masterIndexFile);
		}
		return false;
	}
	if (!getKeywordFilenameCountArray($fileNames, $fpIn, $keyword)) {
		if ($config['debug']) {
			printf('Failed to get list of files %s appeared in from %s.<BR>', $masterKeyword, $masterIndexFile);
		}
		return false;
	}

	$next = ftell($fpIn) + 12;
	$binary = '';
	$WRITE = '';
	foreach ($fileNames as $offset => $occurances) {
		$next += 8;

		if (!getFilename($filename, $fpIn, $offset)) {
			if ($config['debug']) {
				printf('Failed to get file name at filename offset 0x%x.<BR>', $offset);
			}
			return false;
		}
		if ($config['debug']) {
			printf('Adding %s as a file name containing %s (%d occurances).<BR>', $filename, $keyword, $occurances);
		}
		$WRITE .= sprintf(' [File Offset %d(%s)] [Occurances: %d]', $filenameIndex[$filename], $filename, $occurances);
		$binary .= pack('L2', $filenameIndex[$filename], $occurances);
	}

	$format = sprintf("L3a%d", strlen($keyword) + 1);
	$WRITE = sprintf('[Next Offset: %d] [File Count: %d] [Total Occurances: %d] [Keyword: %s]', $nextOffset, $fileCount, $occurances, $keyword) . $WRITE;
	$binary = pack($format, $nextOffset, $fileCount, $occurances, $keyword) . $binary;

	if ($config['debug']) {
		printf('<B>Writing[copy] (0x%x): %s</B><BR>', ftell($fpOut), $WRITE);
	}
	if (fwrite($fpOut, $binary) == -1) {
		printf('<B>Failure writing to %s.swp: check permissions and disk capacity.</B><BR>', $masterIndexFile);

		return false;
	}
			
	$relativeOffset += $nextOffset;
	if ($masterKeywordOffset + $relativeOffset < $masterFileOffset) {
		if (!getKeywordAtOffset($masterKeyword, $fpMaster, $masterKeywordOffset + $relativeOffset)) {
			if ($config['debug']) {
				printf('Failure reading next keyword from %s.<BR>', $masterIndexFile);
			}
			return false;
		}
	} else {
		$masterKeyword = false;
	}
	return true;
}

function getKeywordCount($mergeFile, $indexFile, $fpIndexFile) {
	global $config;

	if (!readIndexFile($mergeKeywordIndex, getIndexFileName($mergeFile))) {
		if ($config['debug']) {
			printf("<B>mergeKeywordIndex:</B> could not read sub-index for file %s.", $mergeFile);
		}
		return false;
	}
	$mergeKeywordCount = count($mergeKeywordIndex);

	getNextOffset($keywordOffset, $fpIndexFile, 0);
	getFilenameOffset($filenameOffset, $fpIndexFile);
	for ($offset = $keywordOffset; $offset < $filenameOffset; $offset += $offsetToNext) {
		if (!getKeywordAtOffset($keyword, $fpIndexFile, $offset)) {
			if ($config['debug']) {
				printf('Could not get keyword at offset 0x%x from %s: file is damged.<BR>', $offset, $indexFile);
			}
			return false;
		}
		if (!isset($mergeKeywordIndex[$keyword])) {
			$keywordCount++;
		}
		if (!getNextOffset($offsetToNext, $fpIndexFile, $offset)) {
			if ($config['debug']) {
				printf('Could not get next offset at offset 0x%x from %s: file is damged.<BR>', $offset, $indexFile);
			}
			return false;
		}
	}
	return $keywordCount;
}

function mergeKeywordIndex($mergeFile, $masterIndexFile, $fpMaster, $fp) {
	global $config;

	if (!readIndexFile($keywordIndex, getIndexFileName($mergeFile))) {
		if ($config['debug'])
			printf('Could not index %s.<BR>', $file);

		return false;
	}
	$keywordCount = count($keywordIndex);
	getNextOffset($masterKeywordOffset, $fpMaster, 0);
	getFilenameOffset($masterFileOffset, $fpMaster);

	for ($offset = $masterKeywordOffset; $offset < $masterFileOffset; $offset += $next) {
		if (!getKeywordAtOffset($keyword, $fpMaster, $offset)) {
			if ($config['debug']) {
				printf('Could not get keyword at offset 0x%x from %s: file is damged.<BR>', $offset, $masterIndexFile);
			}
			return false;
		}
		if (!isset($keywordIndex[$keyword])) {	
			$keywordCount++;
		}
		if (!getNextOffset($next, $fpMaster, $offset)) {
			if ($config['debug']) {
				printf('Could not get next offset at offset 0x%x from %s: file is damged.<BR>', $offset, $masterIndexFile);
			}
			return false;
		}
	}

	getNextOffset($next, $fpMaster, $masterFileOffset);
	for ($offset = 0; $next != 0; $offset += $next) {
		getNextOffset($next, $fpMaster, $masterFileOffset + $offset);

		getFilename($filename, $fpMaster, $offset);

		if ($filename != '')
			$filenameIndex[$filename] = $offset;
	}
	$filenameIndex[$mergeFile] = $offset;

	$quickIndexStep = $keywordCount > ($config['partitions']) ? ($config['partitions']) / $keywordCount : 1;
	$quickWordSize = 4;

	if (fseek($fp, 12 + (int)($keywordCount * $quickIndexStep * (4 + $quickWordSize))) == -1) {
		if ($config['debug']) {
			printf('Could not perform jump to 0x%x in %s.<BR>', 12 + (int)($keywordCount * $quickIndexStep * (4 + $quickWordSize)), $masterIndexFile);
		}
		return false;
	}

	if ($config['debug']) {
		printf('<B>Total Keyword Count: %d</B><BR>', $keywordCount);
		printf('<B>Total File Count: %d</B><BR>', count($filenameIndex));
		printf('<B>Quick Index Step: %f</B><BR>', $quickIndexStep);
		printf('<B>First Keyword Location: 0x%x</B><BR>', 12 + (int)($keywordCount * $quickIndexStep * (4 + $quickWordSize)));
	}

	$nextMasterKeywordOffset = $masterKeywordOffset;
	reset($keywordIndex);
	$mergingKeyword = key($keywordIndex);
	if (!getKeywordAtOffset($masterKeyword, $fpMaster, $masterKeywordOffset)) {
		if ($config['debug']) {
			printf('Failed to find first keyword at 0x%x in %s.<BR>', $masterKeywordOffset, $masterIndexFile);
		}
		return false;
	}

	$keywordNumber = -1;
	$relativeOffset = 0;
	while ($mergingKeyword && $masterKeyword) {
		if ($config['debug']) {
			printf('Comparing "%s" and "%s".<BR>', $masterKeyword, $mergingKeyword);
		}
		$comparison = strcmp($masterKeyword, $mergingKeyword);

		if ((int)$keywordNumber < (int)($keywordNumber += ($quickIndexStep))) {
			if ($config['debug']) {
				printf('Adding %s to quick index with offset 0x%x.<BR>', $comparison > 0 ? $mergingKeyword : $masterKeyword, ftell($fp));
			}
			$quickIndex[$comparison > 0 ? $mergingKeyword : $masterKeyword] = ftell($fp);
		}

		if ($config['debug']) {
			printf('Comparison of "%s" and "%s" returned %s.<BR>', $masterKeyword, $mergingKeyword, $comparison);
		}
		if ($comparison < 0) {
			if ($config['debug']) {
				printf('Copying record for keyword "%s" from old index file to new index file.<BR>', $masterKeyword);
			}
			if (!copyKeywordRecord($fp, $fpMaster, $masterKeyword)) {
				if ($config['debug']) {
					printf('Faild to copy record for keyword "%s" from old index file to new index file.<BR>', $masterKeyword);
				}
				return false;
			}
			getNextOffset($nextOffset, $fpMaster, $nextMasterKeywordOffset);

			$nextMasterKeywordOffset += $nextOffset;
			if ($nextMasterKeywordOffset < $masterFileOffset) {
				if (!getKeywordAtOffset($masterKeyword, $fpMaster, $nextMasterKeywordOffset)) {
					if ($config['debug']) {
						printf('Failed to find keyword at 0x%x in %s.<BR>', $nextMasterKeywordOffset);
					}
					return false;
				}
			} else {
				$masterKeyword = false;
			}
		} else if ($comparison > 0) {
			if ($config['debug']) {
					printf('Writing record for %s from level 1 index file to new master index file.<BR>', $mergingKeyword);
			}
			$nextOffset = 21 + strlen($mergingKeyword);

			$format = sprintf("L3a%dL2", strlen($mergingKeyword) + 1);
			$WRITE = sprintf('[Next Offset: %d] [File Count: %d] [Total Occurances: %s] [Keyword: %s] [File Offset: %d(%s)] [Occurances: %d]', $nextOffset, 1, $keywordIndex[$mergingKeyword], $mergingKeyword, $filenameIndex[$mergeFile], $mergeFile, $keywordIndex[$mergingKeyword]);
			$binary = pack($format, $nextOffset, 1, $keywordIndex[$mergingKeyword], $mergingKeyword, $filenameIndex[$mergeFile], $keywordIndex[$mergingKeyword]);

			if ($config['debug']) {
				printf('<B>Writing [add] (0x%x): %s</B><BR>', ftell($fp), $WRITE);
			}
			if (fwrite($fp, $binary) == -1) {
				printf('<B>Failure writing to %s.swp: check permissions and disk capacity.</B><BR>', $masterIndexFile);	

				return false;
			}
			if (next($keywordIndex)) {
				$mergingKeyword = key($keywordIndex);
			} else {
				$mergingKeyword = false;
			}
		} else {
			if ($config['debug']) {
					printf('Writing record for %s from merge of old master index file and layer 1 index file.<BR>', $mergingKeyword);
			}
			if (!getKeywordTotal($totalOccurances, $fpMaster, $masterKeyword)) {
				if ($config['debug']) {
					printf('Failed to get total number of occuraces of %s from %s.<BR>', $masterKeyword, $masterIndexFile);
				}
				return false;
			}
			if (!getKeywordFilenameCount($fileCount, $fpMaster, $masterKeyword)) {
				if ($config['debug']) {
					printf('Failed to get number of files %s appeared in from %s.<BR>', $masterKeyword, $masterIndexFile);
				}
				return false;
			}
			if (!getKeywordFilenameCountArray($fileList, $fpMaster, $masterKeyword)) {
				if ($config['debug']) {
					printf('Failed to get list of files %s appeared in from %s.<BR>', $masterKeyword, $masterIndexFile);
				}
				return false;
			}

			$binary = "";
			$WRITE = '';
			$nextWriteOffset = strlen($masterKeyword) + 13;
			$alreadyMerged = false;
			foreach ($fileList as $fileOffset => $occurances) {
				$nextWriteOffset += 8;

				if (!getFilename($filename, $fpMaster, $fileOffset)) {
					if ($config['debug']) {
						printf('Failed to get file name at file index offset 0x%x.<BR>', $fileOffset);
					}
					return false;
				}
				if (strcmp($filename, $mergeFile) != 0) {
					$WRITE .= sprintf(' [File Index: %d(%s)] [Occurances: %d]', $filenameIndex[$filename], $filename, $occurances);
					$binary .= pack('L2', $filenameIndex[$filename], $occurances);
				} else {
					$alreadyMerged = true;
					if ($occurances == $keywordIndex[$masterKeyword]) {
						$WRITE .= sprintf(' [File Index: %d(%s)] [Occurances: %d]', $filenameIndex[$filename], $filename, $occurances);
						$binary .= pack('L2', $filenameIndex[$filename], $occurances);
					} else if ($occurances < $keywordIndex[$masterKeyword]) {
						$WRITE .= sprintf(' [File Index: %d(%s)] [Occurances: %d]', $filenameIndex[$filename], $filename, $keywordIndex[$masterKeyword]);
						$binary .= pack('L2', $filenameIndex[$filename], $keywordIndex[$masterKeyword]);
			
						$totalOccurances += $keywordIndex[$masterKeyword] - $occurances;
					} else {
						$WRITE .= sprintf(' [File Index: %d(%s)] [Occurances: %d]', $filenameIndex[$filename], $filename, $keywordIndex[$masterKeyword]);
						$binary .= pack('L2', $filenameIndex[$filename], $keywordIndex[$masterKeyword]);

						$totalOccurances -= $occurances - $keywordIndex[$masterKeyword];
					}	
				}
			}
			if (!$alreadMerged) {
				$nextWriteOffset += 8;
				$fileCount++;
				$WRITE .= sprintf(' [File Index: %d(%s)] [Occurances: %d]', $filenameIndex[$mergeFile], $filename, $keywordIndex[$masterKeyword]);
				$binary .= pack('L2', $filenameIndex[$mergeFile], $keywordIndex[$masterKeyword]);

				$totalOccurances += $keywordIndex[$masterKeyword];
			}

			$format = sprintf("L3a%d", strlen($masterKeyword) + 1);
			$WRITE = sprintf('[Next Offset: %d] [File Count: %d] [Total Occurances: %d] [Keyword: %s]', $nextWriteOffset, $fileCount, $totalOccurances, $masterKeyword) . $WRITE;
			$binary = pack($format, $nextWriteOffset, $fileCount, $totalOccurances, $masterKeyword) . $binary;

			if ($config['debug']) {
				printf('<B>Writing [merge] (0x%x): %s</B><BR>', ftell($fp), $WRITE);
			}
			if (fwrite($fp, $binary) == -1) {
				printf('<B>Failure writing to %s.swp: check permissions and disk capacity.</B><BR>', $masterIndexFile);	

				return false;
			}
			getNextOffset($nextOffset, $fpMaster, $nextMasterKeywordOffset);

			$nextMasterKeywordOffset += $nextOffset;
			if ($nextMasterKeywordOffset < $masterFileOffset) {
				if (!getKeywordAtOffset($masterKeyword, $fpMaster, $nextMasterKeywordOffset)) {
					if ($config['debug']) {
						printf('Failed to find keyword at 0x%x in %s.<BR>', $nextMasterKeywordOffset);
					}
					return false;
				}
			} else {
				$masterKeyword = false;
			}

			if (next($keywordIndex)) {
				$mergingKeyword = key($keywordIndex);
			} else {
				$mergingKeyword = false;
			}
		}
	}

	if ($mergingKeyword) {
		do {
			if ((int)$keywordNumber < (int)($keywordNumber += (1 / $quickIndexStep))) {
				$quickIndex[$mergingKeyword] = ftell($fp);
			}
			$format = sprintf("L3a%dL2", strlen($mergingKeyword) + 1);

			$WRITE = sprintf('[Next Offset: %d] [File Count: %d] [Total Occurances: %d] [Keyword: %s] [File Offset: %d(%s)] [Occurances: %s]', $nextOffset, 1, $keywordIndex[$mergingKeyword], $mergingKeyword, $filenameIndex[$mergeFile], $mergeFile, $keywordIndex[$mergingKeyword]);
			$binary = pack($format, $nextOffset, 1, $keywordIndex[$mergingKeyword], $mergingKeyword, $filenameIndex[$mergeFile], $keywordIndex[$mergingKeyword]);
		
			if ($config['debug']) {	
				printf('<B>Writing [add] (0x%x): %s</B><BR>', ftell($fp), $WRITE);	
			}
			fwrite($fp, $binary);
		} while (next($keywordIndex));
	} else if ($masterKeyword) {
		do {
			if ($config['debug']) {
				printf('Copying record for keyword "%s" from old index file to new index file.<BR>', $masterKeyword);
			}
			if (!copyKeywordRecord($fp, $fpMaster, $masterKeyword)) {
				if ($config['debug']) {
					printf('Faild to copy record for keyword "%s" from old index file to new index file.<BR>', $masterKeyword);
				}
				return false;
			}
			getNextOffset($nextOffset, $fpMaster, $nextMasterKeywordOffset);

			$nextMasterKeywordOffset += $nextOffset;
			if ($nextMasterKeywordOffset < $masterFileOffset) {
				if (!getKeywordAtOffset($masterKeyword, $fpMaster, $nextMasterKeywordOffset)) {
					if ($config['debug']) {
						printf('Failed to find keyword at 0x%x in %s.<BR>', $nextMasterKeywordOffset);
					}
					return false;
				}
			} else {
				$masterKeyword = false;
			}
		} while (($masterKeywordOffset + $relativeOffset) < $masterFileOffset);
	}

	$filenameOffset = ftell($fp);	
	foreach ($filenameIndex as $filename => $count) {
		$filenameSize = strlen($filename) + 1;
		$format = sprintf("La%d", $filenameSize);
		$binary = pack($format, $filenameSize + 4, $filename);
	
		fwrite($fp, $binary);
	}
	fwrite($fp, pack('L', 0));

	fseek($fp, 0);
	$binary = pack('LL', 12 + (int)($keywordCount * $quickIndexStep * (4 + $quickWordSize)), count($quickIndex) * $quickIndexStep);
	$format = sprintf("a%dL",  $quickWordSize);
	foreach ($quickIndex as $keyword => $offset) {
		$binary .= pack($format, $keyword, $offset);
	}
	$binary .= pack('L', $filenameOffset);

 	fwrite($fp, $binary);

	if (rename($config['indexFile'] . '.swp', $config['indexFile']) == true) {
		return true;
	} else {
		return false;
	}
}

function indexFile($mergeFile, $masterIndexFile) {
	global $config;

	if (!($fp = fopen($masterIndexFile . '.swp', 'wb'))) {
		printf('<B>Could not open index swap file, check permissions and disk capacity for %s.</B><BR>', $masterIndexFile . '.swp');

		return false;

	} else if (!($fpMaster = @fopen($masterIndexFile, 'rb'))) {
		if ($config['debug'])
			printf('Master index file (%s) not found; creating master index file from %s.<BR>', $masterIndexFile, $mergeFile);

		if (!createMasterIndexFromFile($fp, $mergeFile)) {
			printf('<B>Failed to create initial master index file (%s) from %s.</B><BR>', $masterIndexFile, $mergeFile);

			fclose($fp);
			return false;
		}
		fclose($fp);
	
		if ($config['debug'])
			printf('Moving swapped master index file (%s) into production.<BR>', $masterIndexFile);

		rename($masterIndexFile . '.swp', $masterIndexFile);
		return true;
	}
	if (!getNextOffset($masterKeywordOffset, $fpMaster, 0)) {
		printf('<B>Could not find keyword index in %s: file is damaged (delete and re-index).</B><BR>', $masterIndexFile);

		fclose($fp);
		return false;
	}
	if (!getFilenameOffset($masterFileOffset, $fpMaster)) {
		printf('<B>Could not find file name index in %s: file is damaged (delete and re-index).</B><BR>', $masterIndexFile);

		fclose($fp);
		return false; 
	}
	if (!mergeFileIntoIndexFile($mergeFile, $masterIndexFile, $fpMaster, $fp)) {
		printf('<B>Could not merge %s into %s: check permissions and disk capacity.</B><BR>', $mergeFile, $masterIndexFile);

		fclose($fp);
		return false; 
	}

	fclose($fp);
	return true;
}

function isIndexing() {
	global $config;

	if (file_exists($config['indexDir'] . $config['fileSeparator'] . '.indexing')) {
		return true;
	} else {
		return false;
	}
}

function alreadyIndexed() {
	global $config;

	/* This should be checked for writting. */
	if (is_dir($config['indexDir']) && is_readable($config['indexFile'])) {
		return true;
	} else {
		return false;
	}
}

function initializeIndex() {
	global $config;

	if (is_dir($config['indexDir']) || mkdir($config['indexDir'], 0700)) {
		return true;
	} else {
		return false;
	}
}

function getIndexFileName($filename) {
	global $config;

	return $config['indexDir'] . $config['fileSeparator'] . md5($filename) . '.idx';
}

function getFilesToIndex(&$fileList) {
	global $config;
	$result = array();
	$isMasterIndex = is_file($config['indexFile']);

	foreach($fileList as $file) {
		$indexFile = getIndexFileName($file);

		if (filemtime($file) > filectime($indexFile) || filectime($file) > filectime($indexFile)) {
			$result[] = $file;
		}

		$indexFile = "";
	}
	return $result;
}

function indexFiles(&$fileList) {
	foreach ($fileList as $file) {
		if (!refreshIndexingLock())
			return false;
		if (($text = getFileText($file)) != false) { 
			subIndexFile($fileIndex, $text);
			$text = "";

			writeIndexFile($fileIndex, getIndexFileName($file));
			$file = "";
			$fileIndex = array();
		}
	}
}

function generateMasterIndex(&$masterIndex, &$fileList) {
	global $config;

	foreach($fileList as $file) {
		if (!refreshIndexingLock())
			return false;
		if (readIndexFile($fileIndex, getIndexFileName($file))) {
			mergeIndex($masterIndex, $fileIndex, $file);
		}
		$file = "";
		$fileIndex = array();
	}
	ksort($masterIndex['keywords']);
	
	return true;
}

function indexFileExists() {
	if (file_exists($config['indexFile'])) {
		return true;
	} else {
		return false;
	}
}

function createIndex() {
	global $config;

	if (isIndexing()) {
		return false;
	} else {
		if (is_dir($config['indexDir']) == false) {
			if (!initializeIndex()) {
				return false;
			}
		}
		if (!requestIndexingLock()) {
			return false;
		}
	}

	if (!alreadyIndexed()) {
		if (!initializeIndex()) {
			releaseIndexingLock();
			return false;
		}
	}

	$fileList = findext($config['fsBase'], $config['exts']);
	$needToIndex = getFilesToIndex($fileList);
	if (count($needToIndex) == 0) {
		if (indexFileExists()) {
			releaseIndexingLock();
			return false;
		}
	}
	indexFiles($needToIndex);

	if (1) {
		if ($config['debug']) {
			printf("%d files require indexing.<BR>", count($needToIndex));
		}
		foreach($needToIndex as $filename) {
			if ($config['debug']) {
				printf("Indexing %s<BR>", $filename); flush();
			}
			indexFile($filename, $config['indexFile']);
		}
	} else {
		$result = false;
		if (generateMasterIndex($masterIndex, $fileList)) {
			$fileList = array();
	
			if (writeMasterIndexFile($masterIndex)) {
				$result = true;
			}
		}
		$fileList = array();
		$masterIndex = array();
	}

	releaseIndexingLock();

	return $result;
}

function getFileText($filename) {
	if (($fpASCII = fopen($filename, 'r')) == false) {
		return false;
	}

	$text = fread($fpASCII, filesize($filename));

	fclose($fpASCII);

	$text = strtolower($text);

	//Space out HTML tags.
   $content = str_replace('<br', ' <br', $text);
   $content = str_replace('<p', ' <p', $tett);
   $content = str_replace('&nbsp;', ' ', $content); //'&nbsp' parts show up if they are broken, so convert to spaces.

	//Collapse ticks and quotes and &nbsp;

	$text = strip_tags($text);

	return $text;
}

function writeIndexFile(&$words, $filename) {
	if (($fp = fopen($filename, 'w')) == false) {
		return false;
	}

	ksort($words);
	foreach($words as $word => $count) {
		if (strlen($word) > 0) {
			fwrite($fp, $word . ":" . $count . "\n");
		}
	}
	
	fclose($fp);	
	return true;
}

function readIndexFile(&$words, $filename) {
	if (($fp = fopen($filename, 'r')) == false) {
		return false;
	}

	while ($temp = fgetcsv($fp, 512, ':')) {
		$words[$temp[0]] = $temp[1];
	}
	$temp = array();

	fclose($fp);	

	return true;
}

function subIndexFile(&$result, &$text) {
	//Collapse ticks and quotes.
   $text = str_replace('\'', '', $text);
   $text = str_replace('\"', '', $text);
 
	$splitexpr = '/[[:punct:][:space:]]+/';

	$temp = preg_split($splitexpr, $text);
	foreach($temp as $word) {
		$result[$word]++;
	}
	$temp = array();
}

function mergeIndex(&$result, &$from, $filename) {
	if (!isset($result['filenames'])) {
		$result['filenames'] = array();
		$offset = 0;
	} else {
		$length = strlen(end($result['filenames'])) + 1;
		$offset = key($result['filenames']) + $length;
	}
	$result['filenames'][$offset] = $filename;

	foreach($from as $keyword => $count) {
		if (!isset($result['keywords'][$keyword])) {
			$result['keywords'][$keyword] = array();
		}
		$result['keywords'][$keyword][$offset] = $count;
	}
}

function createMasterIndexFromFile($fp, $file) {
		global $config;

		if (!readIndexFile($keywordIndex, getIndexFileName($file))) {
			if ($config['debug'])
				printf('Could not index %s.<BR>', $file);

			return false;
		}
		$quickWordSize = 4;
		$mergingKeywordCount = count($keywordIndex);
		$quickIndexStep = $mergingKeywordCount > ($config['partitions']) ? $mergingKeywordCount / ($config['partitions']) : 1;
		fseek($fp, 12 + ($mergingKeywordCount * $quickIndexStep * (4 + $quickWordSize)));

		$keywordNumber = 0;
		$quickIndex[key($keywordIndex)] = ftell($fp);
		foreach ($keywordIndex as $keyword => $occurances) {
			if (!refreshIndexingLock()) {
				fclose($fp);

				return false;
			}
			if ((int)$keywordNumber < (int)($keywordNumber += (1 / $quickIndexStep))) {
				if ($config['debug'])
					printf('Adding word to quick index: %s<BR>', $keyword);

				$quickIndex[$keyword] = ftell($fp);
			}
			$keywordSize = strlen($keyword) + 1;
			$format = sprintf('L3a%dL2', $keywordSize);
			$binary = pack($format, 20 + $keywordSize, 1, $occurances, $keyword, 0, $occurances);

			fwrite($fp, $binary);
		}
		$keywordIndex = array();
		$filenameOffset = ftell($fp);
		$indexFilenameSize = strlen($file) + 1;

		if ($config['debug'])
			printf('Writing file name, %s, at file location 0x%x with eof at 0x%x.<BR>', $file, $filenameOffset, $filenameOffset + $indexFilenameSize);

		$format = sprintf('La%dL', $indexFilenameSize);
		$binary = pack($format, $indexFilenameSize + 4, $file, 0);
		fwrite($fp, $binary);

		fseek($fp, 0);
		$format = sprintf('a%dL',  $quickWordSize);
		$binary = pack('L2', 12 + count($quickIndex) * (4 + $quickWordSize), count($quickIndex));
		foreach ($quickIndex as $keyword => $offset) {
			if ($config['debug'])
				printf('Writing word to quick index: %s at file location 0x%x<BR>', $keyword, $offset);

			$binary .= pack($format, $keyword, $offset);
		}
		$quickIndex = array();
		$binary .= pack('L', $filenameOffset);
		fwrite($fp, $binary);

		return true;
}

function getFileIndex(&$fileIndex, $indexFile) {
	global $config;

	if (!($fp = fopen($indexFile, 'rb'))) {
		if ($config['debug']) {
			printf('<B>getFileIndex:</B> Failed to open index file (<B>%s</B>).<BR>', $indexFile);
		}
		return false;
	} else if (!getFilenameOffset($fileIndexOffset, $fp)) {
		if ($config['debug']) {
			printf('<B>getFileIndex:</B> Could not get file index offset from index file (<B>%s</B>).<BR>', $indexFile);
		}
		fclose($fp);
		return false;
	}

	$offset = 0;
	$nextOffset = 0;
	while (getNextOffset($nextOffset, $fp, $fileIndexOffset + $offset)) {
		if (!getFilename($filename, $fp, $offset)) {
			if ($config['debug']) {
				printf('<B>getFileIndex:</B> Could not find file name at offset <B>0x%x</B> in index file (<B>%s</B>).<BR>', $fileIndexOffset + $offset, $indexFile);
			}
			fclose($fp);
			return false;
		}
		if ($config['debug']) {
			printf('<B>getFileIndex:</B> Adding <B>%s</B> to file name index with file name offset of <B>0x%x</B>.<BR>', $filename, $offset);
		}
		$fileIndex[$filename] = $offset;

		$offset += $nextOffset;
	}

	fclose($fp);
	return true;
}

/*
** Local Variables:
** c-basic-offset: 3
** tab-width: 3
** End:
** vim:
** vim: noet ts=3 sw=3
*/

?>
