aol_rerank.php

<?php
// This program will re-rank the aol original itemrank list by 
// sense traffic at a specific hour
// The program will read an itemrank file, a sense traffic file and 
// similarity file

// The following is the format of the itemrank file.
//
// query \t url \t score \n
// the higher score, the higher rank.
// 
// the format of sense traffic
// sense_name \t prob \n
// 
// the format of sense similarity
// urls \t sense \t similarity_score \n
 
include_once("aol_utility.php");
class aol_rerank extends aol_utility {
	// this class will use all sense to compute the new rank 
	protected $item_fp = null;
	protected $sim_fp = null;
	protected $traffice_fp = null;
	protected $itemarnk = array();
	protected $itemarnk_score = array();
	protected $merge_rank = array();
	protected $merge_score = array();
	protected $sim = array();
	protected $traffic = array();
	protected $sense_score = array();
	protected $sense_rank = array();
	protected $alpha = 1.0;
	protected $output_prefix = null;
	protected $output_a = null;
	public function __construct($para){
		if ( isset($para["itemrank"]) ){
			$this->item_fp = fopen($para["itemrank"], "r");
			if ($this->item_fp == NULL){
				fprintf(STDERR, "%s can't be opened\n", $para["itemrank"]);
				exit(-1);
			}
		}else{
			fprintf(STDERR,"please specify the input file with option \"-itemrank\"\n");
			exit(-1);
		}
		if ( isset($para["sense_traffic"]) ){
			$this->traffic_fp = fopen($para["sense_traffic"], "r");
			if ($this->traffic_fp == NULL){
				fprintf(STDERR, "%s can't be opened\n", $para["sense_traffic"]);
				exit(-1);
			}
		}else{
			fprintf(STDERR,"please specify the input file with option \"-sense_traffic\"\n");
			exit(-1);
		}
		if ( isset($para["similarity"]) ){
			$this->sim_fp = fopen($para["similarity"], "r");
			if ($this->sim_fp == NULL){
				fprintf(STDERR, "%s can't be opened\n", $para["similarity"]);
				exit(-1);
			}
		}else{
			fprintf(STDERR,"please specify the input file with option \"-similarity\"\n");
			exit(-1);
		}
		if ( isset($para["o"]) ){
			$this->output_prefix = $para["o"];
			for ($a = 0.0;$a <= 1.0;$a+=0.1){
				$index = sprintf("%.1lf", $a); // if use double as the array index, it will get an error 
				$this->output_a[$index] = fopen($this->output_prefix.".".$index.".txt", "w");
				if ($this->output_a[$index] == NULL){
					fprintf(STDERR,$this->output_prefix.".".$index." can't be open\n");
					exit(-1);
				}
			}
		}else{
			fprintf(STDERR,"please specify the output prefix with option \"-o\"\n");
			exit(-1);
		}
		$this->ReadingInput();
		parent::__construct($para);

	}

	public function __destruct() {
		for ($a = 0.0;$a <= 1.0;$a+=0.1){
			$index = sprintf("%.1lf", $a); // if use double as the array index, it will get an error 
			fclose($this->output_a[$index]);
		}
		parent::__destruct();
	}
	private function ReadingInput() {
		while (!feof($this->item_fp)){
			$line = fgets($this->item_fp);
			if (empty($line) || $line == "\n"){
				continue;
			}
			$list = preg_split("/\t|\n/", $line);
			$q = $list[0];
			//$u = preg_quote($list[2]);
			$u = $list[1];
			$rank_score = $list[2]; 
			$this->itemrank_score[$q][$u] = doubleval($rank_score);
		}
		foreach ($this->itemrank_score as $q => $ranking){
			arsort($this->itemrank_score[$q]);
			$rank = 1;
			foreach($this->itemrank_score[$q] as $u => $score){
				$this->itemrank[$q][$u] = $rank;
				$rank++;
			}
			//echo $q."\n";
			//print_r($this->itemrank_score[$q]);
			//print_r($this->itemrank[$q]);
			// we have an assumption that the itemrank score never tie. 
		}
		fclose($this->item_fp);

		while (!feof($this->sim_fp)){
			$line = fgets($this->sim_fp);
			if (empty($line) || $line == "\n"){
				continue;
			}
			$list = preg_split("/\t|\n/", $line);
			$u = $list[0];
			$sense = $list[1];
			$score = $list[2]; 
			$this->sim[$u][$sense] = doubleval($score);
		}
		//print_r($this->sim);
		fclose($this->sim_fp);

		while (!feof($this->traffic_fp)){
			$line = fgets($this->traffic_fp);
			if (empty($line) || $line == "\n"){
				continue;
			}
			$list = preg_split("/\t|\n/", $line);
			$sense = $list[0];
			$score = $list[1]; 
			$this->traffic[$sense] = doubleval($score);
		}
		//print_r($this->traffic);
		fclose($this->traffic_fp);
	}

	public function query_newrank($q){
		$this->sense_score[$q] = array();
		$this->sense_rank[$q] = array();

		// compute new rank
		foreach ($this->itemrank[$q] as $u => $r){
			$sum = 0.0;
			if (!isset($this->sim[$u])){
				//skip the url the we haven't crawl the ram content
				continue;
			}			
			foreach ($this->traffic as $sense => $v){
				//echo "$u-$sense-\n";
				if ( isset($this->sim[$u][$sense]) ){
					$sum += $v * $this->sim[$u][$sense];
				}
			}
			$this->sense_score[$q][$u] = $sum;

		}
		arsort($this->sense_score[$q]);
		$rank = 1;
		foreach ($this->sense_score[$q] as $u => $v){
			$this->sense_rank[$q][$u] = $rank;
			$rank++;
		}
		//echo $q."\n";
		//print_r($this->sense_score[$q]);
		//print_r($this->sense_rank[$q]);
		// we have another assumption that the sense score never tie. 
	}
	private function query_merge($q){
		for ($a = 0.0; $a<= 1.0;$a+=0.1 ){
			$index = sprintf("%.1lf", $a);
			$this->merge_score[$index][$q] = array();
			$this->merge_rank[$index][$q] = array();
			foreach ($this->itemrank[$q] as $u => $r){
				if (!isset($this->sim[$u])){
					//skip the url the we haven't crawl the ram content
					continue;
				}
				$r_s = $this->sense_rank[$q][$u];
				$score = $a * 1.0/ (double) $r_s + 
					(1- $a) * 1.0 / (double) $r;
				$this->merge_score[$index][$q][$u] = $score;
			}
			arsort($this->merge_score[$index][$q]);
			$rank = 1;
			foreach ($this->merge_score[$index][$q] as $u => $v){
				$this->merge_rank[$index][$q][$u] = $rank;
				$rank++;
			}
		}
	}
	public function query_rerank($q){
		$this->query_newrank($q);
		$this->query_merge($q);
		//print_r($this->merge_rank[$q]);
	}
	
	public static function AolRerank($argc, $argv){
		// sample command
		// php aol_rerank.php -itemrank rerank_input.txt -sense_traffic traffic.txt -similarity sim.txt -o rerank3
		$para = ParameterParser($argc, $argv);
		$obj = new aol_rerank($para);
		$querys = array_keys($obj->itemrank);
		foreach ($querys as $q){
			$obj->query_rerank($q);
		}

		for ($a = 0.0; $a<= 1.0;$a+=0.1 ){
			$index = sprintf("%.1lf", $a);
			foreach ($obj->merge_score[$index] as $q => $ranking){
				foreach ($ranking as $u => $score){
					fprintf($obj->output_a[$index], "%s\t%s\t%lf\n", $q, $u, $score);
				}
				//$obj->dump_score($q);
			}
		}
	}
}


class aol_rerank_with_top_sense extends aol_rerank{
	// it computes the new rank with N senses those are most similar to the target url
	
	// sample command
	// php aol_rerank_dump.php -itemrank Itemrank.txt 
	// -sense_traffic data_txt/sense_traffic.txt -similarity data_txt/similarity.txt 
	// -o rerank3 -dump rerankdump 
	public $NSense;
	public function __construct($para){
		if ( isset($para["NSense"]) ){
			$this->NSense = intval($para["NSense"]);
		}else {
			fprintf(STDERR, "please specify the NSense parameter with option -NSense\n");
			exit(-1);
		}
		parent::__construct($para);
	}
	public function query_newrank($q){
		$this->sense_score[$q] = array();
		$this->sense_rank[$q] = array();
		
		// compute new rank
		foreach ($this->itemrank[$q] as $u => $r){
			$sum = 0.0;
			$counter = 0;
			if (!isset($this->sim[$u])){
				//skip the url the we haven't crawl the ram content
				continue;
			}
			arsort($this->sim[$u]);
			foreach ($this->sim[$u] as $sense => $v){
				if ($counter >= $this->NSense){
					break;
				}
				if ( isset($this->traffic[$sense]) ){
					$sum+= $v * $this->traffic[$sense];
					$counter++;
				}
			}
			$this->sense_score[$q][$u] = $sum;

		}
		arsort($this->sense_score[$q]);
		$rank = 1;
		foreach ($this->sense_score[$q] as $u => $v){
			$this->sense_rank[$q][$u] = $rank;
			$rank++;
		}
	}
	public static function AolRerank($argc, $argv){
		$para = ParameterParser($argc, $argv);
		$obj = new aol_rerank_with_top_sense($para);
		$querys = array_keys($obj->itemrank);
		foreach ($querys as $q){
			$obj->query_rerank($q);
		}

		for ($a = 0.0; $a<= 1.0;$a+=0.1 ){
			$index = sprintf("%.1lf", $a);
			foreach ($obj->merge_score[$index] as $q => $ranking){
				foreach ($ranking as $u => $score){
					fprintf($obj->output_a[$index], "%s\t%s\t%lf\n", $q, $u, $score);
				}
			}
		}
	}
}
?>