/**
 * 
 */
package edu.berkeley.nlp.PCFGLA;

import java.util.Arrays;
import java.util.List;

import quicktime.streaming.PresIdleParams;

import edu.berkeley.nlp.PCFGLA.smoothing.Smoother;
import edu.berkeley.nlp.syntax.StateSet;
import edu.berkeley.nlp.syntax.StateSetWithFeatures;
import edu.berkeley.nlp.syntax.Tree;
import edu.berkeley.nlp.util.Counter;
import fig.basic.Indexer;
import edu.berkeley.nlp.util.Numberer;
import fig.basic.Pair;
import edu.berkeley.nlp.util.PriorityQueue;

/**
 * @author petrov
 *
 */
public class HierarchicalFullyConnectedAdaptiveLexiconWithFeatures extends
		HierarchicalFullyConnectedAdaptiveLexicon {

	private static final long serialVersionUID = 1L;
	Indexer<String> featureIndexer;
	SimpleLexicon simpleLex; 
	private final int minFeatureCount = 50; 
	
	public HierarchicalFullyConnectedAdaptiveLexiconWithFeatures(short[] numSubStates, int smoothingCutoff, double[] smoothParam, Smoother smoother, StateSetTreeList trainTrees, int knownWordCount) {
		super(numSubStates, knownWordCount);//smoothingCutoff, smoothParam, smoother, trainTrees, knownWordCount);
		simpleLex = new SimpleLexicon(numSubStates,-1);
		init(trainTrees);
//		super.init(trainTrees);
		
	}

//	public HierarchicalFullyConnectedAdaptiveLexiconWithFeatures newInstance() {
//		return new HierarchicalFullyConnectedAdaptiveLexiconWithFeatures(this.numSubStates,this.knownWordCount);
//	}

  public void init(StateSetTreeList trainTrees){
  	for (Tree<StateSet> tree : trainTrees){
  		List<StateSet> words = tree.getYield();
  		for (StateSet word : words){
				String sig = word.getWord();
				wordIndexer.add(sig);
  		}
  	}
  	wordCounter = new int[wordIndexer.size()];
  	Counter<String> ixCounter = new Counter<String>();
  	featureIndexer = new Indexer<String>();
  	for (Tree<StateSet> tree : trainTrees){
  		List<StateSet> words = tree.getYield();
  		int ind = 0;  		
  		for (StateSet word : words){
  			String wordString = word.getWord();
  			wordCounter[wordIndexer.indexOf(wordString)]++;
  			
				String sig = getSignature(word.getWord(), ind++);
				wordIndexer.add(sig);
				tallyWordFeatures(word.getWord(), ixCounter);
  		}  		
  	}
  	
  	featureIndexer = new Indexer<String>();
  	for (String word : ixCounter.keySet()){
  		if (ixCounter.getCount(word) >= minFeatureCount){
  			System.out.println("keeping: \t"+word);
  			featureIndexer.add(word);
  		}
			else
				System.out.println("too rare:\t"+word);

  	}
  	
  	simpleLex.wordCounter = wordCounter;
  	labelTrees(trainTrees);
  	
  	tagWordIndexer = new IntegerIndexer[numStates];
  	for (int tag=0; tag<numStates; tag++){
  		tagWordIndexer[tag] = new IntegerIndexer(featureIndexer.size());
  	}

  	boolean[] lexTag = new boolean[numStates]; 
  	for (Tree<StateSet> tree : trainTrees){
  		List<StateSet> words = tree.getYield();
  		List<StateSet> tags = tree.getPreTerminalYield();
  		int ind = 0;
  		for (StateSet word : words){
  			int tag = tags.get(ind).getState();
  			StateSetWithFeatures wordF = (StateSetWithFeatures)word;

  			for (Integer f : wordF.features){
  				tagWordIndexer[tag].add(f);
  			}
				lexTag[tag] = true;
				ind++;
  		}  		
  	}


  	expectedCounts = new double[numStates][][];
  	scores = new double[numStates][][];
  	for (int tag=0; tag<numStates; tag++){
  		if (!lexTag[tag]) {
  			tagWordIndexer[tag] = null;
  			continue;
  		}
//  		else tagWordIndexer[tag] = tagIndexer;
//  		expectedCounts[tag] = new double[numSubStates[tag]][tagWordIndexer[tag].size()];
  		scores[tag] = new double[numSubStates[tag]][tagWordIndexer[tag].size()];
  	}
  	nWords = wordIndexer.size();
  	
		this.scores = null;
		this.hierarchicalScores = null;
		this.finalLevels = null;
		rules = new HierarchicalAdaptiveLexicalRule[numStates][];
  	for (int tag=0; tag<numStates; tag++){
  		if (tagWordIndexer[tag]==null) {
  			rules[tag] = new HierarchicalAdaptiveLexicalRule[0];
  			continue;
  		}
  		rules[tag] = new HierarchicalAdaptiveLexicalRule[tagWordIndexer[tag].size()];
  		for (int word=0; word<rules[tag].length; word++){
  			rules[tag][word] = new HierarchicalAdaptiveLexicalRule();
  		}
  	}


  }

	
	/**
	 * @param word
	 * @param ixCounter 
	 */
	private void tallyWordFeatures(String word, Counter<String> ixCounter) {
		int length = word.length();
		if (length>4){
			for (int i=1; i<4; i++){
//				String prefix = "PREF-"+word.substring(0,i);
//				featureIndexer.add(prefix);
//				ixCounter.incrementCount(prefix, 1.0);
				String suffix = "SUFF-"+word.substring(length-i);
				featureIndexer.add(suffix);
				ixCounter.incrementCount(suffix, 1.0);
			}
		}
	}

	public StateSet tallyFeatures(StateSet stateSet, boolean update) {
		String word = stateSet.getWord();
    String lowered = word.toLowerCase();
		int loc = stateSet.from;
    String sig = simpleLex.getNewSignature(word, loc);
    StateSetWithFeatures newStateSet = new StateSetWithFeatures(stateSet);
    if (update) featureIndexer.add(sig);
		newStateSet.features.add(featureIndexer.indexOf(sig));
		
		if (update) featureIndexer.add("UNK");
		newStateSet.features.add(featureIndexer.indexOf("UNK"));
		int length = word.length();
		if (length>4){
			for (int i=1; i<4; i++){
//				String prefix = "PREF-"+lowered.substring(0,i);
//				int prefInd = featureIndexer.indexOf(prefix);
//				if (prefInd>=0)
//					newStateSet.features.add(prefInd);
				String suffix = "SUFF-"+lowered.substring(length-i);
				int suffInd = featureIndexer.indexOf(suffix);
				if (suffInd>=0)
					newStateSet.features.add(suffInd);
			}
		}
    int wlen = word.length();
    int numCaps = 0;
    boolean hasDigit = false;
    boolean hasDash = false;
    boolean hasLower = false;
    for (int i = 0; i < wlen; i++) {
      char ch = word.charAt(i);
      if (Character.isDigit(ch)) {
        hasDigit = true;
      } else if (ch == '-') {
        hasDash = true;
      } else if (Character.isLetter(ch)) {
        if (Character.isLowerCase(ch)) {
          hasLower = true;
        } else if (Character.isTitleCase(ch)) {
          hasLower = true;
          numCaps++;
        } else {
          numCaps++;
        }
      }
    }
    char ch0 = word.charAt(0);
    if (Character.isUpperCase(ch0) || Character.isTitleCase(ch0)) {
      if (loc == 0 && numCaps == 1) {
      	if (update) featureIndexer.add("INITC");
      	newStateSet.features.add(featureIndexer.indexOf("INITC"));
//        if (isKnown(lowered)) {
//          sb.append("-KNOWNLC");
//        }
      } else {
      	if (update) featureIndexer.add("CAPS");
      	newStateSet.features.add(featureIndexer.indexOf("CAPS"));
      }
    } else if (!Character.isLetter(ch0) && numCaps > 0) {
    	if (update) featureIndexer.add("CAPS");
    	newStateSet.features.add(featureIndexer.indexOf("CAPS"));
    } else if (hasLower) { // (Character.isLowerCase(ch0)) {
    	if (update) featureIndexer.add("LC");
    	newStateSet.features.add(featureIndexer.indexOf("LC"));
    }
    if (hasDigit) {
    	if (update) featureIndexer.add("NUM");
    	newStateSet.features.add(featureIndexer.indexOf("NUM"));
    }
    if (hasDash) {
    	if (update) featureIndexer.add("DASH");
    	newStateSet.features.add(featureIndexer.indexOf("DASH"));
    }
    if (lowered.endsWith("s") && wlen >= 3) {
      // here length 3, so you don't miss out on ones like 80s
      char ch2 = lowered.charAt(wlen - 2);
      // not -ess suffixes or greek/latin -us, -is
      if (ch2 != 's' && ch2 != 'i' && ch2 != 'u') {
      	if (update) 	featureIndexer.add("s");
      	newStateSet.features.add(featureIndexer.indexOf("s"));
      }
    } else if (word.length() >= 5 && !hasDash && !(hasDigit && numCaps > 0)) {
      // don't do for very short words;
      // Implement common discriminating suffixes
/*          	if (Corpus.myLanguage==Corpus.GERMAN){
    		sb.append(lowered.substring(lowered.length()-1));
    	}else{*/
//      if (lowered.endsWith("ed")) {
//        sb.append("-ed");
//      } else if (lowered.endsWith("ing")) {
//        sb.append("-ing");
//      } else if (lowered.endsWith("ion")) {
//        sb.append("-ion");
//      } else if (lowered.endsWith("er")) {
//        sb.append("-er");
//      } else if (lowered.endsWith("est")) {
//        sb.append("-est");
//      } else if (lowered.endsWith("ly")) {
//        sb.append("-ly");
//      } else if (lowered.endsWith("ity")) {
//        sb.append("-ity");
//      } else if (lowered.endsWith("y")) {
//        sb.append("-y");
//      } else if (lowered.endsWith("al")) {
//        sb.append("-al");
        // } else if (lowered.endsWith("ble")) {
        // sb.append("-ble");
        // } else if (lowered.endsWith("e")) {
        // sb.append("-e");
      }
    return newStateSet;
	}

	public void labelTrees(StateSetTreeList trainTrees){
		for (Tree<StateSet> tree : trainTrees){
//  		List<StateSet> words = tree.getYield();
  		int ind = 0;
  		for (Tree<StateSet>  word : tree.getTerminals()){
  			StateSetWithFeatures wordF = new StateSetWithFeatures(word.getLabel());
//  			wordF.wordIndex = wordIndexer.indexOf(word.getWord());
  			if (wordF.wordIndex<0 || wordF.wordIndex>=wordCounter.length){
  				System.out.println("Have never seen this word before: "+wordF.getWord()+" "+wordF.wordIndex);
  				System.out.println(tree);
  			}
  			else if (wordCounter[wordF.wordIndex]<=knownWordCount){
					wordF = (StateSetWithFeatures) tallyFeatures(wordF, false);
  			}
  			else 
  				wordF.sigIndex = -1;
  			featureIndexer.add(wordF.getWord());
  			wordF.features.add(featureIndexer.indexOf(wordF.getWord()));
				word.setLabel(wordF);
				ind++;
  		}  		
  	}
	}
//	StateSetWithFeatures lastStateSet;

	public double[] score(StateSet stateSet, short tag, boolean noSmoothing, boolean isSignature) {
		double[] res = new double[numSubStates[tag]];
		Arrays.fill(res,1);
		StateSetWithFeatures stateSetF = null;
		if (stateSet.wordIndex == -2) {
			stateSetF = new StateSetWithFeatures(stateSet);
			int wordIndex = wordIndexer.indexOf(stateSet.getWord());
			if (wordIndex<0||(wordIndex>=0 && (wordCounter[wordIndex]<=knownWordCount))){
				stateSetF = (StateSetWithFeatures)tallyFeatures(stateSet, false);
			} 
			int f = featureIndexer.indexOf(stateSet.getWord());
			if (f>=0)
				stateSetF.features.add(f);
//			stateSetF.wordIndex = -3;
//			stateSet = lastStateSet;
//		} else if (stateSet.wordIndex == -3){
//			stateSet = lastStateSet;
		} else {
			stateSetF = (StateSetWithFeatures) stateSet;
		}

		boolean noFeat = true;
		for (int f : stateSetF.features){
//			if (f>tagWordIndexer[tag].size())
//				System.out.println("hier");
			if (f<0)
				continue;
			int tagF = tagWordIndexer[tag].indexOf(f);
			if (tagF<0) 
				continue;
				
			noFeat = false;
			double[] resF = rules[tag][tagF].scores; 
			for (int i=0; i<res.length; i++){
				res[i] *= resF[i];
			}
		}
//		if (noFeat) {
//			System.out.println("No features for word "+stateSet.getWord()+" "+wordIndexer.indexOf(stateSet.getWord()));
//		}
		return res;
	}

  public String toString() {
  	StringBuffer sb = new StringBuffer();
  	Numberer tagNumberer = Numberer.getGlobalNumberer("tags");
		PriorityQueue<Pair<Integer,Integer>> pQ = new PriorityQueue<Pair<Integer,Integer>>();
  	for (int tag=0; tag<rules.length; tag++){
    	int[] counts = new int[6];
  		String tagS = (String)tagNumberer.object(tag);
  		if (rules[tag].length==0) continue;
  		for (int word=0; word<featureIndexer.size(); word++){
  			int wordT = tagWordIndexer[tag].indexOf(word);
  			if (wordT<0) continue;
    		String w = featureIndexer.get(word);
  			if (w.length()>4 && w.substring(0, 4).equals("SUFF")){
  				pQ.add(new Pair(tag,word), rules[tag][wordT].scores[0]);
  			}
  		}
  	}
		while (pQ.hasNext()){
			Pair<Integer,Integer> p = pQ.next();
			int word = p.getSecond();
			int tag = p.getFirst();
  		String tagS = (String)tagNumberer.object(tag);
			int wordT = tagWordIndexer[tag].indexOf(word);
			sb.append(tagS+" "+ featureIndexer.get(word)+"\n");
			sb.append(rules[tag][wordT].toString());
			sb.append("\n\n");
		}
		sb.append("-----------Start unsorted----------\n");
  	for (int tag=0; tag<rules.length; tag++){
    	int[] counts = new int[6];
  		String tagS = (String)tagNumberer.object(tag);
  		if (rules[tag].length==0) continue;
  		for (int word=0; word<featureIndexer.size(); word++){
  			int wordT = tagWordIndexer[tag].indexOf(word);
  			if (wordT<0) continue;
  			sb.append(tagS+" "+ featureIndexer.get(word)+"\n");
  			sb.append(rules[tag][wordT].toString());
  			sb.append("\n\n");
  			counts[rules[tag][wordT].hierarchy.getDepth()]++;
  		}
  		System.out.print(tagNumberer.object(tag)+", lexical rules per level: ");
  		for (int i=1; i<6; i++){
  			System.out.print(counts[i]+" ");
  		}
  		System.out.print("\n");

  	}
  	return sb.toString();
  }
  


}
