Thread: Matrix Generator

Hi,

Im having some problems with my code (In bold). Please can someone look into it.

import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.StringTokenizer;

public class MatrixGenerator {
private Map<String, String> lexicon;
private Map<String, Integer> freq;
private String corpus;
private double[][] result;
private String tagset;
public MatrixGenerator(Map<String, String> lexicon, String corpus, String tagset) {
this.corpus = corpus;
this.lexicon = lexicon;
this.tagset = tagset;
countLexems();
System.out.println("Tagset: " + tagset);
System.out.println();
System.out.println("Corpus: " + corpus);
System.out.println();
System.out.println("Frequencies:");
System.out.println();
printMap(freq);
}
public double[][] createMatrixB() {
String[] t = tagset.split(" ");
String[] l = lexicon.keySet().toArray(new String[0]);


result = new double[l.length][t.length];
for (int j = 0; j < result.length; j++) {
Arrays.fill(result[j], 0.0);
for (int k = 0; k < result[j].length; k++) {
String categories = lexicon.get(l[j]);
if (categories.contains(t[k])) {
result[j][k] = 1.0 / categories.split(",").length;
}
}
}


for (int j = 0; j < result.length; j++) {
for (int k = 0; k < result[j].length; k++) {
double a = result[j][k] * freq.get(l[j]);
double b = 0.0;
for (int index = 0; index < result.length; index++) {
b += result[index][k] * freq.get(l[index]);
}
result[j][k] = a / b;
}
}

System.out.println();
System.out
.println("Matrix B (word-tag, lexicon and tags ordered as printed above, learned from corpus):");
ViterbiMatrixTools.printMatrix(result);
return result;
}

public double[][] createMatrixA(double diff) {
String[] t = tagset.split(" ");
double[][] res = new double[t.length][t.length];
int cols = res[0].length;
double average = 1.0 / cols;
for (int i = 0; i < res.length; i++) {
Arrays.fill(res[i], average);
}
Random r = new Random();
for (int i = 0; i < res.length; i++) {
double var = 1.0;
while (var > diff)
var = r.nextDouble();
int pos = r.nextInt(res[i].length);
boolean plus = true;
for (int j = 0; j < res[i].length; j++) {
// if not even, skip one random position

if (res[i].length % 2 != 0 && j == pos)
continue;
if (plus && res[i][j] - var > 0) {
res[i][j] += var;
plus = false;
} else if (res[i][j] - var > 0) {
res[i][j] -= var;
plus = true;
}
}
}
System.out.println();
System.out.println("Matrix A (tag-tag, pseudo-random-generated):");
ViterbiMatrixTools.printMatrix(res);
System.out.println();
return res;
}


private void printMap(Map<String, ?> map) {
for (Map<String, ?> e : map.entrySet()) {
System.out.println(e.getKey() + ": " + e.getValue());
}
}


private void countLexems() {
freq = new HashMap<String, Integer>();
for (StringTokenizer tok = new StringTokenizer(corpus, " \t\n.,", true);
tok.hasMoreTokens(); ) {
String rec = tok.nextToken();
if (lexicon.containsKey(rec)) {
if (freq.containsKey(rec)) {
freq.put(rec, freq.get(rec) + 1);
} else {
freq.put(rec, 1);
}
}
}
}

}

I am trying to do POS tagger in Java( Part of speech). The program has 5 classes- Tagger, Abstract HMM, Matrix Generator, Viterbi HMM and Viterbi Matrix Tools. HMM is Hidden Markov Model and is implemented using the Viterbi algo. All the other classes are working fine except for the Matrix Generator. The code(in bold) shows error "incompatible types" and "cannot find symbol getKey() and getValue().

Originally Posted by Norm

The compiler sees that the variables are incompatible types.
It can not find the methods: getKey and getValue in the class: Map.

So how do I go about it??

hi...thank you so much...i finally managed to solve it. But now im having problem with my tagger.. could you please look into it. The output should say whether a word is noun, verb or verb noun. I can build it but i cant run and see the output. Btw i am using Netbeans

import java.util.HashMap;
import java.util.Map;
import org.junit.Test;

public class Tagger {
String tagset;
String corpus;
Map<String, String> lexicon;
String lexiconString;
@Test
public void testTagger() {

System.out.println(tag("layers that fly that food . that fly sneaks ."));
}

private void initData() {
lexicon = new HashMap<String, String>();
lexicon.put("fly", "vn");
lexicon.put("layers", "vn");
lexicon.put("sneaks", "v");
lexicon.put("food", "n");
lexicon.put("that", "det,rp");
lexicon.put(".", "period");
StringBuilder lexBuf = new StringBuilder();
for (String word : lexicon.keySet()) {
StringBuilder append = lexBuf.append(word).append(" ");
}
lexiconString = lexBuf.toString();
tagset = "det rp vn v n period";
corpus = "that fly layers. that fly sneaks. that, that sneaks food" +
"layers. that fly layers that food. layers.";
}


public String tag(String input) {
initData();
ViterbiHMM hmm = new ViterbiHMM(tagset, lexiconString, input, lexicon,
corpus, true);
String[] in = input.split(" ");
String[] re = hmm.mostProbableSequence().split(" ");
StringBuilder buf = new StringBuilder();
for (int i = 0; i < in.length; i++) {
StringBuilder append = buf.append(in[i]).append(":").append(re[i]).append(" ");
}
return buf.toString();
}
}

i cant execute the program...i can just build it.. The thing is whenever i try to put in a main method it shows error but when i don't there is none. What can i do?? I really don't know what is wrong..

ok...u see if i put in the
public static void main(String[] args) {
i have to remove the word "tag" from
System.out.println(tag("layers that fly that food . that fly sneaks .")); as it says it is not a static method. So when i remove the word "tag" i can compile and run it and the output shows "layers that fly that food . that fly sneaks ." Beside that, i also have to remove the @Test and
public void testTagger() { to make it work

When i dont do all that...it is as this (print screen) Untitled.jpg

ok...u see if i put in the
public static void main(String[] args) {
i have to remove the word "tag" from
System.out.println(tag("layers that fly that food . that fly sneaks .")); as it says it is not a static method. So when i remove the word "tag" i can compile and run it and the output shows "layers that fly that food . that fly sneaks ." Beside that, i also have to remove the @Test and
public void testTagger() { to make it work

When i dont do all that...it is as this (print screen) Untitled.jpg

run:
Exception in thread "main" java.lang.RuntimeException: Uncompilable source code - non-static method tag(java.lang.String) cannot be referenced from a static context
at Tagger.main(Tagger.java:12)
Java Result: 1
BUILD SUCCESSFUL (total time: 3 seconds)

how do i go about creating an instance of the class??

Hi, thank you so much for replaying soon. The code is not having any errors now, but there is no output.

 
import java.util.HashMap;
import java.util.Map;
import org.junit.Test;
 
 
class Tag{
   String tag;
}
class InstanceOf { 
    public static void main(String args[]) { 
        Tag tag = new Tag();
    }
}
 
 public class Tagger {
 	String tagset;
 	String corpus;
 	Map<String, String> lexicon;
 	String lexiconString;
 	@Test
 	public void testTagger() {
 
 		System.out.println(tag("layers that fly that food . that fly sneaks ."));
 	}
 
 	private void initData() {
 	    lexicon = new HashMap<String, String>();
 	    lexicon.put("fly", "vn");
 	    lexicon.put("layers", "vn");
 	    lexicon.put("sneaks", "v");
 	    lexicon.put("food", "n");
 	    lexicon.put("that", "det,rp");
 	    lexicon.put(".", "period");
 	    StringBuilder lexBuf = new StringBuilder();
 	    for (String word : lexicon.keySet()) {
            StringBuilder append = lexBuf.append(word).append(" ");
 	    }
 	    lexiconString = lexBuf.toString();
 	    tagset = "det rp vn v n period";
 	    corpus = "that fly layers. that fly sneaks. that, that sneaks food" +
 	    		"layers. that fly layers that food. layers.";
 	}
 
 
 	public String tag(String input) {
 	    initData();
 	    ViterbiHMM hmm = new ViterbiHMM(tagset, lexiconString, input, lexicon,
 	            corpus, true);
 	    String[] in = input.split(" ");
 	    String[] re = hmm.mostProbableSequence().split(" ");
 	    StringBuilder buf = new StringBuilder();
 	    for (int i = 0; i < in.length; i++) {
            StringBuilder append = buf.append(in[i]).append(":").append(re[i]).append(" ");
 	    }
 	    return buf.toString();
 	}
}

hi..i am still not able to get an output. So im sending all five classes that i have created with the hope u can go thru it and check for any mistakes that i have done. Thank you very much and sorry for all the trouble.

Tagger.java
import java.util.HashMap;
import java.util.Map;
import org.junit.Test;
 
 
class Tag{
   String tag;
}
class InstanceOf { 
    public static void main(String args[]) { 
        Tag tag = new Tag();
        System.out.print(tag);
    }
}
 
 public class Tagger {
 	String tagset;
 	String corpus;
 	Map<String, String> lexicon;
 	String lexiconString;
 	@Test
 	public void testTagger() {
 
 		System.out.println(tag("layers that fly that food . that fly sneaks ."));
 	}
 
 	private void initData() {
 	    lexicon = new HashMap<String, String>();
 	    lexicon.put("fly", "vn");
 	    lexicon.put("layers", "vn");
 	    lexicon.put("sneaks", "v");
 	    lexicon.put("food", "n");
 	    lexicon.put("that", "det,rp");
 	    lexicon.put(".", "period");
 	    StringBuilder lexBuf = new StringBuilder();
 	    for (String word : lexicon.keySet()) {
            StringBuilder append = lexBuf.append(word).append(" ");
 	    }
 	    lexiconString = lexBuf.toString();
 	    tagset = "det rp vn v n period";
 	    corpus = "that fly layers. that fly sneaks. that, that sneaks food" +
 	    		"layers. that fly layers that food. layers.";
 	}
 
 
 	public String tag(String input) {
 	    initData();
 	    ViterbiHMM hmm = new ViterbiHMM(tagset, lexiconString, input, lexicon,
 	            corpus, true);
 	    String[] in = input.split(" ");
 	    String[] re = hmm.mostProbableSequence().split(" ");
 	    StringBuilder buf = new StringBuilder();
 	    for (int i = 0; i < in.length; i++) {
            StringBuilder append = buf.append(in[i]).append(":").append(re[i]).append(" ");
 	    }
 	    return buf.toString();
 	}
}

 AbstractHMM.java 
import java.util.Map;
 
 abstract class AbstractHMM {
     protected String a;
     protected int n;
     protected String obsA;
     protected String obs;
     protected double[][] A;
     protected double[][] B;
     protected String mostProbableSequence;
     public AbstractHMM(String a, String obsA, String obs,
             Map<String, String> lexicon, String corpus) {
         MatrixGenerator gen = new MatrixGenerator(lexicon, corpus, a);
         this.a = a;
         this.n = obs.length();
         this.obsA = obsA;
         this.obs = obs;
         this.A = gen.createMatrixA(0.01);
         this.B = gen.createMatrixB();
     }
 
     public abstract String mostProbableSequence();
 }

MatrixGenerator .java  
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.StringTokenizer;
 
 
  public class MatrixGenerator {
  	private Map<String, String> lexicon;
  	private Map<String, Integer> freq;
  	private String corpus;
  	private double[][] result;
 	private String tagset;
  	public MatrixGenerator(Map<String, String> lexicon, String corpus, String tagset) {
  		this.corpus = corpus;
  		this.lexicon = lexicon;
  		this.tagset = tagset;
  		countLexems();
  		System.out.println("Tagset: " + tagset);
  		System.out.println();
  		System.out.println("Corpus: " + corpus);
  		System.out.println();
  		System.out.println("Frequencies:");
  		System.out.println();
  		printMap(freq);
  	}
  	public double[][] createMatrixB() {
  		String[] t = tagset.split(" ");
  		String[] l = lexicon.keySet().toArray(new String[0]);
 
 
  		result = new double[l.length][t.length];
  		for (int j = 0; j < result.length; j++) {
  		    Arrays.fill(result[j], 0.0);
  		    for (int k = 0; k < result[j].length; k++) {
  		        String categories = lexicon.get(l[j]);
  		        if (categories.contains(t[k])) {
  		            result[j][k] = 1.0 / categories.split(",").length;
  		        }
  		    }
  		}
 
 
  		for (int j = 0; j < result.length; j++) {
  		    for (int k = 0; k < result[j].length; k++) {
  		        double a = result[j][k] * freq.get(l[j]);
  		        double b = 0.0;
  		        for (int index = 0; index < result.length; index++) {
  		            b += result[index][k] * freq.get(l[index]);
  		        }
  		        result[j][k] = a / b;
  		    }
  		}
 
  		System.out.println();
  		System.out
  		        .println("Matrix B (word-tag, lexicon and tags ordered as printed above, learned from corpus):");
  		ViterbiMatrixTools.printMatrix(result);
  		return result;
  	}
 
  	public double[][] createMatrixA(double diff) {
  	    String[] t = tagset.split(" ");
  	    double[][] res = new double[t.length][t.length];
  	    int cols = res[0].length;
  	    double average = 1.0 / cols;
  	    for (int i = 0; i < res.length; i++) {
  	        Arrays.fill(res[i], average);
  	    }
  	    Random r = new Random();
  	    for (int i = 0; i < res.length; i++) {
  	        double var = 1.0;
  	        while (var > diff)
 	            var = r.nextDouble();
 	        int pos = r.nextInt(res[i].length);
 	        boolean plus = true;
 	        for (int j = 0; j < res[i].length; j++) {
 	            // if not even, skip one random position
 
 	            if (res[i].length % 2 != 0 && j == pos)
 	                continue;
 	            if (plus && res[i][j] - var > 0) {
 	                res[i][j] += var;
 	                plus = false;
 	            } else if (res[i][j] - var > 0) {
 	                res[i][j] -= var;
 	                plus = true;
 	            }
 	        }
 	    }
 
 	    System.out.println();
 	    System.out.println("Matrix A (tag-tag, pseudo-random-generated):");
 	    ViterbiMatrixTools.printMatrix(res);
 	    System.out.println();
 	    return res;
 	}
 
 
 	private void printMap(Map<String, ?> map) {
 	    for (Map.Entry<String, ?> e : map.entrySet()) {
        System.out.println(e.getKey() + ": " + e.getValue());
    }
}
 
 
	private void countLexems() {
	    freq = new HashMap<String, Integer>();
	    for (StringTokenizer tok = new StringTokenizer(corpus, " \t\n.,", true);
	         tok.hasMoreTokens(); ) {
	        String rec = tok.nextToken();
	        if (lexicon.containsKey(rec)) {
	            if (freq.containsKey(rec)) {
	                freq.put(rec, freq.get(rec) + 1);
	            } else {
	                freq.put(rec, 1);
	            }
	        }
	    }
	}
 
}

ViterbiHMM.java
 
import java.util.Arrays;
import java.util.Map;
 
  public class ViterbiHMM extends AbstractHMM {
      private double[][] delta;
      private int[][] psi;
      private String[] hiddenAlphabet;
      private String[] observableAlphabet;
      private String[] observation;
      private boolean absoluteValues;
      public ViterbiHMM(String a, String obsA, String obs,
              Map<String, String> lexicon, String corpus, boolean absoluteValues) {
          super(a, obsA, ". " + obs, lexicon, corpus);
          this.absoluteValues = absoluteValues;
      }
    @Override
      public String mostProbableSequence() {
 
  	init();
  	induct();
 
          System.out.println("Delta:");
          ViterbiMatrixTools.printMatrix(delta);
          System.out.println();
          System.out.println("Psi:");
          ViterbiMatrixTools.printMatrix(psi);
          System.out.println();
          return getResult();
     }
 
      private void init() {
          hiddenAlphabet = a.split("\\s");
          observableAlphabet = obsA.split("\\s");
          observation = obs.split("\\s");
          delta = new double[hiddenAlphabet.length][observation.length];
          delta[delta.length - 1][0] = 1.0;
          psi = new int[hiddenAlphabet.length][observation.length - 1];
          for (int i = 0; i < psi.length; i++) {
              Arrays.fill(psi[i], -1);
          }
      }
 
 
      private void induct() {
          // (1)
          for (int i = 1; i < observation.length; i++) {
              // (2)
             for (int j = 0; j < hiddenAlphabet.length; j++) {
                 double prevTagMax = ViterbiMatrixTools.maximimumForCol(
                         i - 1, delta);
                 // (3)
                  int lexIndex = getIndex(observation[i], observableAlphabet);
                  // (4)
 
                  double prob = probForWordToTag(lexIndex + 1, j, B, A);
                  double res = prevTagMax * prob;
                  delta[j][i] = res;
                  if (res > 0.0)
                      psi[j][i - 1] = ViterbiMatrixTools.indexOfMaximimumForCol(
                              i - 1, delta);
              }
 
          }
     }
 
 
      private double probForWordToTag(int i, int j, double[][] b, double[][] a) {
          // delta has the leading period, therefore - 1 for previous:
 
          int prevIndex = ViterbiMatrixTools.indexOfMaximimumForCol(i - 1, delta);
          // b doesn't have the leading p, therefore -1 for recent:
 
          if (absoluteValues)
              return (b[i - 1][j] / ViterbiMatrixTools.sumForCol(j, B))
                      * (A[prevIndex][j] / ViterbiMatrixTools.sumForRow(
                              prevIndex, A));
          else {
             return b[i - 1][j] * A[prevIndex][j];
         }
     }
 
 
     private String getResult() {
         String[] resultArray = new String[psi[0].length];
         int lastIndexInPsi = ViterbiMatrixTools.indexOfMaximimumForCol(
                 delta[0].length - 1, delta);
         if (lastIndexInPsi == -1) {
             System.out.println("no tag-sequence found for input, exit.");
             System.exit(0);
         }
         int lastValueInPsi = psi[lastIndexInPsi][psi[0].length - 1];
         String lastTag = hiddenAlphabet[lastIndexInPsi];
         resultArray[resultArray.length - 1] = lastTag;
         // retrieve other tags:
 
        for (int i = psi[0].length - 2; i >= 0; i--) {
            resultArray[i] = hiddenAlphabet[lastValueInPsi];
            lastValueInPsi = psi[lastValueInPsi][i];
     }
        StringBuilder resultString = new StringBuilder();
        for (int i = 0; i < resultArray.length; i++) {
            resultString.append(resultArray[i]);
            if (i < resultArray.length - 1)
                resultString.append(" ");
        }
        return resultString.toString();
    }
 
 
    int getIndex(String string, String[] lexicon) {
        for (int i = 0; i < lexicon.length; i++) {
            if (string.equals(lexicon[i]))
                return i;
        }
 
        System.out.println("Word '" + string + "' not found in lexicon, exit.");
        System.exit(0);
        return -1;
    }
 
}

ViterbiMatrixTools.java
 
import java.text.NumberFormat;
 
 
  public class ViterbiMatrixTools {
 
  	static double sumForCol(int i, double[][] matrix) {
  	    double sum = 0;
  	    for (int j = 0; j < matrix.length; j++) {
  	        sum += matrix[j][i];
  	    }
  	    return sum;
  	}
 
 
  	static double sumForRow(int i, double[][] matrix) {
  	    double sum = 0;
  	    for (int j = 0; j < matrix[i].length; j++) {
  	        sum += matrix[i][j];
  	    }
  	    return sum;
  	}
 
 
  	static double maximimumForCol(int i, double[][] matrix) {
  	    double maxValue = 0.0;
  	    for (int j = 0; j < matrix.length; j++) {
  	        maxValue = Math.max(maxValue, matrix[j][i]);
  	    }
  	    return maxValue;
  	}
 
 
  	static int indexOfMaximimumForCol(int i, double[][] matrix) {
  	    int maxIndex = -1;
  	    double maxValue = -1.0;
  	    for (int j = 0; j < matrix.length; j++) {
  	        if (matrix[j][i] > maxValue) {
  	            maxIndex = j;
  	            maxValue = matrix[j][i];
  	        }
  	    }
  	    return maxIndex;
  	}
 
 
  	static int indexOfMaximimumForCol(int i, int[][] matrix) {
  	    int maxIndex = -1;
  	    int maxValue = -1;
  	    for (int j = 0; j < matrix.length; j++) {
  	        if (matrix[j][i] > maxValue) {
  	            maxIndex = j;
  	            maxValue = matrix[j][i];
  	        }
  	    }
  	    return maxIndex;
  	}
 
 
  	static void printMatrix(double[][] matrix) {
  	    for (int i = 0; i < matrix.length; i++) {
  	        for (int j = 0; j < matrix[i].length; j++) {
  	            String myString = NumberFormat.getInstance().format(
  	                    matrix[i][j]);
  	            if (myString.length() < 5) {
  	                for (int k = myString.length(); k < 5; k++) {
  	                    myString += " ";
  	                }
  	            }
  	            System.out.print(myString + "   ");
  	        }
  	        System.out.println();
  	    }
  	}
 
 
 	static void printMatrix(int[][] matrix) {
 	    for (int i = 0; i < matrix.length; i++) {
 	        for (int j = 0; j < matrix[i].length; j++) {
 	            if (matrix[i][j] >= 0)
 	                System.out.print(" ");
 	            System.out.print(matrix[i][j] + "   ");
 	        }
 	        System.out.println();
 	    }
 	}
 
}

none of the classes are giving an output...

Do i need to use the try and catch method to invoke the methods??

how about get method?