diff --git a/project/src/zbf/search/recommend/CosineDis.java b/project/src/zbf/search/recommend/CosineDis.java index 053dd6a..501f49d 100644 --- a/project/src/zbf/search/recommend/CosineDis.java +++ b/project/src/zbf/search/recommend/CosineDis.java @@ -1,6 +1,6 @@ package zbf.search.recommend; -import java.io.UnsupportedEncodingException; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -9,44 +9,41 @@ import zbf.search.util.StdOutUtil; public class CosineDis { - public static double getSimilarity(String doc1, String doc2) { + public static double getSimilarity(ArrayList doc1, ArrayList doc2) { + if (doc1 != null && doc1.size() > 0 && doc2 != null && doc2.size() > 0) { - if (doc1 != null && doc1.trim().length() > 0 && doc2 != null - && doc2.trim().length() > 0) { + Map AlgorithmMap = new HashMap(); - Map AlgorithmMap = new HashMap(); - - for (int i = 0; i < doc1.length(); i++) { - char d1 = doc1.charAt(i); - int charIndex = getId(d1); - int[] fq = AlgorithmMap.get(charIndex); - if (fq != null && fq.length == 2) { + for (int i = 0; i < doc1.size(); i++) { + String d1 = doc1.get(i); + long sIndex = hashId(d1); + int[] fq = AlgorithmMap.get(sIndex); + if (fq != null) { fq[0]++; } else { fq = new int[2]; fq[0] = 1; fq[1] = 0; - AlgorithmMap.put(charIndex, fq); + AlgorithmMap.put(sIndex, fq); } - } - for (int i = 0; i < doc2.length(); i++) { - char d2 = doc2.charAt(i); - int charIndex = getId(d2); - int[] fq = AlgorithmMap.get(charIndex); - if (fq != null && fq.length == 2) { + for (int i = 0; i < doc2.size(); i++) { + String d2 = doc2.get(i); + long sIndex = hashId(d2); + int[] fq = AlgorithmMap.get(sIndex); + if (fq != null) { fq[1]++; } else { fq = new int[2]; fq[0] = 0; fq[1] = 1; - AlgorithmMap.put(charIndex, fq); + AlgorithmMap.put(sIndex, fq); } } - Iterator iterator = AlgorithmMap.keySet().iterator(); + Iterator iterator = AlgorithmMap.keySet().iterator(); double sqdoc1 = 0; double sqdoc2 = 0; double denominator = 0; @@ -64,40 +61,25 @@ public class CosineDis { } } - public static boolean isHanZi(char ch) { - // 判断是否汉字 - return (ch >= 0x4E00 && ch <= 0x9FA5); - } - - /** - * 根据输入的Unicode字符，获取它的GB2312编码或者ascii编码， - * - * @param ch - * 输入的GB2312中文字符或者ASCII字符(128个) - * @return ch在GB2312中的位置，-1表示该字符不认识 - */ - public static short getGB2312Id(char ch) { - try { - byte[] buffer = Character.toString(ch).getBytes("GB2312"); - if (buffer.length != 2) { - // 正常情况下buffer应该是两个字节，否则说明ch不属于GB2312编码，故返回'?'，此时说明不认识该字符 - return -1; - } - int b0 = (int) (buffer[0] & 0x0FF) - 161; // 编码从A1开始，因此减去0xA1=161 - int b1 = (int) (buffer[1] & 0x0FF) - 161; // 第一个字符和最后一个字符没有汉字，因此每个区只收16*6-2=94个汉字 - return (short) (b0 * 94 + b1); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); + public static long hashId(String s) { + long seed = 131; // 31 131 1313 13131 131313 etc.. BKDRHash + long hash = 0; + for (int i = 0; i < s.length(); i++) { + hash = (hash * seed) + s.charAt(i); } - return -1; - } - - public static short getId(char ch) { - return (short) ch; + return hash; } public static void main(String[] args) { - StdOutUtil.out(getSimilarity("asdfg", "gfdsa")); + ArrayList t1 = new ArrayList(); + ArrayList t2 = new ArrayList(); + t1.add("sa"); + t1.add("dfg"); + t1.add("df"); + + t2.add("gfd"); + t2.add("sa"); + StdOutUtil.out(getSimilarity(t1, t2)); } } diff --git a/project/src/zbf/search/recommend/Homepage.java b/project/src/zbf/search/recommend/Homepage.java new file mode 100644 index 0000000..cee1052 --- /dev/null +++ b/project/src/zbf/search/recommend/Homepage.java @@ -0,0 +1,73 @@ +package zbf.search.recommend; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; + +import org.wltea.analyzer.core.IKSegmenter; +import org.wltea.analyzer.core.Lexeme; + +import zbf.search.util.StdOutUtil; + +import com.mongodb.DB; +import com.mongodb.DBCollection; +import com.mongodb.DBCursor; +import com.mongodb.DBObject; +import com.mongodb.MongoClient; + +public class Homepage { + public static void main(String[] args) throws IOException { + MongoClient mongoClient = new MongoClient("localhost", 30000); + DB db = mongoClient.getDB("academic"); + DBCollection coll = db.getCollection("homepages"); + DBCursor cursor = coll.find(); + + ArrayList doc1 = new ArrayList(); + ArrayList doc2 = new ArrayList(); + + while(cursor.hasNext()) { + if (doc1.size() > 0 && doc2.size() > 0) { + double dist = CosineDis.getSimilarity(doc1, doc2); + StdOutUtil.out(dist); + StdOutUtil.out(""); + doc1 = new ArrayList(); + doc2 = new ArrayList(); + } + DBObject obj = cursor.next(); + String url = (String) obj.get("url"); + String text = (String) obj.get("text"); + if (text.length() > 10) { + if (doc1.size() == 0) { + doc1 = getTokens(text); + StdOutUtil.out(url); + //StdOutUtil.out(doc1.toString()); + } else { + doc2 = getTokens(text); + StdOutUtil.out(url); + //StdOutUtil.out(doc2.toString()); + } + } + } + + } + + private static ArrayList getTokens(String text) throws IOException { + ArrayList tokens = new ArrayList(); + InputStream is = new ByteArrayInputStream(text.getBytes("UTF-8")); + BufferedReader input = new BufferedReader(new InputStreamReader(is)); + + // true为智能划词, false为最细粒度划词 + IKSegmenter segmenter = new IKSegmenter(input, true); + Lexeme token = null; + do { + token = segmenter.next(); + if (token != null) + if (token.getLexemeText().length() > 2) + tokens.add(token.getLexemeText()); + } while (token != null); + return tokens; + } +} diff --git a/project/src/zbf/search/recommend/LevenshteinDis.java b/project/src/zbf/search/recommend/LevenshteinDis.java deleted file mode 100644 index c9fe689..0000000 --- a/project/src/zbf/search/recommend/LevenshteinDis.java +++ /dev/null @@ -1,58 +0,0 @@ -package zbf.search.recommend; - -public class LevenshteinDis { - - public static void main(String[] args) { - // 要比较的两个字符串 - String str1 = "今天星期四"; - String str2 = "今天是星期五"; - levenshtein(str1, str2); - } - - public static void levenshtein(String str1, String str2) { - - int len1 = str1.length(); - int len2 = str2.length(); - - int[][] dif = new int[len1 + 1][len2 + 1]; - - for (int a = 0; a <= len1; a++) { - dif[a][0] = a; - } - for (int a = 0; a <= len2; a++) { - dif[0][a] = a; - } - // 计算两个字符是否一样，计算左上的值 - int temp; - for (int i = 1; i <= len1; i++) { - for (int j = 1; j <= len2; j++) { - if (str1.charAt(i - 1) == str2.charAt(j - 1)) { - temp = 0; - } else { - temp = 1; - } - // 取三个值中最小的 - dif[i][j] = min(dif[i - 1][j - 1] + temp, dif[i][j - 1] + 1, - dif[i - 1][j] + 1); - } - } - System.out.println("字符串\"" + str1 + "\"与\"" + str2 + "\"的比较"); - // 取数组右下角的值，同样不同位置代表不同字符串的比较 - System.out.println("差异步骤：" + dif[len1][len2]); - // 计算相似度 - float similarity = 1 - (float) dif[len1][len2] - / Math.max(str1.length(), str2.length()); - System.out.println("相似度：" + similarity); - } - - private static int min(int... is) { - int min = Integer.MAX_VALUE; - for (int i : is) { - if (min > i) { - min = i; - } - } - return min; - } - -} \ No newline at end of file