mirror of
https://gh-proxy.org/https://github.com/klinsmaya/VerticleSearchEngine.git
synced 2026-11-09 01:47:14 +08:00
use cosDis for homepages
This commit is contained in:
pelick
@@ -1,6 +1,6 @@
|
||||
package zbf.search.recommend;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
@@ -9,44 +9,41 @@ import zbf.search.util.StdOutUtil;
|
||||
|
||||
public class CosineDis {
|
||||
|
||||
public static double getSimilarity(String doc1, String doc2) {
|
||||
public static double getSimilarity(ArrayList<String> doc1, ArrayList<String> doc2) {
|
||||
if (doc1 != null && doc1.size() > 0 && doc2 != null && doc2.size() > 0) {
|
||||
|
||||
if (doc1 != null && doc1.trim().length() > 0 && doc2 != null
|
||||
&& doc2.trim().length() > 0) {
|
||||
Map<Long, int[]> AlgorithmMap = new HashMap<Long, int[]>();
|
||||
|
||||
Map<Integer, int[]> AlgorithmMap = new HashMap<Integer, int[]>();
|
||||
|
||||
for (int i = 0; i < doc1.length(); i++) {
|
||||
char d1 = doc1.charAt(i);
|
||||
int charIndex = getId(d1);
|
||||
int[] fq = AlgorithmMap.get(charIndex);
|
||||
if (fq != null && fq.length == 2) {
|
||||
for (int i = 0; i < doc1.size(); i++) {
|
||||
String d1 = doc1.get(i);
|
||||
long sIndex = hashId(d1);
|
||||
int[] fq = AlgorithmMap.get(sIndex);
|
||||
if (fq != null) {
|
||||
fq[0]++;
|
||||
} else {
|
||||
fq = new int[2];
|
||||
fq[0] = 1;
|
||||
fq[1] = 0;
|
||||
AlgorithmMap.put(charIndex, fq);
|
||||
AlgorithmMap.put(sIndex, fq);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
for (int i = 0; i < doc2.length(); i++) {
|
||||
char d2 = doc2.charAt(i);
|
||||
int charIndex = getId(d2);
|
||||
int[] fq = AlgorithmMap.get(charIndex);
|
||||
if (fq != null && fq.length == 2) {
|
||||
for (int i = 0; i < doc2.size(); i++) {
|
||||
String d2 = doc2.get(i);
|
||||
long sIndex = hashId(d2);
|
||||
int[] fq = AlgorithmMap.get(sIndex);
|
||||
if (fq != null) {
|
||||
fq[1]++;
|
||||
} else {
|
||||
fq = new int[2];
|
||||
fq[0] = 0;
|
||||
fq[1] = 1;
|
||||
AlgorithmMap.put(charIndex, fq);
|
||||
AlgorithmMap.put(sIndex, fq);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Iterator<Integer> iterator = AlgorithmMap.keySet().iterator();
|
||||
Iterator<Long> iterator = AlgorithmMap.keySet().iterator();
|
||||
double sqdoc1 = 0;
|
||||
double sqdoc2 = 0;
|
||||
double denominator = 0;
|
||||
@@ -64,40 +61,25 @@ public class CosineDis {
|
||||
}
|
||||
}
|
||||
|
||||
public static boolean isHanZi(char ch) {
|
||||
// 判断是否汉字
|
||||
return (ch >= 0x4E00 && ch <= 0x9FA5);
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据输入的Unicode字符，获取它的GB2312编码或者ascii编码，
|
||||
*
|
||||
* @param ch
|
||||
* 输入的GB2312中文字符或者ASCII字符(128个)
|
||||
* @return ch在GB2312中的位置，-1表示该字符不认识
|
||||
*/
|
||||
public static short getGB2312Id(char ch) {
|
||||
try {
|
||||
byte[] buffer = Character.toString(ch).getBytes("GB2312");
|
||||
if (buffer.length != 2) {
|
||||
// 正常情况下buffer应该是两个字节，否则说明ch不属于GB2312编码，故返回'?'，此时说明不认识该字符
|
||||
return -1;
|
||||
}
|
||||
int b0 = (int) (buffer[0] & 0x0FF) - 161; // 编码从A1开始，因此减去0xA1=161
|
||||
int b1 = (int) (buffer[1] & 0x0FF) - 161; // 第一个字符和最后一个字符没有汉字，因此每个区只收16*6-2=94个汉字
|
||||
return (short) (b0 * 94 + b1);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
e.printStackTrace();
|
||||
public static long hashId(String s) {
|
||||
long seed = 131; // 31 131 1313 13131 131313 etc.. BKDRHash
|
||||
long hash = 0;
|
||||
for (int i = 0; i < s.length(); i++) {
|
||||
hash = (hash * seed) + s.charAt(i);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
public static short getId(char ch) {
|
||||
return (short) ch;
|
||||
return hash;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
StdOutUtil.out(getSimilarity("asdfg", "gfdsa"));
|
||||
ArrayList<String> t1 = new ArrayList<String>();
|
||||
ArrayList<String> t2 = new ArrayList<String>();
|
||||
t1.add("sa");
|
||||
t1.add("dfg");
|
||||
t1.add("df");
|
||||
|
||||
t2.add("gfd");
|
||||
t2.add("sa");
|
||||
|
||||
StdOutUtil.out(getSimilarity(t1, t2));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package zbf.search.recommend;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.wltea.analyzer.core.IKSegmenter;
|
||||
import org.wltea.analyzer.core.Lexeme;
|
||||
|
||||
import zbf.search.util.StdOutUtil;
|
||||
|
||||
import com.mongodb.DB;
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCursor;
|
||||
import com.mongodb.DBObject;
|
||||
import com.mongodb.MongoClient;
|
||||
|
||||
public class Homepage {
|
||||
public static void main(String[] args) throws IOException {
|
||||
MongoClient mongoClient = new MongoClient("localhost", 30000);
|
||||
DB db = mongoClient.getDB("academic");
|
||||
DBCollection coll = db.getCollection("homepages");
|
||||
DBCursor cursor = coll.find();
|
||||
|
||||
ArrayList<String> doc1 = new ArrayList<String>();
|
||||
ArrayList<String> doc2 = new ArrayList<String>();
|
||||
|
||||
while(cursor.hasNext()) {
|
||||
if (doc1.size() > 0 && doc2.size() > 0) {
|
||||
double dist = CosineDis.getSimilarity(doc1, doc2);
|
||||
StdOutUtil.out(dist);
|
||||
StdOutUtil.out("");
|
||||
doc1 = new ArrayList<String>();
|
||||
doc2 = new ArrayList<String>();
|
||||
}
|
||||
DBObject obj = cursor.next();
|
||||
String url = (String) obj.get("url");
|
||||
String text = (String) obj.get("text");
|
||||
if (text.length() > 10) {
|
||||
if (doc1.size() == 0) {
|
||||
doc1 = getTokens(text);
|
||||
StdOutUtil.out(url);
|
||||
//StdOutUtil.out(doc1.toString());
|
||||
} else {
|
||||
doc2 = getTokens(text);
|
||||
StdOutUtil.out(url);
|
||||
//StdOutUtil.out(doc2.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static ArrayList<String> getTokens(String text) throws IOException {
|
||||
ArrayList<String> tokens = new ArrayList<String>();
|
||||
InputStream is = new ByteArrayInputStream(text.getBytes("UTF-8"));
|
||||
BufferedReader input = new BufferedReader(new InputStreamReader(is));
|
||||
|
||||
// true为智能划词, false为最细粒度划词
|
||||
IKSegmenter segmenter = new IKSegmenter(input, true);
|
||||
Lexeme token = null;
|
||||
do {
|
||||
token = segmenter.next();
|
||||
if (token != null)
|
||||
if (token.getLexemeText().length() > 2)
|
||||
tokens.add(token.getLexemeText());
|
||||
} while (token != null);
|
||||
return tokens;
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
package zbf.search.recommend;
|
||||
|
||||
public class LevenshteinDis {
|
||||
|
||||
public static void main(String[] args) {
|
||||
// 要比较的两个字符串
|
||||
String str1 = "今天星期四";
|
||||
String str2 = "今天是星期五";
|
||||
levenshtein(str1, str2);
|
||||
}
|
||||
|
||||
public static void levenshtein(String str1, String str2) {
|
||||
|
||||
int len1 = str1.length();
|
||||
int len2 = str2.length();
|
||||
|
||||
int[][] dif = new int[len1 + 1][len2 + 1];
|
||||
|
||||
for (int a = 0; a <= len1; a++) {
|
||||
dif[a][0] = a;
|
||||
}
|
||||
for (int a = 0; a <= len2; a++) {
|
||||
dif[0][a] = a;
|
||||
}
|
||||
// 计算两个字符是否一样，计算左上的值
|
||||
int temp;
|
||||
for (int i = 1; i <= len1; i++) {
|
||||
for (int j = 1; j <= len2; j++) {
|
||||
if (str1.charAt(i - 1) == str2.charAt(j - 1)) {
|
||||
temp = 0;
|
||||
} else {
|
||||
temp = 1;
|
||||
}
|
||||
// 取三个值中最小的
|
||||
dif[i][j] = min(dif[i - 1][j - 1] + temp, dif[i][j - 1] + 1,
|
||||
dif[i - 1][j] + 1);
|
||||
}
|
||||
}
|
||||
System.out.println("字符串\"" + str1 + "\"与\"" + str2 + "\"的比较");
|
||||
// 取数组右下角的值，同样不同位置代表不同字符串的比较
|
||||
System.out.println("差异步骤：" + dif[len1][len2]);
|
||||
// 计算相似度
|
||||
float similarity = 1 - (float) dif[len1][len2]
|
||||
/ Math.max(str1.length(), str2.length());
|
||||
System.out.println("相似度：" + similarity);
|
||||
}
|
||||
|
||||
private static int min(int... is) {
|
||||
int min = Integer.MAX_VALUE;
|
||||
for (int i : is) {
|
||||
if (min > i) {
|
||||
min = i;
|
||||
}
|
||||
}
|
||||
return min;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user