|
@@ -248,9 +248,9 @@ public class Word2VecWordSimilarity {
|
|
|
}
|
|
|
|
|
|
private static double calculateLengthPenalty(int len1, int len2) {
|
|
|
- // 加强长度差异惩罚
|
|
|
+ // 对长度差异进行惩罚
|
|
|
double ratio = Math.min(len1, len2) / (double) Math.max(len1, len2);
|
|
|
- return Math.pow(ratio, 1.2); // 进一步增加惩罚力度
|
|
|
+ return Math.pow(ratio, 0.5); // 使用平方根来减轻惩罚程度
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -283,48 +283,60 @@ public class Word2VecWordSimilarity {
|
|
|
List<Term> terms1 = HanLP.segment(word1);
|
|
|
List<Term> terms2 = HanLP.segment(word2);
|
|
|
|
|
|
- // 提取主要词性(名词、动词、形容词)
|
|
|
- Set<String> mainNatures1 = terms1.stream()
|
|
|
- .map(t -> t.nature.toString())
|
|
|
- .filter(n -> n.startsWith("n") || n.startsWith("v") || n.startsWith("a"))
|
|
|
- .collect(Collectors.toSet());
|
|
|
- Set<String> mainNatures2 = terms2.stream()
|
|
|
- .map(t -> t.nature.toString())
|
|
|
- .filter(n -> n.startsWith("n") || n.startsWith("v") || n.startsWith("a"))
|
|
|
- .collect(Collectors.toSet());
|
|
|
+ log.info("分词结果: terms1={}, terms2={}", terms1, terms2);
|
|
|
|
|
|
- // 计算主要词性的交集
|
|
|
- Set<String> commonNatures = new HashSet<>(mainNatures1);
|
|
|
- commonNatures.retainAll(mainNatures2);
|
|
|
+ // 提取谓语动词和核心名词
|
|
|
+ Map<String, String> keyTerms1 = extractKeyTerms(terms1);
|
|
|
+ Map<String, String> keyTerms2 = extractKeyTerms(terms2);
|
|
|
|
|
|
- // 如果主要词性完全不同,直接返回false
|
|
|
- if (commonNatures.isEmpty() && !mainNatures1.isEmpty() && !mainNatures2.isEmpty()) {
|
|
|
- log.info("主要词性完全不同: natures1={}, natures2={}", mainNatures1, mainNatures2);
|
|
|
- return false;
|
|
|
- }
|
|
|
+ log.info("核心词分析: keyTerms1={}, keyTerms2={}", keyTerms1, keyTerms2);
|
|
|
+
|
|
|
+ // 检查谓语动词的相似性
|
|
|
+ boolean verbSimilar = checkVerbSimilarity(keyTerms1.get("verb"), keyTerms2.get("verb"));
|
|
|
|
|
|
+ // 检查核心名词的重叠
|
|
|
+ Set<String> nouns1 = Arrays.stream(keyTerms1.getOrDefault("noun", "").split(","))
|
|
|
+ .filter(s -> !s.isEmpty())
|
|
|
+ .collect(Collectors.toSet());
|
|
|
+ Set<String> nouns2 = Arrays.stream(keyTerms2.getOrDefault("noun", "").split(","))
|
|
|
+ .filter(s -> !s.isEmpty())
|
|
|
+ .collect(Collectors.toSet());
|
|
|
+ Set<String> commonNouns = new HashSet<>(nouns1);
|
|
|
+ commonNouns.retainAll(nouns2);
|
|
|
+
|
|
|
+ // 计算相似度
|
|
|
double similarity = calculateSimilarity(word1, word2);
|
|
|
|
|
|
- // 根据词性和长度差异动态调整阈值
|
|
|
+ // 动态调整阈值
|
|
|
double adjustedThreshold = threshold;
|
|
|
|
|
|
- // 如果词性差异大,提高阈值
|
|
|
- if (commonNatures.size() < Math.min(mainNatures1.size(), mainNatures2.size())) {
|
|
|
+ // 问句和陈述句的处理
|
|
|
+ boolean isQuestion1 = isQuestion(word1);
|
|
|
+ boolean isQuestion2 = isQuestion(word2);
|
|
|
+
|
|
|
+ // 如果一个是问句一个是陈述句,且谓语动词不相似,提高阈值
|
|
|
+ if (isQuestion1 != isQuestion2 && !verbSimilar) {
|
|
|
adjustedThreshold *= 1.3;
|
|
|
}
|
|
|
|
|
|
- // 如果长度差异大,提高阈值
|
|
|
+ // 如果谓语动词不相似,提高阈值
|
|
|
+ if (!verbSimilar) {
|
|
|
+ adjustedThreshold *= 1.2;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 长度差异调整
|
|
|
int lenDiff = Math.abs(terms1.size() - terms2.size());
|
|
|
if (lenDiff > 2) {
|
|
|
adjustedThreshold *= (1.0 + lenDiff * 0.1);
|
|
|
}
|
|
|
|
|
|
- boolean isSimilar = similarity >= adjustedThreshold;
|
|
|
+ boolean isSimilar = similarity >= adjustedThreshold &&
|
|
|
+ (verbSimilar || !commonNouns.isEmpty());
|
|
|
|
|
|
- log.info("相似度判断完成: rawSimilarity={}, adjustedThreshold={}, isSimilar={}, " +
|
|
|
- "mainNatures1={}, mainNatures2={}, commonNatures={}",
|
|
|
- similarity, adjustedThreshold, isSimilar,
|
|
|
- mainNatures1, mainNatures2, commonNatures);
|
|
|
+ log.info("相似度判断完成: rawSimilarity={}, adjustedThreshold={}, verbSimilar={}, " +
|
|
|
+ "commonNouns={}, isQuestion1={}, isQuestion2={}, isSimilar={}",
|
|
|
+ similarity, adjustedThreshold, verbSimilar,
|
|
|
+ commonNouns, isQuestion1, isQuestion2, isSimilar);
|
|
|
|
|
|
return isSimilar;
|
|
|
} catch (Exception e) {
|
|
@@ -333,6 +345,41 @@ public class Word2VecWordSimilarity {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ private static Map<String, String> extractKeyTerms(List<Term> terms) {
|
|
|
+ Map<String, String> keyTerms = new HashMap<>();
|
|
|
+ List<String> nouns = new ArrayList<>();
|
|
|
+ String mainVerb = null;
|
|
|
+
|
|
|
+ for (Term term : terms) {
|
|
|
+ String nature = term.nature.toString();
|
|
|
+ if (nature.startsWith("v") && mainVerb == null) {
|
|
|
+ mainVerb = term.word;
|
|
|
+ } else if (nature.startsWith("n")) {
|
|
|
+ nouns.add(term.word);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ keyTerms.put("verb", mainVerb);
|
|
|
+ keyTerms.put("noun", String.join(",", nouns));
|
|
|
+ return keyTerms;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean checkVerbSimilarity(String verb1, String verb2) {
|
|
|
+ if (verb1 == null || verb2 == null) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 定义一些同义动词组
|
|
|
+ Set<Set<String>> synonymGroups = new HashSet<>();
|
|
|
+ synonymGroups.add(new HashSet<>(Arrays.asList("穿", "戴", "套")));
|
|
|
+ synonymGroups.add(new HashSet<>(Arrays.asList("买", "购买", "选购")));
|
|
|
+ synonymGroups.add(new HashSet<>(Arrays.asList("洗", "清洗", "淋")));
|
|
|
+
|
|
|
+ // 检查两个动词是否属于同一个同义组
|
|
|
+ return verb1.equals(verb2) ||
|
|
|
+ synonymGroups.stream().anyMatch(group -> group.contains(verb1) && group.contains(verb2));
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* 获取与给定词最相似的N个词
|
|
|
*/
|
|
@@ -436,12 +483,62 @@ public class Word2VecWordSimilarity {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ private static boolean isQuestion(String text) {
|
|
|
+ // 问号判断
|
|
|
+ if (text.contains("?") || text.contains("?")) {
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 疑问词判断
|
|
|
+ Set<String> questionWords = new HashSet<>(Arrays.asList(
|
|
|
+ "什么", "怎么", "怎样", "如何", "哪", "谁", "为什么", "几",
|
|
|
+ "多少", "是否", "能否", "可否", "吗", "呢", "吧", "啊",
|
|
|
+ "嘛", "呀", "哪里", "哪儿", "何时", "为何", "多久"
|
|
|
+ ));
|
|
|
+
|
|
|
+ // 分词后检查是否包含疑问词
|
|
|
+ List<Term> terms = HanLP.segment(text);
|
|
|
+ for (Term term : terms) {
|
|
|
+ if (questionWords.contains(term.word)) {
|
|
|
+ log.debug("检测到疑问词: {}", term.word);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 语气词判断(句尾)
|
|
|
+ if (terms.size() > 0) {
|
|
|
+ String lastWord = terms.get(terms.size() - 1).word;
|
|
|
+ Set<String> questionTones = new HashSet<>(Arrays.asList(
|
|
|
+ "吗", "呢", "吧", "啊", "嘛", "呀", "么"
|
|
|
+ ));
|
|
|
+ if (questionTones.contains(lastWord)) {
|
|
|
+ log.debug("检测到句尾疑问语气词: {}", lastWord);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 特殊句式判断
|
|
|
+ String[] questionPatterns = {
|
|
|
+ "是不是", "对不对", "行不行", "要不要", "能不能", "可不可以",
|
|
|
+ "有没有", "对吧", "是吧", "好吧"
|
|
|
+ };
|
|
|
+ for (String pattern : questionPatterns) {
|
|
|
+ if (text.contains(pattern)) {
|
|
|
+ log.debug("检测到疑问句式: {}", pattern);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ log.debug("未检测到问句特征");
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
// 使用示例
|
|
|
public static void main(String[] args) {
|
|
|
try {
|
|
|
// 计算相似度
|
|
|
- String word1 = "你要选择什么产品名称?";
|
|
|
- String word2 = "产品代码";
|
|
|
+ String word1 = "你穿的是什么颜色的衣服?";
|
|
|
+ String word2 = "我穿的是卫衣";
|
|
|
// double similarity = calculateSimilarity(word1, word2);
|
|
|
//System.out.println("相似度: " + similarity);
|
|
|
|
|
@@ -450,11 +547,8 @@ public class Word2VecWordSimilarity {
|
|
|
// System.out.println("相似词: " + similarWords);
|
|
|
|
|
|
// 判断是否相似
|
|
|
- for(int i=0;i<=10;i++){
|
|
|
- boolean isSimilar = areWordsSimilar(word1, word2, 0.7);
|
|
|
- System.out.println("是否相似: " + isSimilar);
|
|
|
- }
|
|
|
-
|
|
|
+ boolean isSimilar = areWordsSimilar(word1, word2, 0.7);
|
|
|
+ System.out.println("是否相似: " + isSimilar);
|
|
|
|
|
|
} catch (Exception e) {
|
|
|
e.printStackTrace();
|