Browse Source

代码提交

cxd 4 months ago
parent
commit
50445946e7

+ 129 - 35
gqy-common/src/main/java/com/gqy/common/utils/hanlp/Word2VecWordSimilarity.java

@@ -248,9 +248,9 @@ public class Word2VecWordSimilarity {
     }
     }
 
 
     private static double calculateLengthPenalty(int len1, int len2) {
     private static double calculateLengthPenalty(int len1, int len2) {
-        // 加强长度差异惩罚
+        // 对长度差异进行惩罚
         double ratio = Math.min(len1, len2) / (double) Math.max(len1, len2);
         double ratio = Math.min(len1, len2) / (double) Math.max(len1, len2);
-        return Math.pow(ratio, 1.2);  // 进一步增加惩罚力
+        return Math.pow(ratio, 0.5);  // 使用平方根来减轻惩罚程
     }
     }
 
 
     /**
     /**
@@ -283,48 +283,60 @@ public class Word2VecWordSimilarity {
             List<Term> terms1 = HanLP.segment(word1);
             List<Term> terms1 = HanLP.segment(word1);
             List<Term> terms2 = HanLP.segment(word2);
             List<Term> terms2 = HanLP.segment(word2);
 
 
-            // 提取主要词性(名词、动词、形容词)
-            Set<String> mainNatures1 = terms1.stream()
-                    .map(t -> t.nature.toString())
-                    .filter(n -> n.startsWith("n") || n.startsWith("v") || n.startsWith("a"))
-                    .collect(Collectors.toSet());
-            Set<String> mainNatures2 = terms2.stream()
-                    .map(t -> t.nature.toString())
-                    .filter(n -> n.startsWith("n") || n.startsWith("v") || n.startsWith("a"))
-                    .collect(Collectors.toSet());
+            log.info("分词结果: terms1={}, terms2={}", terms1, terms2);
 
 
-            // 计算主要词性的交集
-            Set<String> commonNatures = new HashSet<>(mainNatures1);
-            commonNatures.retainAll(mainNatures2);
+            // 提取谓语动词和核心名词
+            Map<String, String> keyTerms1 = extractKeyTerms(terms1);
+            Map<String, String> keyTerms2 = extractKeyTerms(terms2);
 
 
-            // 如果主要词性完全不同,直接返回false
-            if (commonNatures.isEmpty() && !mainNatures1.isEmpty() && !mainNatures2.isEmpty()) {
-                log.info("主要词性完全不同: natures1={}, natures2={}", mainNatures1, mainNatures2);
-                return false;
-            }
+            log.info("核心词分析: keyTerms1={}, keyTerms2={}", keyTerms1, keyTerms2);
+
+            // 检查谓语动词的相似性
+            boolean verbSimilar = checkVerbSimilarity(keyTerms1.get("verb"), keyTerms2.get("verb"));
 
 
+            // 检查核心名词的重叠
+            Set<String> nouns1 = Arrays.stream(keyTerms1.getOrDefault("noun", "").split(","))
+                    .filter(s -> !s.isEmpty())
+                    .collect(Collectors.toSet());
+            Set<String> nouns2 = Arrays.stream(keyTerms2.getOrDefault("noun", "").split(","))
+                    .filter(s -> !s.isEmpty())
+                    .collect(Collectors.toSet());
+            Set<String> commonNouns = new HashSet<>(nouns1);
+            commonNouns.retainAll(nouns2);
+
+            // 计算相似度
             double similarity = calculateSimilarity(word1, word2);
             double similarity = calculateSimilarity(word1, word2);
 
 
-            // 根据词性和长度差异动态调整阈值
+            // 动态调整阈值
             double adjustedThreshold = threshold;
             double adjustedThreshold = threshold;
 
 
-            // 如果词性差异大,提高阈值
-            if (commonNatures.size() < Math.min(mainNatures1.size(), mainNatures2.size())) {
+            // 问句和陈述句的处理
+            boolean isQuestion1 = isQuestion(word1);
+            boolean isQuestion2 = isQuestion(word2);
+
+            // 如果一个是问句一个是陈述句,且谓语动词不相似,提高阈值
+            if (isQuestion1 != isQuestion2 && !verbSimilar) {
                 adjustedThreshold *= 1.3;
                 adjustedThreshold *= 1.3;
             }
             }
 
 
-            // 如果长度差异大,提高阈值
+            // 如果谓语动词不相似,提高阈值
+            if (!verbSimilar) {
+                adjustedThreshold *= 1.2;
+            }
+
+            // 长度差异调整
             int lenDiff = Math.abs(terms1.size() - terms2.size());
             int lenDiff = Math.abs(terms1.size() - terms2.size());
             if (lenDiff > 2) {
             if (lenDiff > 2) {
                 adjustedThreshold *= (1.0 + lenDiff * 0.1);
                 adjustedThreshold *= (1.0 + lenDiff * 0.1);
             }
             }
 
 
-            boolean isSimilar = similarity >= adjustedThreshold;
+            boolean isSimilar = similarity >= adjustedThreshold &&
+                               (verbSimilar || !commonNouns.isEmpty());
 
 
-            log.info("相似度判断完成: rawSimilarity={}, adjustedThreshold={}, isSimilar={}, " +
-                    "mainNatures1={}, mainNatures2={}, commonNatures={}",
-                    similarity, adjustedThreshold, isSimilar,
-                    mainNatures1, mainNatures2, commonNatures);
+            log.info("相似度判断完成: rawSimilarity={}, adjustedThreshold={}, verbSimilar={}, " +
+                    "commonNouns={}, isQuestion1={}, isQuestion2={}, isSimilar={}",
+                    similarity, adjustedThreshold, verbSimilar,
+                    commonNouns, isQuestion1, isQuestion2, isSimilar);
 
 
             return isSimilar;
             return isSimilar;
         } catch (Exception e) {
         } catch (Exception e) {
@@ -333,6 +345,41 @@ public class Word2VecWordSimilarity {
         }
         }
     }
     }
 
 
+    private static Map<String, String> extractKeyTerms(List<Term> terms) {
+        Map<String, String> keyTerms = new HashMap<>();
+        List<String> nouns = new ArrayList<>();
+        String mainVerb = null;
+
+        for (Term term : terms) {
+            String nature = term.nature.toString();
+            if (nature.startsWith("v") && mainVerb == null) {
+                mainVerb = term.word;
+            } else if (nature.startsWith("n")) {
+                nouns.add(term.word);
+            }
+        }
+
+        keyTerms.put("verb", mainVerb);
+        keyTerms.put("noun", String.join(",", nouns));
+        return keyTerms;
+    }
+
+    private static boolean checkVerbSimilarity(String verb1, String verb2) {
+        if (verb1 == null || verb2 == null) {
+            return false;
+        }
+
+        // 定义一些同义动词组
+        Set<Set<String>> synonymGroups = new HashSet<>();
+        synonymGroups.add(new HashSet<>(Arrays.asList("穿", "戴", "套")));
+        synonymGroups.add(new HashSet<>(Arrays.asList("买", "购买", "选购")));
+        synonymGroups.add(new HashSet<>(Arrays.asList("洗", "清洗", "淋")));
+
+        // 检查两个动词是否属于同一个同义组
+        return verb1.equals(verb2) ||
+               synonymGroups.stream().anyMatch(group -> group.contains(verb1) && group.contains(verb2));
+    }
+
     /**
     /**
      * 获取与给定词最相似的N个词
      * 获取与给定词最相似的N个词
      */
      */
@@ -436,12 +483,62 @@ public class Word2VecWordSimilarity {
         }
         }
     }
     }
 
 
+    private static boolean isQuestion(String text) {
+        // 问号判断
+        if (text.contains("?") || text.contains("?")) {
+            return true;
+        }
+
+        // 疑问词判断
+        Set<String> questionWords = new HashSet<>(Arrays.asList(
+            "什么", "怎么", "怎样", "如何", "哪", "谁", "为什么", "几",
+            "多少", "是否", "能否", "可否", "吗", "呢", "吧", "啊",
+            "嘛", "呀", "哪里", "哪儿", "何时", "为何", "多久"
+        ));
+
+        // 分词后检查是否包含疑问词
+        List<Term> terms = HanLP.segment(text);
+        for (Term term : terms) {
+            if (questionWords.contains(term.word)) {
+                log.debug("检测到疑问词: {}", term.word);
+                return true;
+            }
+        }
+
+        // 语气词判断(句尾)
+        if (terms.size() > 0) {
+            String lastWord = terms.get(terms.size() - 1).word;
+            Set<String> questionTones = new HashSet<>(Arrays.asList(
+                "吗", "呢", "吧", "啊", "嘛", "呀", "么"
+            ));
+            if (questionTones.contains(lastWord)) {
+                log.debug("检测到句尾疑问语气词: {}", lastWord);
+                return true;
+            }
+        }
+
+        // 特殊句式判断
+        String[] questionPatterns = {
+            "是不是", "对不对", "行不行", "要不要", "能不能", "可不可以",
+            "有没有", "对吧", "是吧", "好吧"
+        };
+        for (String pattern : questionPatterns) {
+            if (text.contains(pattern)) {
+                log.debug("检测到疑问句式: {}", pattern);
+                return true;
+            }
+        }
+
+        log.debug("未检测到问句特征");
+        return false;
+    }
+
     // 使用示例
     // 使用示例
     public static void main(String[] args) {
     public static void main(String[] args) {
         try {
         try {
             // 计算相似度
             // 计算相似度
-            String word1 = "你要选择什么产品名称?";
-            String word2 = "产品代码";
+            String word1 = "你穿的是什么颜色的衣服?";
+            String word2 = "我穿的是卫衣";
           //  double similarity = calculateSimilarity(word1, word2);
           //  double similarity = calculateSimilarity(word1, word2);
             //System.out.println("相似度: " + similarity);
             //System.out.println("相似度: " + similarity);
 
 
@@ -450,11 +547,8 @@ public class Word2VecWordSimilarity {
 //            System.out.println("相似词: " + similarWords);
 //            System.out.println("相似词: " + similarWords);
 
 
             // 判断是否相似
             // 判断是否相似
-            for(int i=0;i<=10;i++){
-                boolean isSimilar = areWordsSimilar(word1, word2, 0.7);
-                System.out.println("是否相似: " + isSimilar);
-            }
-
+            boolean isSimilar = areWordsSimilar(word1, word2, 0.7);
+            System.out.println("是否相似: " + isSimilar);
 
 
         } catch (Exception e) {
         } catch (Exception e) {
             e.printStackTrace();
             e.printStackTrace();