feat: init webtools for demos

Author: phodal

cocoa-core/src/main/kotlin/cc/unitmesh/agent/Tool.kt

@@ -2,7 +2,9 @@ package cc.unitmesh.agent
 
 
 @Retention(AnnotationRetention.RUNTIME)
-@Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER)
+@Target(AnnotationTarget.FUNCTION, AnnotationTarget.PROPERTY_GETTER, AnnotationTarget.PROPERTY_SETTER,
+    AnnotationTarget.CLASS
+)
 annotation class Tool(
     val name: String = "",
     vararg val value: String = [""],

settings.gradle.kts

@@ -23,6 +23,8 @@ include(":llm-modules:connection-manager")
 include(":code:interpreter")
 include(":code:code-splitter")
 
+include(":tools:web-tools")
+
 dependencyResolutionManagement {
     repositories {
         mavenCentral()

tools/web-tools/build.gradle.kts

@@ -0,0 +1,17 @@
+@Suppress("DSL_SCOPE_VIOLATION")
+plugins {
+    java
+    alias(libs.plugins.jvm)
+    alias(libs.plugins.serialization)
+}
+
+dependencies {
+    implementation(projects.cocoaCore)
+    implementation(libs.kotlin.stdlib)
+    implementation(libs.serialization.json)
+
+    implementation("org.jsoup:jsoup:1.16.1")
+
+    testImplementation(libs.bundles.test)
+    testRuntimeOnly(libs.test.junit.engine)
+}

tools/web-tools/src/main/kotlin/cc/unitmesh/tools/web/Wikimedia.kt

@@ -0,0 +1,49 @@
+package cc.unitmesh.tools.web
+
+import cc.unitmesh.agent.Tool
+import org.jsoup.Jsoup
+import java.net.URLEncoder
+import java.nio.charset.StandardCharsets
+
+
+@Tool(name = "wikimedia", value = ["wikimedia"])
+class Wikimedia {
+    val headers = mapOf(
+        "User-Agent" to "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " +
+                "Chrome/117.0.0.0 Safari/537.36 Edg/113.0.1774.35"
+    )
+
+    private fun removeNestedParentheses(string: String): String {
+        val pattern = "\\([^()]+\\)".toRegex()
+        var result = string
+        while (pattern.containsMatchIn(result)) {
+            result = pattern.replace(result, "")
+        }
+        return result
+    }
+
+    val urlList = mutableListOf<String>()
+    fun run(title: String): MutableList<String> {
+        val url = url(title)
+        val doc = Jsoup.connect(url).headers(headers).get()
+        val mwDivs = doc.select("div.mw-search-result-heading")
+        if (mwDivs.isNotEmpty()) {
+            val resultTitles = mwDivs.map { it.text().trim() }
+                .map { removeNestedParentheses(it) }
+
+            urlList.addAll(resultTitles.map { url(it) })
+        } else {
+            val pageContent = (doc.select("p") + doc.select("ul")).map { it.text().trim() }
+            if (pageContent.any { "may refer to:" in it }) {
+                urlList.addAll(run("[$title]"))
+            } else {
+                urlList.add(url)
+            }
+        }
+
+        return urlList
+    }
+
+    private fun url(it: String) =
+        "https://en.wikipedia.org/w/index.php?search=${URLEncoder.encode(it, StandardCharsets.UTF_8)}"
+}

tools/web-tools/src/test/kotlin/cc/unitmesh/tools/web/WikimediaTest.kt

@@ -0,0 +1,14 @@
+package cc.unitmesh.tools.web;
+
+import org.junit.jupiter.api.Test
+import kotlin.test.Ignore
+
+class MyServiceTest {
+
+
+    @Test
+    @Ignore
+    fun should_return_url_list_when_run_with_valid_title() {
+        Wikimedia().run("Intellij").forEach { println(it) }
+    }
+}