db: return a value of grn_rc type instead of void

.github/ISSUE_TEMPLATE/release.md

@@ -21,7 +21,7 @@ guide: http://groonga.org/ja/docs/contribution/development/release.html (Japanes
   - [ ] Announce
     - [ ] [GitHub Dicussions](https://github.com/groonga/groonga/discussions/categories/releases) (Japanese)
     - [ ] [GitHub Dicussions](https://github.com/groonga/groonga/discussions/categories/releases) (English)
-    - [ ] Twitter (Japanese/English)
+    - [ ] X (Japanese/English)
     - [ ] Facebook (Japanese/English)
 - [ ] Update documentation
   - [ ] Update .rst
@@ -34,7 +34,6 @@ guide: http://groonga.org/ja/docs/contribution/development/release.html (Japanes
   - [ ] Debian
   - [ ] Ubuntu (launchpad)
   - [ ] AlmaLinux
-  - [ ] CentOS
   - [ ] Windows
 - [ ] Upload documentation for https://groonga.org
 - [ ] Blog https://github.com/groonga/groonga.org
@@ -43,7 +42,7 @@ guide: http://groonga.org/ja/docs/contribution/development/release.html (Japanes
 - [ ] Announce
   - [ ] [GitHub Dicussions](https://github.com/groonga/groonga/discussions/categories/releases) (Japanese)
   - [ ] [GitHub Dicussions](https://github.com/groonga/groonga/discussions/categories/releases) (English)
-  - [ ] Twitter (Japanese/English)
+  - [ ] X (Japanese/English)
   - [ ] Facebook (Japanese/English)
 - [ ] Update `base_version`
 - [ ] Update `mingw-w64-groonga/PKGBUILD`

.github/workflows/autotools.yml

@@ -180,7 +180,7 @@ jobs:
 
   macos:
     name: macOS
-    runs-on: macos-12
+    runs-on: macos-latest
     timeout-minutes: 45
     steps:
       - uses: actions/checkout@v4
@@ -229,6 +229,8 @@ jobs:
           echo "TZ=Asia/Tokyo" >> ${GITHUB_ENV}
       - name: "Test: stdio"
         run: |
+          # Floating point number precision mismatch
+          rm -rf test/command/suite/tokenizers/document_vector_bm25
           grntest \
             --base-dir test/command \
             --n-retries=2 \

.github/workflows/cmake.yml

@@ -310,9 +310,44 @@ jobs:
             --interface=http \
             test/command/suite
 
+  docker:
+    name: Docker
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - "alpine-3.20"
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Login to GitHub Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Pull
+        continue-on-error: true
+        run: |
+          docker compose pull ${{ matrix.target }}
+      - name: Build
+        run: |
+          docker compose build ${{ matrix.target }}
+      - name: Run
+        run: |
+          docker compose run ${{ matrix.target }}
+      - name: Push
+        if: github.ref_name == 'main'
+        continue-on-error: true
+        run: |
+          docker compose push ${{ matrix.target }}
+
   macos:
     name: macOS
-    runs-on: macos-12
+    runs-on: macos-latest
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
@@ -374,26 +409,30 @@ jobs:
             --groonga-install-prefix=/usr/local \
             -v
       - name: "Test: mruby"
+        # TODO: Disabled temporarily as a workaround for the rpath issue in Rroonga.
+        if: false
         run: |
           USE_SYSTEM=yes test/mruby/run-test.rb
       - name: "Test: stdio"
         run: |
-          grntest \
-            --base-dir test/command \
-            --n-retries=2 \
-            --read-timeout=30 \
-            --reporter=mark \
-            test/command/suite
+          DYLD_LIBRARY_PATH=/usr/local/lib \
+            grntest \
+              --base-dir test/command \
+              --n-retries=2 \
+              --read-timeout=30 \
+              --reporter=mark \
+              test/command/suite
       - name: "Test: HTTP: load: Apache Arrow"
         run: |
-          grntest \
-            --base-dir test/command \
-            --n-retries=2 \
-            --read-timeout=30 \
-            --reporter=mark \
-            --input-type=apache-arrow \
-            --interface=http \
-            test/command/suite
+          DYLD_LIBRARY_PATH=/usr/local/lib \
+            grntest \
+              --base-dir test/command \
+              --n-retries=2 \
+              --read-timeout=30 \
+              --reporter=mark \
+              --input-type=apache-arrow \
+              --interface=http \
+              test/command/suite
 
   windows-mingw:
     name: Windows MinGW
@@ -719,9 +758,11 @@ jobs:
 
           $VC_REDIST_VERSION = (Get-Content "${Env:VC_PREFIX}\Auxiliary\Build\Microsoft.VCRedistVersion.default.txt")
           $VC_REDIST_DIR = "${Env:VC_PREFIX}\Redist\MSVC\${VC_REDIST_VERSION}\${Env:VC_ARCHITECTURE}\Microsoft.VC${{ matrix.vc-toolset-version }}.CRT"
+          $VC_REDIST_DIR_OPEN_MP = "${Env:VC_PREFIX}\Redist\MSVC\${VC_REDIST_VERSION}\${Env:VC_ARCHITECTURE}\Microsoft.VC${{ matrix.vc-toolset-version }}.OpenMP"
           $VC_REDIST_VCRUNTIME = "${VC_REDIST_DIR}\vcruntime140.dll"
           $VC_REDIST_VCRUNTIME_1 = "${VC_REDIST_DIR}\vcruntime140_1.dll"
           $VC_REDIST_MSVCP = "${VC_REDIST_DIR}\msvcp140.dll"
+          $VC_REDIST_VCOMP = "${VC_REDIST_DIR_OPEN_MP}\vcomp140.dll"
           Copy-Item `
             ${VC_REDIST_VCRUNTIME} `
             "${Env:FULL_INSTALL_FOLDER_WITH_VCRUNTIME}\bin"
@@ -733,6 +774,9 @@ jobs:
           Copy-Item `
             ${VC_REDIST_MSVCP} `
             "${Env:FULL_INSTALL_FOLDER_WITH_VCRUNTIME}\bin"
+          Copy-Item `
+            ${VC_REDIST_VCOMP} `
+            "${Env:FULL_INSTALL_FOLDER_WITH_VCRUNTIME}\bin"
           Copy-Item `
             "${Env:GITHUB_WORKSPACE}\packages\windows\vcruntime\vs${{ matrix.vs-version }}\readme.txt" `
             ${GROONGA_VC_REDIST_LICENSE_DIR}

.gitignore

@@ -138,6 +138,7 @@
 /vendor/CRoaring-*
 /vendor/c-blosc2-*
 /vendor/h3-*
+/vendor/llama.cpp-*
 /vendor/lz4-*
 /vendor/mecab-*
 /vendor/mecab/*.bin

.pre-commit-config.yaml

@@ -26,6 +26,7 @@ repos:
           ?include/groonga/column\.h|
           ?include/groonga/command_arguments\.hpp|
           ?include/groonga/dat\.h|
+          ?include/groonga/db\.h|
           ?include/groonga/distance\.h|
           ?include/groonga/error\.h|
           ?include/groonga/expr\.h|
@@ -35,6 +36,7 @@ repos:
           ?include/groonga/hash\.h|
           ?include/groonga/highlighter\.h|
           ?include/groonga/ii\.h|
+          ?include/groonga/language_model\.h|
           ?include/groonga/obj\.h|
           ?include/groonga/operator\.h|
           ?include/groonga/output\.h|
@@ -82,6 +84,7 @@ repos:
           ?lib/grn_geo\.h|
           ?lib/grn_ii\.h|
           ?lib/grn_io\.h|
+          ?lib/grn_language_model\.hpp|
           ?lib/grn_load\.h|
           ?lib/grn_mrb\.h|
           ?lib/grn_obj\.h|
@@ -102,6 +105,7 @@ repos:
           ?lib/h3\.c|
           ?lib/highlighter\.c|
           ?lib/ii\.c|
+          ?lib/language_model\.cpp|
           ?lib/load\.c|
           ?lib/normalizer\.c|
           ?lib/mrb/mrb_bulk\.c|
@@ -139,6 +143,7 @@ repos:
           ?lib/util\.c|
           ?lib/vector\.cpp|
           ?plugins/functions/h3\.c|
+          ?plugins/functions/language_model\.c|
           ?plugins/functions/number\.cpp|
           ?plugins/tokenizers/h3_index\.c|
           ?plugins/tokenizers/mecab\.c|

CMakeLists.txt

@@ -221,6 +221,10 @@ set(GRN_RELATIVE_RUBY_SCRIPTS_DIR
     "${CMAKE_INSTALL_LIBDIR}/${GRN_PROJECT_NAME}/scripts/ruby")
 set(GRN_RUBY_SCRIPTS_DIR
     "${CMAKE_INSTALL_PREFIX}/${GRN_RELATIVE_RUBY_SCRIPTS_DIR}")
+set(GRN_RELATIVE_LANGUAGE_MODELS_DIR
+    "${CMAKE_INSTALL_DATADIR}/${GRN_PROJECT_NAME}/language_models")
+set(GRN_LANGUAGE_MODELS_DIR
+    "${CMAKE_INSTALL_PREFIX}/${GRN_RELATIVE_LANGUAGE_MODELS_DIR}")
 
 if(GRN_C_COMPILER_GNU_LIKE)
   set(GRN_C_COMPILE_FLAGS "${GRN_C_COMPILE_FLAGS} $<$<CONFIG:Debug>:-g3 -O0>")
@@ -1499,6 +1503,71 @@ else()
   set(GRN_WITH_H3 FALSE)
 endif()
 
+set(GRN_LLAMA_CPP_BUNDLED_VERSION "b3861")
+set(GRN_LLAMA_CPP_BUNDLED_SHA256
+    "53fe4caa810979a1b9ab92cd0d6d3f2fc34337b507f4e0e8a7c9bd106a47413c")
+set(GRN_WITH_LLAMA_CPP
+    "auto"
+    CACHE STRING "Use llama.cpp for LLM.")
+set(GRN_WITH_LLAMA_CPP_BUNDLED FALSE)
+if(NOT "${GRN_WITH_LLAMA_CPP}" STREQUAL "no")
+  if("${GRN_WITH_LLAMA_CPP}" STREQUAL "bundled")
+    set(llama_FOUND FALSE)
+  else()
+    if("${GRN_WITH_LLAMA_CPP}" STREQUAL "system")
+      find_package(llama REQUIRED)
+    else()
+      find_package(llama)
+    endif()
+  endif()
+  if(llama_FOUND)
+    set(GRN_WITH_LLAMA_CPP TRUE)
+    target_link_libraries(grn_dependencies INTERFACE llama)
+    message(STATUS "llama.cpp: system")
+  elseif(CMAKE_VERSION VERSION_GREATER_EQUAL "3.20")
+    # We need to use function to create a variable scope
+    function(grn_build_llama_cpp)
+      set(LLAMA_CPP_SOURCE_BASE_NAME
+          "llama.cpp-${GRN_LLAMA_CPP_BUNDLED_VERSION}.tar.gz")
+      set(LLAMA_CPP_SOURCE_LOCAL_PATH
+          "${CMAKE_CURRENT_SOURCE_DIR}/vendor/${LLAMA_CPP_SOURCE_BASE_NAME}")
+      if(EXISTS ${LLAMA_CPP_SOURCE_LOCAL_PATH})
+        set(LLAMA_CPP_SOURCE_URL ${LLAMA_CPP_SOURCE_LOCAL_PATH})
+      else()
+        set(LLAMA_CPP_SOURCE_URL
+            "https://github.com/ggerganov/llama.cpp/archive/refs/tags")
+        string(APPEND LLAMA_CPP_SOURCE_URL
+               "/${GRN_LLAMA_CPP_BUNDLED_VERSION}.tar.gz")
+      endif()
+      fetchcontent_declare(
+        llama_cpp
+        ${GRN_FETCH_CONTENT_COMMON_OPTIONS}
+        URL ${LLAMA_CPP_SOURCE_URL}
+        URL_HASH "SHA256=${GRN_LLAMA_CPP_BUNDLED_SHA256}")
+      grn_prepare_fetchcontent()
+      set(LLAMA_ALL_WARNINGS OFF)
+      fetchcontent_makeavailable(llama_cpp)
+      if(CMAKE_VERSION VERSION_LESS 3.28)
+        set_property(DIRECTORY ${llama_cpp_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL
+                                                                TRUE)
+      endif()
+      install(
+        FILES "${llama_cpp_SOURCE_DIR}/AUTHORS"
+              "${llama_cpp_SOURCE_DIR}/LICENSE"
+              "${llama_cpp_SOURCE_DIR}/README.md"
+        DESTINATION "${GRN_DATA_DIR}/llama.cpp")
+    endfunction()
+    grn_build_llama_cpp()
+    target_link_libraries(grn_dependencies INTERFACE llama)
+    set(GRN_WITH_LLAMA_CPP_BUNDLED TRUE)
+    message(STATUS "llama.cpp: bundled")
+  else()
+    set(GRN_WITH_LLAMA_CPP FALSE)
+  endif()
+else()
+  set(GRN_WITH_LLAMA_CPP FALSE)
+endif()
+
 find_program(
   RUBY
   NAMES "ruby"

base_version

@@ -1 +1 @@
-14.0.8
+14.1.0

compose.yaml

@@ -0,0 +1,30 @@
+# Copyright (C) 2020-2024  Sutou Kouhei <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+services:
+  alpine-3.20:
+    image: ghcr.io/${GITHUB_REPOSITORY:-groonga/groonga}:ci-alpine-3.20
+    build:
+      cache_from:
+        - ghcr.io/${GITHUB_REPOSITORY:-groonga/groonga}:ci-alpine-3.20
+      context: dockerfiles
+      dockerfile: alpine.dockerfile
+      args:
+        ALPINE_LINUX_VERSION: "3.20"
+    volumes:
+      - .:/source:delegated
+    command:
+      /source/dockerfiles/run.sh
+

config.h.cmake.in

@@ -44,6 +44,10 @@
   "${GRN_RELATIVE_RUBY_SCRIPTS_DIR}"
 #define GRN_RUBY_SCRIPTS_DIR "${GRN_RUBY_SCRIPTS_DIR}"
 
+#define GRN_RELATIVE_LANGUAGE_MODELS_DIR \
+  "${GRN_RELATIVE_LANGUAGE_MODELS_DIR}"
+#define GRN_LANGUAGE_MODELS_DIR "${GRN_LANGUAGE_MODELS_DIR}"
+
 #define GRN_DLL_FILENAME     L"${GRN_DLL_FILENAME}"
 
 #cmakedefine GRN_WITH_WINDOWS_BACK_TRACE
@@ -90,6 +94,7 @@
 #cmakedefine GRN_WITH_H3_BUNDLED
 #cmakedefine GRN_WITH_KYTEA
 #cmakedefine GRN_WITH_LIBEDIT
+#cmakedefine GRN_WITH_LLAMA_CPP
 #cmakedefine GRN_WITH_LZ4
 #cmakedefine GRN_WITH_MECAB
 #cmakedefine GRN_WITH_MESSAGE_PACK

doc/files.am

@@ -720,8 +720,10 @@ absolute_source_files = \
 	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-katakana-v-sounds.log \
 	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-katakana-wo-sound.log \
 	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-katakana-zu-small-sounds.log \
+	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-latin-alphabet-with.log \
 	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-middle-dot.log \
 	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-prolonged-sound-mark.log \
+	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-to-katakana.log \
 	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-to-romaji-complex.log \
 	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-to-romaji.log \
 	$(top_srcdir)/doc/source/example/reference/normalizers/normalizer-nfkc150-unify-voiced-sound-mark-hiragana.log \
@@ -2039,8 +2041,10 @@ source_files_relative_from_doc_dir = \
 	source/example/reference/normalizers/normalizer-nfkc150-unify-katakana-v-sounds.log \
 	source/example/reference/normalizers/normalizer-nfkc150-unify-katakana-wo-sound.log \
 	source/example/reference/normalizers/normalizer-nfkc150-unify-katakana-zu-small-sounds.log \
+	source/example/reference/normalizers/normalizer-nfkc150-unify-latin-alphabet-with.log \
 	source/example/reference/normalizers/normalizer-nfkc150-unify-middle-dot.log \
 	source/example/reference/normalizers/normalizer-nfkc150-unify-prolonged-sound-mark.log \
+	source/example/reference/normalizers/normalizer-nfkc150-unify-to-katakana.log \
 	source/example/reference/normalizers/normalizer-nfkc150-unify-to-romaji-complex.log \
 	source/example/reference/normalizers/normalizer-nfkc150-unify-to-romaji.log \
 	source/example/reference/normalizers/normalizer-nfkc150-unify-voiced-sound-mark-hiragana.log \

doc/files.cmake

@@ -720,8 +720,10 @@ set(GRN_DOC_SOURCES
     example/reference/normalizers/normalizer-nfkc150-unify-katakana-v-sounds.log
     example/reference/normalizers/normalizer-nfkc150-unify-katakana-wo-sound.log
     example/reference/normalizers/normalizer-nfkc150-unify-katakana-zu-small-sounds.log
+    example/reference/normalizers/normalizer-nfkc150-unify-latin-alphabet-with.log
     example/reference/normalizers/normalizer-nfkc150-unify-middle-dot.log
     example/reference/normalizers/normalizer-nfkc150-unify-prolonged-sound-mark.log
+    example/reference/normalizers/normalizer-nfkc150-unify-to-katakana.log
     example/reference/normalizers/normalizer-nfkc150-unify-to-romaji-complex.log
     example/reference/normalizers/normalizer-nfkc150-unify-to-romaji.log
     example/reference/normalizers/normalizer-nfkc150-unify-voiced-sound-mark-hiragana.log