[Gatsby] Flexsearch × Kuromoji による日本語フレンドリーなサイト内検索の追加

---
id: 1
title: 記事タイトル
date: '2016-10-12T10:00:00+09:00'
author: yamada_taro
description: 手動生成の抜粋
thumbnail: ../images/blog/1/thumb.jpg
tags:
  [
    'タグ1',
    'タグ2',
  ]
---
ここから本文

[
  {
    "id": "yamada_taro",
    "name": "山田 太郎",
    "name_reading": "Yamada Taro",
    "description": "著者の説明",
    "thumbnail": "../images/author/yamada_taro.png",
    "twitter": "",
    "qiita": "",
    "github": ""
  }
]

import { readdir, readFile, writeFile } from 'fs/promises'
import path from 'path'

const DIRECTORY_PATH = 'src/data/blog'
const AUTHOR_JSON_PATH = 'src/data/json/author.json'

 const files = await readdir(DIRECTORY_PATH, { withFileTypes: true })
 const filterFiles = files.filter((file) => file.isFile())
 const author = await readFile(path.resolve(AUTHOR_JSON_PATH))
 const authorJSON = JSON.parse(author.toString('utf8'))

  for (let i = 0; i < filterFiles.length; i++) {
    const file = filterFiles[i]
    documents.push(await parseMarkdownFile(file, authorJSON))
  }

import removeMarkdown from 'remove-markdown'
import grayMatter from 'gray-matter'
// ...
async function parseMarkdownFile(file, authorJSON) {
  const fp = path.join(DIRECTORY_PATH, file.name)

  const markdown = await readFile(fp)

  const matter = grayMatter(markdown)
  const content = removeMarkdown(matter.content)
  const author = authorJSON.find((auth) => auth.id === matter.data.author)

// ...

const document = {
  id: matter.data.id,
  title: matter.data.title,
  date: matter.data.date,
  author: author?.name,
  tag: matter.data.tags,
  keywords: '',
}

  import { tokenize, KuromojiToken } from 'kuromojin'

//...
  const str =
    `${matter.data.title}\n\n` +
    content

  const tokens = await tokenize(content, {
    dicPath: path.resolve(`./src/data/dict`),
  })

  document.keywords = createKeywords(tokens)

/**
 * キーワードのフィルタ
 */
function tokenFilter(token) {
  if (
    !['名詞', '動詞', '形容詞'].includes(token.pos) ||
    /^[!-/:-@[-`{-~、-〜”’・.,_\s\u02B0-\u02FF\u2010-\u27FF\u3001-\u303F\uFF01-\uFF0F\uFF1A-\uFF1E\uFF3B-\uFF40\uFF5B-\uFF65]+$/g.test(
      token.surface_form
    ) ||
    token.surface_form.length < 2
  ) {
    return false
  }
  switch (token.pos) {
    case '名詞':
    case '形容詞':
      return true
    case '動詞':
      return !['基本形', '連用形', '仮定形'].includes(token.conjugated_form)
      break
  }
}

// 表層形
if (!allTokens.includes(token.surface_form)) {
  allTokens.push(token.surface_form)
}

const reading = token.reading || token.surface_form

// ひらがな変換
const hira = reading.replace(/[\u30A2-\u30F3]/g, (m) =>
  String.fromCharCode(m.charCodeAt(0) - 96)
)

// 基本形
if (
  token.surface_form !== token.basic_form &&
  token.basic_form !== '*' &&
  !allTokens.includes(token.basic_form)
) {
  allTokens.push(token.basic_form)
}

// ひらがな
if (token.surface_form !== hira && !allTokens.includes(hira)) {
  allTokens.push(hira)
}

const DIST_FILE_PATH = 'src/data/flexsearch_index.json'
// ...
await writeFile(
  DIST_FILE_PATH,
  JSON.stringify({
    documents,
  })
)

import { copyFile } from 'fs/promises'
// ...
const onPostBootstrap: GatsbyNode['onPostBootstrap'] = async () => {
  try {
    await copyFile(
      'src/data/flexsearch_index.json',
      'public/flexsearch_index.json'
    )
    console.log('copy success flexsearch_index.json')
  } catch (e) {
    return console.error(e)
  }
}

import { Document } from 'flexsearch'

export function onClientEntry() {
  fetch(`${__PATH_PREFIX__}/flexsearch_index.json`)
    .then((response) => {
      return response.json()
    })
    .then(({ documents }) => {

      // Flexsearch
      const index = new Document({
        preset: 'match',
        tokenize: 'reverse',
        document: {
          index: ['keywords', 'author'],
          store: ['author', 'date', 'title'],
        },
      })

      // インデックスに追加
      documents.forEach((doc) => {
        index.add(doc)
      })

      window.__FLEXSEARCH__ = index
    })
    .catch((err) => {
      console.error(err)
    })
}

const [query, setQuery] = React.useState('')
const [results, setResults] = React.useState([])

const handleChange = (e: React.ChangeEvent<HTMLInputElement>) => {
    setQuery(e.target.value)

    if (!e.target.value || /^\s+$/.test(e.target.value)) {
      setResults([])
      return
    }

    const searchData = window.__FLEXSEARCH__.search(e.target.value, {
      enrich: true,
    })

    if (!searchData) {
      setResults([])
      return
    }
// ...

type StoreDoc = {
  author: string
  date: string
  title: string
}

type SearchResult = {
  id: number
  doc: StoreDoc
}

// ...

const documents = searchData.reduce(
      (acc: SearchResult[], cur: { field: string; result: [] }) => {
        cur.result.forEach((result: SearchResult) => {
          if (!acc.find((a: SearchResult) => a.id === result.id)) {
            acc.push(result)
          }
        })
        return acc
      },
      []
    )

setResults(documents)

{results.map(({ doc, id }: SearchResult) => {
  <div key={id}>
     {doc.title}
     {doc.author}
     {doc.date}
  </div>
})}

const isUtf8 = require('isutf8')
// L94
rl.on('line', (buf) => {
  if (isUtf8(buf)) {
    obs.next(iconv.decode(buf, 'utf8'));
  } else {
    obs.next(iconv.decode(buf, 'euc-jp'));
  }
});

const sentence = str.slice(
 Math.max(0, token.word_position - 30),
 Math.min(token.word_position + 30, str.length)
)

Error downloading object: dict/mecab-ipadic-2.7.0-20070801.tar.gz (b62f527): Smudge error: Error downloading dict/mecab-ipadic-2.7.0-20070801.tar.gz (b62f527d881c504576baed9c6ef6561554658b175ce6ae0096a60307e49e3523): batch response: This repository is over its data quota. Account responsible for LFS bandwidth should purchase more data packs to restore access.

node_modules/rxjs/Subscriber.js:246

[Error: ENOENT: no such file or directory, open '...../kuromoji-js-dictionary/.tmp/matrix.def'] {
errno: -2,
code: 'ENOENT',
syscall: 'open',
path: '...../kuromoji-js-dictionary/.tmp/matrix.def'
}

 The value of "length" is out of range. It must be >= 0 && <= 2147483647. Received 2684354560

if (query && window && typeof window.gtag === 'function') {
  window.gtag('event', 'view_search_results', {
     page_location: window.location.origin,
     page_path: window.location.pathname,
     page_title: 'サイト内検索',
     keyword: query.trim(),
  })
}