From 92b6e528281ca0060013193aa33dee043b29a757 Mon Sep 17 00:00:00 2001 From: DefectingCat Date: Thu, 1 Sep 2022 11:56:21 +0800 Subject: [PATCH] Add gists to algolia search --- package.json | 3 +- scripts/build-search.mjs | 112 ++++++++++----------------------------- scripts/gists/index.mjs | 86 ++++++++++++++++++++++++++++++ scripts/posts/index.mjs | 82 ++++++++++++++++++++++++++++ 4 files changed, 198 insertions(+), 85 deletions(-) create mode 100644 scripts/gists/index.mjs create mode 100644 scripts/posts/index.mjs diff --git a/package.json b/package.json index 5a3bdf5..50f340f 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "build": "next build && yarn build:search", "build:export": "cross-env NEXT_BUILD=export next build && cross-env NEXT_BUILD=export next export", "build:search": "node scripts/build-search.mjs", + "build:search-g": "node scripts/build-search.mjs -g", "start": "next start", "lint": "next lint", "test": "jest --watch", @@ -53,4 +54,4 @@ "tailwindcss": "^3.1.8", "typescript": "4.8.2" } -} +} \ No newline at end of file diff --git a/scripts/build-search.mjs b/scripts/build-search.mjs index 0107e60..3464161 100644 --- a/scripts/build-search.mjs +++ b/scripts/build-search.mjs @@ -1,85 +1,19 @@ +/** + * Generate algolia records. + * @params -t for test. + * @params -g add gitsts to records. + */ /* @ts-check */ import { config } from 'dotenv'; import algoliasearch from 'algoliasearch/lite.js'; -import fs from 'fs'; -import path from 'path'; - -const dataPath = 'data/posts'; - -/** - * Build post information for Algolia search. - * @param filename - * @returns - */ -const postLists = () => { - const files = fs.readdirSync(path.join(dataPath)); - - const myPosts = []; - files.map((f) => { - const content = fs.readFileSync(path.join(dataPath, f), 'utf-8'); - // const { data: meta, content } = matter(markdownWithMeta); - - const slug = f.replace(/\.mdx$/, ''); - const regex = /^#{2,3}(?!#)(.*)/gm; - - let lastH2 = ''; - const url = `https://rua.plus/p/${slug}#${head - .toLocaleLowerCase() - .replace(/ /g, '-')}`; - - content.match(regex)?.map((h) => { - const heading = h.split(' '); - const level = heading[0].length; - const head = h.substring(level + 1); - const record = { - content: null, - hierarchy: { - lvl0: 'Post', - lvl1: slug, - lvl2: head, - }, - type: `lvl${level}`, - objectID: url, - url, - }; - - switch (level) { - case 2: { - myPosts.push(record); - lastH2 = head; - break; - } - case 3: { - myPosts.push({ - ...record, - hierarchy: { - ...record.hierarchy, - lvl3: h.substring(level + 1), - }, - }); - break; - } - } - }); - - myPosts.push({ - content: null, - hierarchy: { - lvl0: 'Post', - lvl1: slug, - }, - type: 'lvl1', - objectID: url, - url, - }); - }); - return myPosts; -}; - -async function main() { - // initialize environment variables - config(); +import generateGists from './gists/index.mjs'; +import postLists from './posts/index.mjs'; +async function generateRecords(gists) { + const records = await postLists(); + return gists ? records.concat(await generateGists()) : records; +} +async function pushAlgolia(gists) { if ( !process.env.NEXT_PUBLIC_ALGOLIA_APP_ID && !process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_ADMIN_KEY @@ -88,7 +22,7 @@ async function main() { } try { - const posts = postLists(); + const records = await generateRecords(gists); // initialize the client with your environment variables const client = algoliasearch( @@ -100,7 +34,7 @@ async function main() { const index = client.initIndex('RUA'); // save the objects! - const algoliaResponse = await index.replaceAllObjects(posts); + const algoliaResponse = await index.replaceAllObjects(records); // check the output of the response in the console console.log( @@ -115,10 +49,20 @@ async function main() { } } -function test() { - const posts = postLists(); - posts.map((p) => console.log(p)); +async function test(gists) { + const records = await generateRecords(gists); + console.log(records); +} + +function main() { + // initialize environment variables + config(); + + const args = process.argv.slice(2); + const isTest = args.some((arg) => arg === '-t'); + const gists = args.some((arg) => arg === '-g'); + + isTest ? test(gists) : pushAlgolia(gists); } -// test(); main(); diff --git a/scripts/gists/index.mjs b/scripts/gists/index.mjs new file mode 100644 index 0000000..adda3b7 --- /dev/null +++ b/scripts/gists/index.mjs @@ -0,0 +1,86 @@ +/* @ts-check */ +import { Octokit } from 'octokit'; + +/** + * Get gists. + * @param {number} page + * @param {number} perPage + * @returns + */ +async function getGists(page, perPage) { + const password = process.env.NEXT_PUBLIC_GITHUB_API; + const octokit = new Octokit({ + auth: password, + // @TODO reverse proxy + baseUrl: 'http://api.github.com', + }); + return await octokit.rest.gists.list({ + page, + per_page: perPage, + }); +} + +async function generateRecords(gists, result) { + const pushGist = (d) => { + const url = `https://rua.plus/g/${d.id}`; + const files = d.files; + const record = { + content: null, + hierarchy: { + lvl0: 'Gist', + lvl1: d.description, + }, + type: `lvl1`, + objectID: url, + url, + }; + gists.push(record); + Object.keys(files).map((key) => { + gists.push({ + ...record, + hierarchy: { + ...record.hierarchy, + lvl2: files[key].filename, + }, + type: 'lvl2', + }); + }); + }; + result.data.map(pushGist); +} + +/** + * Generate all gists search records. + */ +async function generateGists() { + const linkMatch = /<(.*?)>/; + const relMatch = /"(.*?)"/; + const perPage = 50; + + const result = await getGists(1, perPage); + const link = result.headers.link?.split(','); + const pageSize = { + prev: null, + next: null, + last: null, + first: null, + }; + link.map((l) => { + const text = l.match(relMatch)?.[1]; + if (!text) return; + const page = new URLSearchParams(l.match(linkMatch)?.[1].split('?')[1]).get( + 'page' + ); + pageSize[text] = Number(page); + }); + const gists = []; + generateRecords(gists, result); + + for (let i = pageSize.next; i <= pageSize.last; i++) { + generateRecords(gists, await getGists(i, perPage)); + } + + return gists; +} + +export default generateGists; diff --git a/scripts/posts/index.mjs b/scripts/posts/index.mjs new file mode 100644 index 0000000..6166c41 --- /dev/null +++ b/scripts/posts/index.mjs @@ -0,0 +1,82 @@ +/* @ts-check */ +import fs from 'fs/promises'; +import path from 'path'; + +const dataPath = 'data/posts'; + +/** + * Build post information for Algolia search. + * @param filename + * @returns + */ +const postLists = async () => { + const files = await fs.readdir(path.join(dataPath)); + + const myPosts = []; + + const getFiles = async (f) => { + const content = await fs.readFile(path.join(dataPath, f), 'utf-8'); + // const { data: meta, content } = matter(markdownWithMeta); + + const slug = f.replace(/\.mdx$/, ''); + const regex = /^#{2,3}(?!#)(.*)/gm; + + let lastH2 = ''; + + const getContent = (h) => { + const heading = h.split(' '); + const level = heading[0].length; + const head = h.substring(level + 1); + const url = `https://rua.plus/p/${slug}#${head + .toLocaleLowerCase() + .replace(/ /g, '-')}`; + const record = { + content: null, + hierarchy: { + lvl0: 'Post', + lvl1: slug, + lvl2: head, + }, + type: `lvl${level}`, + objectID: url, + url, + }; + + switch (level) { + case 2: { + myPosts.push(record); + lastH2 = head; + break; + } + case 3: { + myPosts.push({ + ...record, + hierarchy: { + ...record.hierarchy, + lvl2: lastH2, + lvl3: h.substring(level + 1), + }, + }); + break; + } + } + }; + content.match(regex)?.map(getContent); + + myPosts.push({ + content: null, + hierarchy: { + lvl0: 'Post', + lvl1: slug, + }, + type: 'lvl1', + objectID: `https://rua.plus/p/${slug}`, + url: `https://rua.plus/p/${slug}`, + }); + }; + await Promise.all(files.map(getFiles)); + + return myPosts; +}; + +export default postLists;