Add gists to algolia search

package.json

@@ -7,6 +7,7 @@
     "build": "next build && yarn build:search",
     "build:export": "cross-env NEXT_BUILD=export next build && cross-env NEXT_BUILD=export next export",
     "build:search": "node scripts/build-search.mjs",
+    "build:search-g": "node scripts/build-search.mjs -g",
     "start": "next start",
     "lint": "next lint",
     "test": "jest --watch",
@@ -53,4 +54,4 @@
     "tailwindcss": "^3.1.8",
     "typescript": "4.8.2"
   }
-}
+}

scripts/build-search.mjs

@@ -1,85 +1,19 @@
+/**
+ * Generate algolia records.
+ * @params -t for test.
+ * @params -g add gitsts to records.
+ */
 /* @ts-check */
 import { config } from 'dotenv';
 import algoliasearch from 'algoliasearch/lite.js';
-import fs from 'fs';
-import path from 'path';
-
-const dataPath = 'data/posts';
-
-/**
- * Build post information for Algolia search.
- * @param filename
- * @returns
- */
-const postLists = () => {
-  const files = fs.readdirSync(path.join(dataPath));
-
-  const myPosts = [];
-  files.map((f) => {
-    const content = fs.readFileSync(path.join(dataPath, f), 'utf-8');
-    // const { data: meta, content } = matter(markdownWithMeta);
-
-    const slug = f.replace(/\.mdx$/, '');
-    const regex = /^#{2,3}(?!#)(.*)/gm;
-
-    let lastH2 = '';
-    const url = `https://rua.plus/p/${slug}#${head
-      .toLocaleLowerCase()
-      .replace(/ /g, '-')}`;
-
-    content.match(regex)?.map((h) => {
-      const heading = h.split(' ');
-      const level = heading[0].length;
-      const head = h.substring(level + 1);
-      const record = {
-        content: null,
-        hierarchy: {
-          lvl0: 'Post',
-          lvl1: slug,
-          lvl2: head,
-        },
-        type: `lvl${level}`,
-        objectID: url,
-        url,
-      };
-
-      switch (level) {
-        case 2: {
-          myPosts.push(record);
-          lastH2 = head;
-          break;
-        }
-        case 3: {
-          myPosts.push({
-            ...record,
-            hierarchy: {
-              ...record.hierarchy,
-              lvl3: h.substring(level + 1),
-            },
-          });
-          break;
-        }
-      }
-    });
-
-    myPosts.push({
-      content: null,
-      hierarchy: {
-        lvl0: 'Post',
-        lvl1: slug,
-      },
-      type: 'lvl1',
-      objectID: url,
-      url,
-    });
-  });
-  return myPosts;
-};
-
-async function main() {
-  // initialize environment variables
-  config();
+import generateGists from './gists/index.mjs';
+import postLists from './posts/index.mjs';
 
+async function generateRecords(gists) {
+  const records = await postLists();
+  return gists ? records.concat(await generateGists()) : records;
+}
+async function pushAlgolia(gists) {
   if (
     !process.env.NEXT_PUBLIC_ALGOLIA_APP_ID &&
     !process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_ADMIN_KEY
@@ -88,7 +22,7 @@ async function main() {
   }
 
   try {
-    const posts = postLists();
+    const records = await generateRecords(gists);
 
     // initialize the client with your environment variables
     const client = algoliasearch(
@@ -100,7 +34,7 @@ async function main() {
     const index = client.initIndex('RUA');
 
     // save the objects!
-    const algoliaResponse = await index.replaceAllObjects(posts);
+    const algoliaResponse = await index.replaceAllObjects(records);
 
     // check the output of the response in the console
     console.log(
@@ -115,10 +49,20 @@ async function main() {
   }
 }
 
-function test() {
-  const posts = postLists();
-  posts.map((p) => console.log(p));
+async function test(gists) {
+  const records = await generateRecords(gists);
+  console.log(records);
+}
+
+function main() {
+  // initialize environment variables
+  config();
+
+  const args = process.argv.slice(2);
+  const isTest = args.some((arg) => arg === '-t');
+  const gists = args.some((arg) => arg === '-g');
+
+  isTest ? test(gists) : pushAlgolia(gists);
 }
 
-// test();
 main();

scripts/gists/index.mjs

@@ -0,0 +1,86 @@
+/* @ts-check */
+import { Octokit } from 'octokit';
+
+/**
+ * Get gists.
+ * @param {number} page
+ * @param {number} perPage
+ * @returns
+ */
+async function getGists(page, perPage) {
+  const password = process.env.NEXT_PUBLIC_GITHUB_API;
+  const octokit = new Octokit({
+    auth: password,
+    // @TODO reverse proxy
+    baseUrl: 'http://api.github.com',
+  });
+  return await octokit.rest.gists.list({
+    page,
+    per_page: perPage,
+  });
+}
+
+async function generateRecords(gists, result) {
+  const pushGist = (d) => {
+    const url = `https://rua.plus/g/${d.id}`;
+    const files = d.files;
+    const record = {
+      content: null,
+      hierarchy: {
+        lvl0: 'Gist',
+        lvl1: d.description,
+      },
+      type: `lvl1`,
+      objectID: url,
+      url,
+    };
+    gists.push(record);
+    Object.keys(files).map((key) => {
+      gists.push({
+        ...record,
+        hierarchy: {
+          ...record.hierarchy,
+          lvl2: files[key].filename,
+        },
+        type: 'lvl2',
+      });
+    });
+  };
+  result.data.map(pushGist);
+}
+
+/**
+ * Generate all gists search records.
+ */
+async function generateGists() {
+  const linkMatch = /<(.*?)>/;
+  const relMatch = /"(.*?)"/;
+  const perPage = 50;
+
+  const result = await getGists(1, perPage);
+  const link = result.headers.link?.split(',');
+  const pageSize = {
+    prev: null,
+    next: null,
+    last: null,
+    first: null,
+  };
+  link.map((l) => {
+    const text = l.match(relMatch)?.[1];
+    if (!text) return;
+    const page = new URLSearchParams(l.match(linkMatch)?.[1].split('?')[1]).get(
+      'page'
+    );
+    pageSize[text] = Number(page);
+  });
+  const gists = [];
+  generateRecords(gists, result);
+
+  for (let i = pageSize.next; i <= pageSize.last; i++) {
+    generateRecords(gists, await getGists(i, perPage));
+  }
+
+  return gists;
+}
+
+export default generateGists;

scripts/posts/index.mjs

@@ -0,0 +1,82 @@
+/* @ts-check */
+import fs from 'fs/promises';
+import path from 'path';
+
+const dataPath = 'data/posts';
+
+/**
+ * Build post information for Algolia search.
+ * @param filename
+ * @returns
+ */
+const postLists = async () => {
+  const files = await fs.readdir(path.join(dataPath));
+
+  const myPosts = [];
+
+  const getFiles = async (f) => {
+    const content = await fs.readFile(path.join(dataPath, f), 'utf-8');
+    // const { data: meta, content } = matter(markdownWithMeta);
+
+    const slug = f.replace(/\.mdx$/, '');
+    const regex = /^#{2,3}(?!#)(.*)/gm;
+
+    let lastH2 = '';
+
+    const getContent = (h) => {
+      const heading = h.split(' ');
+      const level = heading[0].length;
+      const head = h.substring(level + 1);
+      const url = `https://rua.plus/p/${slug}#${head
+        .toLocaleLowerCase()
+        .replace(/ /g, '-')}`;
+      const record = {
+        content: null,
+        hierarchy: {
+          lvl0: 'Post',
+          lvl1: slug,
+          lvl2: head,
+        },
+        type: `lvl${level}`,
+        objectID: url,
+        url,
+      };
+
+      switch (level) {
+        case 2: {
+          myPosts.push(record);
+          lastH2 = head;
+          break;
+        }
+        case 3: {
+          myPosts.push({
+            ...record,
+            hierarchy: {
+              ...record.hierarchy,
+              lvl2: lastH2,
+              lvl3: h.substring(level + 1),
+            },
+          });
+          break;
+        }
+      }
+    };
+    content.match(regex)?.map(getContent);
+
+    myPosts.push({
+      content: null,
+      hierarchy: {
+        lvl0: 'Post',
+        lvl1: slug,
+      },
+      type: 'lvl1',
+      objectID: `https://rua.plus/p/${slug}`,
+      url: `https://rua.plus/p/${slug}`,
+    });
+  };
+  await Promise.all(files.map(getFiles));
+
+  return myPosts;
+};
+
+export default postLists;