diff --git a/assets/js/vertex-search.js b/assets/js/vertex-search.js new file mode 100644 index 00000000..179dd05c --- /dev/null +++ b/assets/js/vertex-search.js @@ -0,0 +1,247 @@ +import { marked } from 'marked'; +import DOMPurify from 'dompurify'; + +(function () { + 'use strict'; + + marked.setOptions({ + gfm: true, + breaks: true + }); + + // Only run on the search results page + const config = document.getElementById('vertex-search-config'); + if (!config) return; + + const FUNCTION_URL = config.dataset.searchUrl; + if (!FUNCTION_URL) { + console.warn('Vertex search URL not configured'); + return; + } + + // Get query from URL params (?q=...) + const urlParams = new URLSearchParams(window.location.search); + const query = urlParams.get('q') || ''; + + const statusEl = document.getElementById('vertex-search-status'); + const hitsEl = document.getElementById('vertex-search-hits'); + const summaryEl = document.getElementById('vertex-search-summary'); + const summaryTxt = document.getElementById('summary-text'); + + function escapeHtml(str) { + if (!str) return ''; + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); + } + + function renderSummaryMarkdown(summaryText, citationCount) { + const linkedMarkdown = (summaryText || '').replace(/\[(\d+)\]/g, (_, num) => { + const index = Number(num); + if (!Number.isInteger(index) || index < 1 || index > citationCount) return `[${num}]`; + return `[${num}](#source-row-${index})`; + }); + + const html = marked.parse(linkedMarkdown); + return DOMPurify.sanitize(html, { + ALLOWED_TAGS: [ + 'a', 'p', 'ul', 'ol', 'li', 'strong', 'em', 'code', 'pre', + 'blockquote', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'br' + ], + ALLOWED_ATTR: ['href', 'title', 'target', 'rel', 'id', 'class'] + }); + } + + function renderResultSnippet(snippetText, result) { + if (!snippetText) return ''; + const normalized = normalizeSnippetText(snippetText, result); + const html = marked.parse(normalized); + return DOMPurify.sanitize(html, { + ALLOWED_TAGS: ['p', 'b', 'strong', 'em', 'code', 'pre', 'br', 'ul', 'ol', 'li', 'a'], + ALLOWED_ATTR: ['href', 'title', 'target', 'rel'] + }); + } + + function decodeHtmlEntities(text) { + const textarea = document.createElement('textarea'); + textarea.innerHTML = String(text || ''); + return textarea.value; + } + + function normalizeSnippetText(text, result) { + let output = decodeHtmlEntities(text); + + // Repair malformed highlight markers like "bIssue/b" or "bmedley issues/b". + output = output.replace(/(^|[\s(\[{])b([^\n]{1,120}?)\/b(?=[\s)\]}.,;:!?]|$)/gi, + (match, prefix, value) => `${prefix}${value.trim()}`); + + // Repair malformed URLs where emphasis markers leak into the URL text. + output = output + .replace(/https?:\/\/b/gi, 'https://') + .replace(/\bb([a-z0-9.-]+)\/b/gi, '$1') + .replace(/\/(b)([a-z0-9._-]+)/gi, '/$2') + .replace(/([a-z0-9._-]+)\/b(?=\/|\b)/gi, '$1') + .replace(/\s*\.\.\.\s*/g, ' '); + + output = normalizeGithubIssueMentions(output, result); + + // Drop noisy raw URL tails often appended by snippet extraction. + output = output.replace(/https?:\/\/\S+$/i, ''); + + // Normalize whitespace for cleaner one-line snippets. + output = output.replace(/\s+/g, ' ').trim(); + return output; + } + + function normalizeGithubIssueMentions(text, result) { + const repo = result?.repo; + if (!repo) return text; + + // Fix malformed markdown links like: [Issue 609](https://bgithub/b.com/Interlisp/bmedley/b/bissues/b/609) + let output = text.replace( + /\[(Issue\s+#?\d+)\]\((https?:\/\/[^)]+)\)/gi, + (match, label, rawUrl) => { + const issueNumberMatch = label.match(/(\d+)/); + if (!issueNumberMatch) return match; + const issueNumber = issueNumberMatch[1]; + const cleanUrl = `https://github.com/Interlisp/${repo}/issues/${issueNumber}`; + return `[${label}](${cleanUrl})`; + } + ); + + // Convert plain text mentions like "Issue 609" or "Issue #609" into links. + output = output.replace(/\bIssue\s+#?(\d+)\b/g, (match, num) => { + const issueUrl = `https://github.com/Interlisp/${repo}/issues/${num}`; + return `[Issue ${num}](${issueUrl})`; + }); + + return output; + } + + function formatDisplayUrl(rawUrl) { + if (!rawUrl) return ''; + + try { + const parsed = new URL(rawUrl); + const host = parsed.host.replace(/^www\./, ''); + const pathSegments = parsed.pathname.split('/').filter(Boolean); + const path = pathSegments.length > 0 ? ` › ${pathSegments.join(' › ')}` : ''; + return `${host}${path}`; + } catch (_) { + return rawUrl; + } + } + + async function doSearch(q) { + if (!q) { + statusEl.textContent = 'Enter a search query above.'; + return; + } + + // Update page title to reflect query + document.title = `Search: ${q}`; + statusEl.innerHTML = ' Searching for ' + + escapeHtml(q) + '…'; + + try { + const url = new URL(FUNCTION_URL); + url.searchParams.set('q', q); + + // Pass referring section as context if coming from a content page + const ref = document.referrer; + if (ref) { + try { + const refPath = new URL(ref).pathname.split('/').filter(Boolean); + if (refPath.length > 0) url.searchParams.set('context', refPath[0]); + } catch (_) {} + } + + const resp = await fetch(url.toString()); + if (!resp.ok) throw new Error(`HTTP ${resp.status}`); + const data = await resp.json(); + + // Hide spinner + statusEl.style.display = 'none'; + + // Show AI summary if available + if (data.summary?.summaryText) { + const refs = data.summary?.citations || []; + const validRefs = refs.filter(r => r.title || r.uri); + + summaryTxt.innerHTML = renderSummaryMarkdown(data.summary.summaryText, validRefs.length); + + const inlineRefs = summaryTxt.querySelectorAll('a[href^="#source-row-"]'); + inlineRefs.forEach((el) => { + el.classList.add('summary-inline-citation'); + el.setAttribute('aria-label', `Jump to source ${el.textContent}`); + }); + + summaryEl.style.display = 'block'; + + // Remove stale source rows before rendering the latest list. + const previousCitations = summaryEl.querySelector('.search-citations'); + if (previousCitations) previousCitations.remove(); + + if (validRefs.length > 0) { + const citationHtml = validRefs.map((ref, i) => ` +
+ [${i + 1}] + ${ref.uri + ? `${escapeHtml(ref.title || ref.uri)}` + : `${escapeHtml(ref.title || 'Unknown source')}` + } +
+ `).join(''); + + const citationsDiv = document.createElement('div'); + citationsDiv.className = 'search-citations mt-3'; + citationsDiv.innerHTML = '

Sources

' + citationHtml; + summaryEl.querySelector('.ai-summary').appendChild(citationsDiv); + } + } else { + summaryTxt.innerHTML = ''; + summaryEl.style.display = 'none'; + } + + // Show results + if (!data.results || data.results.length === 0) { + hitsEl.innerHTML = `

No results found for ${escapeHtml(q)}.

`; + return; + } + + // Show result count + hitsEl.innerHTML = + `

${data.results.length} results for ${escapeHtml(q)}

` + + data.results.map(r => ` +
+
+ ${escapeHtml(r.title || 'Untitled')} +
+

${escapeHtml(formatDisplayUrl(r.url))}

+ ${r.snippet ? `
${renderResultSnippet(r.snippet, r)}
` : ''} + ${(r.type || r.repo || r.state) + ? `

+ ${r.type ? `${escapeHtml(r.type)}` : ''} + ${r.repo ? `${escapeHtml(r.repo)}` : ''} + ${r.state ? `${escapeHtml(r.state)}` : ''} +

` + : ''} +
+ `).join(''); + + } catch (err) { + statusEl.textContent = 'Search error: ' + err.message; + console.error('Vertex search error:', err); + } + } + + // Run search on page load + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', () => doSearch(query)); + } else { + doSearch(query); + } + +})(); \ No newline at end of file diff --git a/assets/scss/_styles_project.scss b/assets/scss/_styles_project.scss index 7f05b07e..d1c67314 100644 --- a/assets/scss/_styles_project.scss +++ b/assets/scss/_styles_project.scss @@ -93,3 +93,170 @@ padding: 0.5rem 0; } } + +// Vertex AI Search styles +.ai-summary { + background: #f0f7ff; + border-left: 4px solid #1a73e8; + padding: 1rem 1.25rem; + margin-bottom: 1.5rem; + border-radius: 0 4px 4px 0; + + .ai-badge { + font-size: 0.75rem; + font-weight: 600; + color: #1a73e8; + text-transform: uppercase; + letter-spacing: 0.05em; + display: block; + margin-bottom: 0.5rem; + } + + p { margin: 0 0 0.75rem; } + + p:last-child { + margin-bottom: 0; + } + + ul, + ol { + margin: 0.5rem 0 0.75rem 1.25rem; + } + + li { + margin-bottom: 0.35rem; + } + + h1, + h2, + h3, + h4, + h5, + h6 { + margin: 0.9rem 0 0.5rem; + font-size: 1rem; + font-weight: 700; + color: #124fa8; + } + + code { + color: #0b4aa2; + background: rgba(26, 115, 232, 0.1); + border-radius: 3px; + padding: 0 0.2rem; + } +} + +.summary-inline-citation { + margin-left: 0.1rem; + text-decoration: none; + font-weight: 600; + color: #1a73e8; + + &:hover, + &:focus { + text-decoration: underline; + } +} + +.td-search-hit { + padding: 1rem 0; + border-bottom: 1px solid #e8e8e8; + + &:last-child { border-bottom: none; } + + &__title { + margin: 0 0 0.25rem; + font-size: 1rem; + line-height: 1.35; + a { + color: #1a0dab; + text-decoration: none; + } + + a:hover, + a:focus { + text-decoration: underline; + } + } + + &__snippet { + font-size: 0.875rem; + color: #545454; + margin: 0.25rem 0; + + p { + margin: 0 0 0.4rem; + } + + p:last-child { + margin-bottom: 0; + } + + b, + strong { + color: #1f1f1f; + font-weight: 700; + } + } + + &__url { + color: #0b8043; + margin: 0; + word-break: break-word; + } + + &__meta { + margin-top: 0.4rem; + } +} + +.search-meta-chip { + display: inline-block; + margin-right: 0.35rem; + margin-bottom: 0.2rem; + padding: 0.08rem 0.45rem; + font-size: 0.72rem; + border-radius: 999px; + background: #eef2f7; + color: #4a5568; + border: 1px solid #dde3eb; +} + +.td-search-loading, +.td-search-no-results, +.td-search-error { + padding: 1rem 0; + color: #666; +} + +.search-citations { + border-top: 1px solid #d0e4ff; + padding-top: 0.75rem; + margin-top: 0.75rem; + + .citations-label { + font-size: 0.7rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: #1a73e8; + margin-bottom: 0.4rem; + } + } + + .search-citation { + font-size: 0.8rem; + margin-bottom: 0.25rem; + + .citation-number { + color: #1a73e8; + font-weight: 600; + margin-right: 0.4rem; + } + + a { + color: #444; + &:hover { color: #1a73e8; } + } + } \ No newline at end of file diff --git a/config/_default/params.yaml b/config/_default/params.yaml index c4f0ba91..9b7c6ffe 100644 --- a/config/_default/params.yaml +++ b/config/_default/params.yaml @@ -57,6 +57,9 @@ copyright: # gcs_engine_id: search engine gcs_engine_id: 33ef4cbe0703b4f3a +# Vertex AI Search Cloud Function URL +vertex_search_url: "https://us-central1-interlispsearch.cloudfunctions.net/search" + # Zotero Group ID for bibliography management zotero_group_id: 2914042 diff --git a/layouts/_partials/hooks/head-end.html b/layouts/_partials/hooks/head-end.html index f800ce9f..d4466940 100644 --- a/layouts/_partials/hooks/head-end.html +++ b/layouts/_partials/hooks/head-end.html @@ -1,3 +1,6 @@ + +{{ $searchJS := resources.Get "js/vertex-search.js" | js.Build | fingerprint }} + {{ partial "bibliography-json-ld.html" . }} diff --git a/layouts/search.html b/layouts/search.html new file mode 100644 index 00000000..8739b3d1 --- /dev/null +++ b/layouts/search.html @@ -0,0 +1,33 @@ +{{ define "main" }} +
+
+

Search Results

+ +
+ + {{/* Pass the function URL to JS */}} + + + + +
+ Searching... +
+ +
+ +
+
+
+{{ end }} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 82470702..3c49465a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2,7 +2,9 @@ "name": "Interlisp.github.io", - "lockfileVersion": 2, + "version": "1.0.0", + + "lockfileVersion": 3, "requires": true, @@ -10,12 +12,14 @@ "": { - "name": "Interlisp.github.io", - "dependencies": { + "dompurify": "^3.3.1", + "jquery": "^3.7.1", + "marked": "^15.0.12", + "tabpanel": "^0.2.0" }, @@ -44,6 +48,8 @@ "dev": true, + "license": "ISC", + "dependencies": { "minipass": "^7.0.4" @@ -68,6 +74,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "@nodelib/fs.stat": "2.0.5", @@ -94,6 +102,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">= 8" @@ -112,6 +122,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "@nodelib/fs.scandir": "2.1.5", @@ -128,67 +140,35 @@ }, - "node_modules/adm-zip": { - - "version": "0.5.16", - - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", - - "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", - - "dev": true, - - "engines": { - - "node": ">=12.0" - - } - - }, - - "node_modules/ansi-regex": { - - "version": "5.0.1", - - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "node_modules/@types/trusted-types": { - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "version": "2.0.7", - "dev": true, + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", - "engines": { + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", - "node": ">=8" + "license": "MIT", - } + "optional": true }, - "node_modules/ansi-styles": { + "node_modules/adm-zip": { - "version": "4.3.0", + "version": "0.5.17", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.17.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "integrity": "sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ==", "dev": true, - "dependencies": { - - "color-convert": "^2.0.1" - - }, + "license": "MIT", "engines": { - "node": ">=8" - - }, - - "funding": { - - "url": "https://github.com/chalk/ansi-styles?sponsor=1" + "node": ">=12.0" } @@ -196,14 +176,16 @@ "node_modules/anymatch": { - "version": "3.1.2", + "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", "dev": true, + "license": "ISC", + "dependencies": { "normalize-path": "^3.0.0", @@ -222,11 +204,11 @@ "node_modules/autoprefixer": { - "version": "10.4.8", + "version": "10.5.0", - "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.8.tgz", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.5.0.tgz", - "integrity": "sha512-75Jr6Q/XpTqEf6D2ltS5uMewJIx5irCU1oBYJrWjFenq/m12WRRrz6g15L1EIoYvPLXTbEry7rDOwrcYNj77xw==", + "integrity": "sha512-FMhOoZV4+qR6aTUALKX2rEqGG+oyATvwBt9IIzVR5rMa2HRWPkxf+P+PAJLD1I/H5/II+HuZcBJYEFBpq39ong==", "dev": true, @@ -246,21 +228,29 @@ "url": "https://tidelift.com/funding/github/npm/autoprefixer" + }, + + { + + "type": "github", + + "url": "https://github.com/sponsors/ai" + } ], - "dependencies": { + "license": "MIT", - "browserslist": "^4.21.3", + "dependencies": { - "caniuse-lite": "^1.0.30001373", + "browserslist": "^4.28.2", - "fraction.js": "^4.2.0", + "caniuse-lite": "^1.0.30001787", - "normalize-range": "^0.1.2", + "fraction.js": "^5.3.4", - "picocolors": "^1.0.0", + "picocolors": "^1.1.1", "postcss-value-parser": "^4.2.0" @@ -286,20 +276,54 @@ }, + "node_modules/baseline-browser-mapping": { + + "version": "2.10.29", + + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.29.tgz", + + "integrity": "sha512-Asa2krT+XTPZINCS+2QcyS8WTkObE77RwkydwF7h6DmnKqbvlalz93m/dnphUyCa6SWSP51VgtEUf2FN+gelFQ==", + + "dev": true, + + "license": "Apache-2.0", + + "bin": { + + "baseline-browser-mapping": "dist/cli.cjs" + + }, + + "engines": { + + "node": ">=6.0.0" + + } + + }, + "node_modules/binary-extensions": { - "version": "2.2.0", + "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", "dev": true, + "license": "MIT", + "engines": { "node": ">=8" + }, + + "funding": { + + "url": "https://github.com/sponsors/sindresorhus" + } }, @@ -314,6 +338,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "fill-range": "^7.1.1" @@ -330,11 +356,11 @@ "node_modules/browserslist": { - "version": "4.21.3", + "version": "4.28.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.3.tgz", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", - "integrity": "sha512-898rgRXLAyRkM1GryrrBHGkqA5hlpkV5MhtZwg9QXeiyLUYs2k00Un05aX5l2/yJIOObYKOpS2JNo8nJDE7fWQ==", + "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", "dev": true, @@ -354,19 +380,31 @@ "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + + { + + "type": "github", + + "url": "https://github.com/sponsors/ai" + } ], + "license": "MIT", + "dependencies": { - "caniuse-lite": "^1.0.30001370", + "baseline-browser-mapping": "^2.10.12", - "electron-to-chromium": "^1.4.202", + "caniuse-lite": "^1.0.30001782", - "node-releases": "^2.0.6", + "electron-to-chromium": "^1.5.328", - "update-browserslist-db": "^1.0.5" + "node-releases": "^2.0.36", + + "update-browserslist-db": "^1.2.3" }, @@ -386,11 +424,11 @@ "node_modules/caniuse-lite": { - "version": "1.0.30001378", + "version": "1.0.30001792", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001378.tgz", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001792.tgz", - "integrity": "sha512-JVQnfoO7FK7WvU4ZkBRbPjaot4+YqxogSDosHv0Hv5mWpUESmN+UubMU6L/hGz8QlQ2aY5U0vR6MOs6j/CXpNA==", + "integrity": "sha512-hVLMUZFgR4JJ6ACt1uEESvQN1/dBVqPAKY0hgrV70eN3391K6juAfTjKZLKvOMsx8PxA7gsY1/tLMMTcfFLLpw==", "dev": true, @@ -410,33 +448,33 @@ "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - } + }, - ] + { - }, + "type": "github", - "node_modules/chokidar": { + "url": "https://github.com/sponsors/ai" - "version": "3.5.3", + } - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + ], - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "license": "CC-BY-4.0" - "dev": true, + }, - "funding": [ + "node_modules/chokidar": { - { + "version": "3.6.0", - "type": "individual", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", - "url": "https://paulmillr.com/funding/" + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", - } + "dev": true, - ], + "license": "MIT", "dependencies": { @@ -462,6 +500,12 @@ }, + "funding": { + + "url": "https://paulmillr.com/funding/" + + }, + "optionalDependencies": { "fsevents": "~2.3.2" @@ -480,6 +524,8 @@ "dev": true, + "license": "BlueOak-1.0.0", + "engines": { "node": ">=18" @@ -490,93 +536,69 @@ "node_modules/cliui": { - "version": "7.0.4", + "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", "dev": true, + "license": "ISC", + "dependencies": { "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", + "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" - } - - }, - - "node_modules/color-convert": { - - "version": "2.0.1", - - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - - "dev": true, - - "dependencies": { - - "color-name": "~1.1.4" - }, "engines": { - "node": ">=7.0.0" + "node": ">=12" } }, - "node_modules/color-name": { - - "version": "1.1.4", - - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - - "dev": true - - }, - - "node_modules/dependency-graph": { + "node_modules/cliui/node_modules/ansi-regex": { - "version": "0.11.0", + "version": "5.0.1", - "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, + "license": "MIT", + "engines": { - "node": ">= 0.6.0" + "node": ">=8" } }, - "node_modules/dir-glob": { + "node_modules/cliui/node_modules/ansi-styles": { - "version": "3.0.1", + "version": "4.3.0", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dev": true, + "license": "MIT", + "dependencies": { - "path-type": "^4.0.0" + "color-convert": "^2.0.1" }, @@ -584,23 +606,17 @@ "node": ">=8" - } - - }, - - "node_modules/electron-to-chromium": { - - "version": "1.4.224", + }, - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.224.tgz", + "funding": { - "integrity": "sha512-dOujC5Yzj0nOVE23iD5HKqrRSDj2SD7RazpZS/b/WX85MtO6/LzKDF4TlYZTBteB+7fvSg5JpWh0sN7fImNF8w==", + "url": "https://github.com/chalk/ansi-styles?sponsor=1" - "dev": true + } }, - "node_modules/emoji-regex": { + "node_modules/cliui/node_modules/emoji-regex": { "version": "8.0.0", @@ -608,91 +624,311 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "dev": true, + + "license": "MIT" }, - "node_modules/escalade": { + "node_modules/cliui/node_modules/string-width": { - "version": "3.1.1", + "version": "4.2.3", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dev": true, + "license": "MIT", + + "dependencies": { + + "emoji-regex": "^8.0.0", + + "is-fullwidth-code-point": "^3.0.0", + + "strip-ansi": "^6.0.1" + + }, + "engines": { - "node": ">=6" + "node": ">=8" } }, - "node_modules/fast-glob": { + "node_modules/cliui/node_modules/strip-ansi": { - "version": "3.2.11", + "version": "6.0.1", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.11.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", "dev": true, - "dependencies": { - - "@nodelib/fs.stat": "^2.0.2", - - "@nodelib/fs.walk": "^1.2.3", - - "glob-parent": "^5.1.2", + "license": "MIT", - "merge2": "^1.3.0", + "dependencies": { - "micromatch": "^4.0.4" + "ansi-regex": "^5.0.1" }, "engines": { - "node": ">=8.6.0" + "node": ">=8" } }, - "node_modules/fastq": { + "node_modules/cliui/node_modules/wrap-ansi": { - "version": "1.13.0", + "version": "7.0.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "dev": true, + "license": "MIT", + "dependencies": { - "reusify": "^1.0.4" + "ansi-styles": "^4.0.0", + + "string-width": "^4.1.0", + + "strip-ansi": "^6.0.0" + + }, + + "engines": { + + "node": ">=10" + + }, + + "funding": { + + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/fill-range": { + "node_modules/color-convert": { - "version": "7.1.1", + "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", "dev": true, + "license": "MIT", + "dependencies": { - "to-regex-range": "^5.0.1" + "color-name": "~1.1.4" + + }, + + "engines": { + + "node": ">=7.0.0" + + } + + }, + + "node_modules/color-name": { + + "version": "1.1.4", + + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + + "dev": true, + + "license": "MIT" + + }, + + "node_modules/dependency-graph": { + + "version": "0.11.0", + + "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz", + + "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==", + + "dev": true, + + "license": "MIT", + + "engines": { + + "node": ">= 0.6.0" + + } + + }, + + "node_modules/dir-glob": { + + "version": "3.0.1", + + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + + "dev": true, + + "license": "MIT", + + "dependencies": { + + "path-type": "^4.0.0" + + }, + + "engines": { + + "node": ">=8" + + } + + }, + + "node_modules/dompurify": { + + "version": "3.4.3", + + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.3.tgz", + + "integrity": "sha512-VVwJidIJcp1hpg2OMXML3ZVRPYSZiq4aX7qBh83BSIpOaRDqI+qxhXjjIWnpzkOXhmp0L81lnoME1mnCc9H48A==", + + "license": "(MPL-2.0 OR Apache-2.0)", + + "optionalDependencies": { + + "@types/trusted-types": "^2.0.7" + + } + + }, + + "node_modules/electron-to-chromium": { + + "version": "1.5.354", + + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.354.tgz", + + "integrity": "sha512-JaBHwWcfIdmSAfWM5l3uwjGd431j8YEMikZ+K/2nXVuBqJKyZ0f+2h4n4JY5AyNiZmnY9qQr2RU3v9DxDmHMNg==", + + "dev": true, + + "license": "ISC" + + }, + + "node_modules/escalade": { + + "version": "3.2.0", + + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + + "dev": true, + + "license": "MIT", + + "engines": { + + "node": ">=6" + + } + + }, + + "node_modules/fast-glob": { + + "version": "3.3.3", + + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + + "dev": true, + + "license": "MIT", + + "dependencies": { + + "@nodelib/fs.stat": "^2.0.2", + + "@nodelib/fs.walk": "^1.2.3", + + "glob-parent": "^5.1.2", + + "merge2": "^1.3.0", + + "micromatch": "^4.0.8" + + }, + + "engines": { + + "node": ">=8.6.0" + + } + + }, + + "node_modules/fastq": { + + "version": "1.20.1", + + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + + "dev": true, + + "license": "ISC", + + "dependencies": { + + "reusify": "^1.0.4" + + } + + }, + + "node_modules/fill-range": { + + "version": "7.1.1", + + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + + "dev": true, + + "license": "MIT", + + "dependencies": { + + "to-regex-range": "^5.0.1" }, @@ -706,14 +942,16 @@ "node_modules/fraction.js": { - "version": "4.2.0", + "version": "5.3.4", - "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", - "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==", + "integrity": "sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==", "dev": true, + "license": "MIT", + "engines": { "node": "*" @@ -722,9 +960,9 @@ "funding": { - "type": "patreon", + "type": "github", - "url": "https://www.patreon.com/infusion" + "url": "https://github.com/sponsors/rawify" } @@ -732,14 +970,16 @@ "node_modules/fs-extra": { - "version": "10.1.0", + "version": "11.3.5", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.5.tgz", - "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "integrity": "sha512-eKpRKAovdpZtR1WopLHxlBWvAgPny3c4gX1G5Jhwmmw4XJj0ifSD5qB5TOo8hmA0wlRKDAOAhEE1yVPgs6Fgcg==", "dev": true, + "license": "MIT", + "dependencies": { "graceful-fs": "^4.2.0", @@ -752,7 +992,7 @@ "engines": { - "node": ">=12" + "node": ">=14.14" } @@ -760,16 +1000,18 @@ "node_modules/fsevents": { - "version": "2.3.2", + "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "dev": true, "hasInstallScript": true, + "license": "MIT", + "optional": true, "os": [ @@ -796,6 +1038,8 @@ "dev": true, + "license": "ISC", + "engines": { "node": "6.* || 8.* || >= 10.*" @@ -814,6 +1058,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">=12" @@ -838,6 +1084,8 @@ "dev": true, + "license": "ISC", + "dependencies": { "is-glob": "^4.0.1" @@ -854,21 +1102,23 @@ "node_modules/globby": { - "version": "13.1.2", + "version": "13.2.2", - "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", + "resolved": "https://registry.npmjs.org/globby/-/globby-13.2.2.tgz", - "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", + "integrity": "sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==", "dev": true, + "license": "MIT", + "dependencies": { "dir-glob": "^3.0.1", - "fast-glob": "^3.2.11", + "fast-glob": "^3.3.0", - "ignore": "^5.2.0", + "ignore": "^5.2.4", "merge2": "^1.4.1", @@ -890,15 +1140,43 @@ }, + "node_modules/globby/node_modules/slash": { + + "version": "4.0.0", + + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + + "dev": true, + + "license": "MIT", + + "engines": { + + "node": ">=12" + + }, + + "funding": { + + "url": "https://github.com/sponsors/sindresorhus" + + } + + }, + "node_modules/graceful-fs": { - "version": "4.2.10", + "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true + "dev": true, + + "license": "ISC" }, @@ -914,6 +1192,8 @@ "hasInstallScript": true, + "license": "MIT", + "dependencies": { "adm-zip": "^0.5.16", @@ -940,14 +1220,16 @@ "node_modules/ignore": { - "version": "5.2.0", + "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", "dev": true, + "license": "MIT", + "engines": { "node": ">= 4" @@ -966,6 +1248,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "binary-extensions": "^2.0.0" @@ -990,6 +1274,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">=0.10.0" @@ -1008,6 +1294,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">=8" @@ -1026,6 +1314,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "is-extglob": "^2.1.1" @@ -1050,6 +1340,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">=0.12.0" @@ -1064,20 +1356,24 @@ "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.7.1.tgz", - "integrity": "sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg==" + "integrity": "sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg==", + + "license": "MIT" }, "node_modules/jsonfile": { - "version": "6.1.0", + "version": "6.2.1", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", - "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", "dev": true, + "license": "MIT", + "dependencies": { "universalify": "^2.0.0" @@ -1094,17 +1390,49 @@ "node_modules/lilconfig": { - "version": "2.0.6", + "version": "3.1.3", - "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.0.6.tgz", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", - "integrity": "sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg==", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", "dev": true, + "license": "MIT", + "engines": { - "node": ">=10" + "node": ">=14" + + }, + + "funding": { + + "url": "https://github.com/sponsors/antonk52" + + } + + }, + + "node_modules/marked": { + + "version": "15.0.12", + + "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.12.tgz", + + "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", + + "license": "MIT", + + "bin": { + + "marked": "bin/marked.js" + + }, + + "engines": { + + "node": ">= 18" } @@ -1120,6 +1448,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">= 8" @@ -1138,6 +1468,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "braces": "^3.0.3", @@ -1156,14 +1488,16 @@ "node_modules/minipass": { - "version": "7.1.2", + "version": "7.1.3", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", "dev": true, + "license": "BlueOak-1.0.0", + "engines": { "node": ">=16 || 14 >=14.17" @@ -1182,6 +1516,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "minipass": "^7.1.2" @@ -1198,11 +1534,11 @@ "node_modules/nanoid": { - "version": "3.3.11", + "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", "dev": true, @@ -1218,6 +1554,8 @@ ], + "license": "MIT", + "bin": { "nanoid": "bin/nanoid.cjs" @@ -1234,13 +1572,15 @@ "node_modules/node-releases": { - "version": "2.0.6", + "version": "2.0.44", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.44.tgz", - "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==", + "integrity": "sha512-5WUyunoPMsvvEhS8AxHtRzP+oA8UCkJ7YRxatWKjngndhDGLiqEVAQKWjFAiAiuL8zMRGzGSJxFnLetoa43qGQ==", + + "dev": true, - "dev": true + "license": "MIT" }, @@ -1254,6 +1594,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">=0.10.0" @@ -1262,64 +1604,52 @@ }, - "node_modules/normalize-range": { + "node_modules/path-type": { - "version": "0.1.2", + "version": "4.0.0", - "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha1-LRDAa9/TEuqXd2laTShDlFa3WUI=", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true, + "license": "MIT", + "engines": { - "node": ">=0.10.0" + "node": ">=8" } }, - "node_modules/path-type": { + "node_modules/picocolors": { - "version": "4.0.0", - - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - - "dev": true, - - "engines": { - - "node": ">=8" - - } - - }, - - "node_modules/picocolors": { - - "version": "1.1.1", + "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true + "dev": true, + + "license": "ISC" }, "node_modules/picomatch": { - "version": "2.3.1", + "version": "2.3.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, + "license": "MIT", + "engines": { "node": ">=8.6" @@ -1344,6 +1674,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">=0.10.0" @@ -1354,11 +1686,11 @@ "node_modules/postcss": { - "version": "8.5.6", + "version": "8.5.14", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.14.tgz", - "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "integrity": "sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==", "dev": true, @@ -1390,6 +1722,8 @@ ], + "license": "MIT", + "dependencies": { "nanoid": "^3.3.11", @@ -1410,21 +1744,23 @@ "node_modules/postcss-cli": { - "version": "10.0.0", + "version": "10.1.0", - "resolved": "https://registry.npmjs.org/postcss-cli/-/postcss-cli-10.0.0.tgz", + "resolved": "https://registry.npmjs.org/postcss-cli/-/postcss-cli-10.1.0.tgz", - "integrity": "sha512-Wjy/00wBBEgQqnSToznxLWDnATznokFGXsHtF/3G8glRZpz5KYlfHcBW/VMJmWAeF2x49zjgy4izjM3/Wx1dKA==", + "integrity": "sha512-Zu7PLORkE9YwNdvOeOVKPmWghprOtjFQU3srMUGbdz3pHJiFh7yZ4geiZFMkjMfB0mtTFR3h8RemR62rPkbOPA==", "dev": true, + "license": "MIT", + "dependencies": { "chokidar": "^3.3.0", "dependency-graph": "^0.11.0", - "fs-extra": "^10.0.0", + "fs-extra": "^11.0.0", "get-stdin": "^9.0.0", @@ -1440,7 +1776,7 @@ "read-cache": "^1.0.0", - "slash": "^4.0.0", + "slash": "^5.0.0", "yargs": "^17.0.0" @@ -1468,33 +1804,47 @@ "node_modules/postcss-load-config": { - "version": "4.0.1", + "version": "4.0.2", - "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.1.tgz", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.2.tgz", - "integrity": "sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA==", + "integrity": "sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==", "dev": true, - "dependencies": { + "funding": [ - "lilconfig": "^2.0.5", + { - "yaml": "^2.1.1" + "type": "opencollective", - }, + "url": "https://opencollective.com/postcss/" - "engines": { + }, - "node": ">= 14" + { - }, + "type": "github", - "funding": { + "url": "https://github.com/sponsors/ai" + + } + + ], + + "license": "MIT", + + "dependencies": { - "type": "opencollective", + "lilconfig": "^3.0.0", - "url": "https://opencollective.com/postcss/" + "yaml": "^2.3.4" + + }, + + "engines": { + + "node": ">= 14" }, @@ -1526,14 +1876,36 @@ "node_modules/postcss-reporter": { - "version": "7.0.5", + "version": "7.1.0", - "resolved": "https://registry.npmjs.org/postcss-reporter/-/postcss-reporter-7.0.5.tgz", + "resolved": "https://registry.npmjs.org/postcss-reporter/-/postcss-reporter-7.1.0.tgz", - "integrity": "sha512-glWg7VZBilooZGOFPhN9msJ3FQs19Hie7l5a/eE6WglzYqVeH3ong3ShFcp9kDWJT1g2Y/wd59cocf9XxBtkWA==", + "integrity": "sha512-/eoEylGWyy6/DOiMP5lmFRdmDKThqgn7D6hP2dXKJI/0rJSO1ADFNngZfDzxL0YAxFvws+Rtpuji1YIHj4mySA==", "dev": true, + "funding": [ + + { + + "type": "opencollective", + + "url": "https://opencollective.com/postcss/" + + }, + + { + + "type": "github", + + "url": "https://github.com/sponsors/ai" + + } + + ], + + "license": "MIT", + "dependencies": { "picocolors": "^1.0.0", @@ -1548,14 +1920,6 @@ }, - "funding": { - - "type": "opencollective", - - "url": "https://opencollective.com/postcss/" - - }, - "peerDependencies": { "postcss": "^8.1.0" @@ -1572,7 +1936,9 @@ "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", - "dev": true + "dev": true, + + "license": "MIT" }, @@ -1586,6 +1952,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">= 0.8" @@ -1630,7 +1998,9 @@ } - ] + ], + + "license": "MIT" }, @@ -1644,6 +2014,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "pify": "^2.3.0" @@ -1662,6 +2034,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "picomatch": "^2.2.1" @@ -1686,6 +2060,8 @@ "dev": true, + "license": "MIT", + "engines": { "node": ">=0.10.0" @@ -1696,14 +2072,16 @@ "node_modules/reusify": { - "version": "1.0.4", + "version": "1.1.0", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", "dev": true, + "license": "MIT", + "engines": { "iojs": ">=1.0.0", @@ -1752,6 +2130,8 @@ ], + "license": "MIT", + "dependencies": { "queue-microtask": "^1.2.2" @@ -1762,17 +2142,19 @@ "node_modules/slash": { - "version": "4.0.0", + "version": "5.1.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", - "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==", "dev": true, + "license": "MIT", + "engines": { - "node": ">=12" + "node": ">=14.16" }, @@ -1794,61 +2176,11 @@ "dev": true, - "engines": { - - "node": ">=0.10.0" - - } - - }, - - "node_modules/string-width": { - - "version": "4.2.3", - - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - - "dev": true, - - "dependencies": { - - "emoji-regex": "^8.0.0", - - "is-fullwidth-code-point": "^3.0.0", - - "strip-ansi": "^6.0.1" - - }, - - "engines": { - - "node": ">=8" - - } - - }, - - "node_modules/strip-ansi": { - - "version": "6.0.1", - - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - - "dev": true, - - "dependencies": { - - "ansi-regex": "^5.0.1" - - }, + "license": "BSD-3-Clause", "engines": { - "node": ">=8" + "node": ">=0.10.0" } @@ -1866,14 +2198,16 @@ "node_modules/tar": { - "version": "7.5.7", + "version": "7.5.15", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.7.tgz", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.15.tgz", - "integrity": "sha512-fov56fJiRuThVFXD6o6/Q354S7pnWMJIVlDBYijsTNx6jKSE4pvrDTs6lUnmGvNyfJwFQQwWy3owKz1ucIhveQ==", + "integrity": "sha512-dzGK0boVlC4W5QFuQN1EFSl3bIDYsk7Tj40U6eIBnK2k/8ml7TZ5agbI5j5+qnoVcAA+rNtBml8SEiLxZpNqRQ==", "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { "@isaacs/fs-minipass": "^4.0.0", @@ -1898,13 +2232,15 @@ "node_modules/thenby": { - "version": "1.3.4", + "version": "1.4.1", + + "resolved": "https://registry.npmjs.org/thenby/-/thenby-1.4.1.tgz", - "resolved": "https://registry.npmjs.org/thenby/-/thenby-1.3.4.tgz", + "integrity": "sha512-D5a/bO0KdalOE3q8MlrRmSxjbKZHT3MQmXkJP+r97Vw8MMwOZKOwUSEyTtK7eSMj2y0kyAjpYMRMZmmLw1FtNQ==", - "integrity": "sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==", + "dev": true, - "dev": true + "license": "Apache-2.0" }, @@ -1918,6 +2254,8 @@ "dev": true, + "license": "MIT", + "dependencies": { "is-number": "^7.0.0" @@ -1934,14 +2272,16 @@ "node_modules/universalify": { - "version": "2.0.0", + "version": "2.0.1", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", - "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", "dev": true, + "license": "MIT", + "engines": { "node": ">= 10.0.0" @@ -1952,11 +2292,11 @@ "node_modules/update-browserslist-db": { - "version": "1.0.5", + "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.5.tgz", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-dteFFpCyvuDdr9S/ff1ISkKt/9YZxKjI9WlRR99c180GaztJtRa/fn18FdxGVKVsnPY7/a/FDN68mcvUmP4U7Q==", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", "dev": true, @@ -1976,61 +2316,37 @@ "url": "https://tidelift.com/funding/github/npm/browserslist" - } - - ], - - "dependencies": { - - "escalade": "^3.1.1", - - "picocolors": "^1.0.0" - - }, - - "bin": { - - "browserslist-lint": "cli.js" - - }, - - "peerDependencies": { - - "browserslist": ">= 4.21.0" - - } + }, - }, + { - "node_modules/wrap-ansi": { + "type": "github", - "version": "7.0.0", + "url": "https://github.com/sponsors/ai" - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + } - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + ], - "dev": true, + "license": "MIT", "dependencies": { - "ansi-styles": "^4.0.0", - - "string-width": "^4.1.0", + "escalade": "^3.2.0", - "strip-ansi": "^6.0.0" + "picocolors": "^1.1.1" }, - "engines": { + "bin": { - "node": ">=10" + "update-browserslist-db": "cli.js" }, - "funding": { + "peerDependencies": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + "browserslist": ">= 4.21.0" } @@ -2046,6 +2362,8 @@ "dev": true, + "license": "ISC", + "engines": { "node": ">=10" @@ -2064,6 +2382,8 @@ "dev": true, + "license": "BlueOak-1.0.0", + "engines": { "node": ">=18" @@ -2074,14 +2394,16 @@ "node_modules/yaml": { - "version": "2.8.2", + "version": "2.9.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", "dev": true, + "license": "ISC", + "bin": { "yaml": "bin.mjs" @@ -2104,17 +2426,19 @@ "node_modules/yargs": { - "version": "17.5.1", + "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.5.1.tgz", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-t6YAJcxDkNX7NFYiVtKvWUz8l+PaKTLiL63mJYWR2GnHq2gjEWISzsLp9wg3aY36dY1j+gfIEL3pIF+XlJJfbA==", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "dev": true, + "license": "MIT", + "dependencies": { - "cliui": "^7.0.2", + "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -2126,7 +2450,7 @@ "y18n": "^5.0.5", - "yargs-parser": "^21.0.0" + "yargs-parser": "^21.1.1" }, @@ -2148,1376 +2472,104 @@ "dev": true, + "license": "ISC", + "engines": { "node": ">=12" } - } - - }, - - "dependencies": { + }, - "@isaacs/fs-minipass": { + "node_modules/yargs/node_modules/ansi-regex": { - "version": "4.0.1", + "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, - "requires": { + "license": "MIT", - "minipass": "^7.0.4" + "engines": { + + "node": ">=8" } }, - "@nodelib/fs.scandir": { + "node_modules/yargs/node_modules/emoji-regex": { - "version": "2.1.5", + "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true, - "requires": { - - "@nodelib/fs.stat": "2.0.5", - - "run-parallel": "^1.1.9" - - } + "license": "MIT" }, - "@nodelib/fs.stat": { - - "version": "2.0.5", + "node_modules/yargs/node_modules/string-width": { - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "version": "4.2.3", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "dev": true + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - }, + "dev": true, - "@nodelib/fs.walk": { + "license": "MIT", - "version": "1.2.8", + "dependencies": { - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "emoji-regex": "^8.0.0", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "is-fullwidth-code-point": "^3.0.0", - "dev": true, + "strip-ansi": "^6.0.1" - "requires": { + }, - "@nodelib/fs.scandir": "2.1.5", + "engines": { - "fastq": "^1.6.0" + "node": ">=8" } }, - "adm-zip": { + "node_modules/yargs/node_modules/strip-ansi": { - "version": "0.5.16", + "version": "6.0.1", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true + "dev": true, - }, + "license": "MIT", - "ansi-regex": { - - "version": "5.0.1", - - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - - "dev": true - - }, - - "ansi-styles": { - - "version": "4.3.0", - - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - - "dev": true, - - "requires": { - - "color-convert": "^2.0.1" - - } - - }, - - "anymatch": { - - "version": "3.1.2", - - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", - - "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - - "dev": true, - - "requires": { - - "normalize-path": "^3.0.0", - - "picomatch": "^2.0.4" - - } - - }, - - "autoprefixer": { - - "version": "10.4.8", - - "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.8.tgz", - - "integrity": "sha512-75Jr6Q/XpTqEf6D2ltS5uMewJIx5irCU1oBYJrWjFenq/m12WRRrz6g15L1EIoYvPLXTbEry7rDOwrcYNj77xw==", - - "dev": true, - - "requires": { - - "browserslist": "^4.21.3", - - "caniuse-lite": "^1.0.30001373", - - "fraction.js": "^4.2.0", - - "normalize-range": "^0.1.2", - - "picocolors": "^1.0.0", - - "postcss-value-parser": "^4.2.0" - - } - - }, - - "binary-extensions": { - - "version": "2.2.0", - - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", - - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - - "dev": true - - }, - - "braces": { - - "version": "3.0.3", - - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - - "dev": true, - - "requires": { - - "fill-range": "^7.1.1" - - } - - }, - - "browserslist": { - - "version": "4.21.3", - - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.3.tgz", - - "integrity": "sha512-898rgRXLAyRkM1GryrrBHGkqA5hlpkV5MhtZwg9QXeiyLUYs2k00Un05aX5l2/yJIOObYKOpS2JNo8nJDE7fWQ==", - - "dev": true, - - "requires": { - - "caniuse-lite": "^1.0.30001370", - - "electron-to-chromium": "^1.4.202", - - "node-releases": "^2.0.6", - - "update-browserslist-db": "^1.0.5" - - } - - }, - - "caniuse-lite": { - - "version": "1.0.30001378", - - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001378.tgz", - - "integrity": "sha512-JVQnfoO7FK7WvU4ZkBRbPjaot4+YqxogSDosHv0Hv5mWpUESmN+UubMU6L/hGz8QlQ2aY5U0vR6MOs6j/CXpNA==", - - "dev": true - - }, - - "chokidar": { - - "version": "3.5.3", - - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - - "dev": true, - - "requires": { - - "anymatch": "~3.1.2", - - "braces": "~3.0.2", - - "fsevents": "~2.3.2", - - "glob-parent": "~5.1.2", - - "is-binary-path": "~2.1.0", - - "is-glob": "~4.0.1", - - "normalize-path": "~3.0.0", - - "readdirp": "~3.6.0" - - } - - }, - - "chownr": { - - "version": "3.0.0", - - "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", - - "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", - - "dev": true - - }, - - "cliui": { - - "version": "7.0.4", - - "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", - - "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - - "dev": true, - - "requires": { - - "string-width": "^4.2.0", - - "strip-ansi": "^6.0.0", - - "wrap-ansi": "^7.0.0" - - } - - }, - - "color-convert": { - - "version": "2.0.1", - - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - - "dev": true, - - "requires": { - - "color-name": "~1.1.4" - - } - - }, - - "color-name": { - - "version": "1.1.4", - - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - - "dev": true - - }, - - "dependency-graph": { - - "version": "0.11.0", - - "resolved": "https://registry.npmjs.org/dependency-graph/-/dependency-graph-0.11.0.tgz", - - "integrity": "sha512-JeMq7fEshyepOWDfcfHK06N3MhyPhz++vtqWhMT5O9A3K42rdsEDpfdVqjaqaAhsw6a+ZqeDvQVtD0hFHQWrzg==", - - "dev": true - - }, - - "dir-glob": { - - "version": "3.0.1", - - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - - "dev": true, - - "requires": { - - "path-type": "^4.0.0" - - } - - }, - - "electron-to-chromium": { - - "version": "1.4.224", - - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.224.tgz", - - "integrity": "sha512-dOujC5Yzj0nOVE23iD5HKqrRSDj2SD7RazpZS/b/WX85MtO6/LzKDF4TlYZTBteB+7fvSg5JpWh0sN7fImNF8w==", - - "dev": true - - }, - - "emoji-regex": { - - "version": "8.0.0", - - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - - "dev": true - - }, - - "escalade": { - - "version": "3.1.1", - - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - - "dev": true - - }, - - "fast-glob": { - - "version": "3.2.11", - - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.11.tgz", - - "integrity": "sha512-xrO3+1bxSo3ZVHAnqzyuewYT6aMFHRAd4Kcs92MAonjwQZLsK9d0SF1IyQ3k5PoirxTW0Oe/RqFgMQ6TcNE5Ew==", - - "dev": true, - - "requires": { - - "@nodelib/fs.stat": "^2.0.2", - - "@nodelib/fs.walk": "^1.2.3", - - "glob-parent": "^5.1.2", - - "merge2": "^1.3.0", - - "micromatch": "^4.0.4" - - } - - }, - - "fastq": { - - "version": "1.13.0", - - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", - - "integrity": "sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==", - - "dev": true, - - "requires": { - - "reusify": "^1.0.4" - - } - - }, - - "fill-range": { - - "version": "7.1.1", - - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - - "dev": true, - - "requires": { - - "to-regex-range": "^5.0.1" - - } - - }, - - "fraction.js": { - - "version": "4.2.0", - - "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", - - "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==", - - "dev": true - - }, - - "fs-extra": { - - "version": "10.1.0", - - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", - - "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", - - "dev": true, - - "requires": { - - "graceful-fs": "^4.2.0", - - "jsonfile": "^6.0.1", - - "universalify": "^2.0.0" - - } - - }, - - "fsevents": { - - "version": "2.3.2", - - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - - "dev": true, - - "optional": true - - }, - - "get-caller-file": { - - "version": "2.0.5", - - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - - "dev": true - - }, - - "get-stdin": { - - "version": "9.0.0", - - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-9.0.0.tgz", - - "integrity": "sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA==", - - "dev": true - - }, - - "glob-parent": { - - "version": "5.1.2", - - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - - "dev": true, - - "requires": { - - "is-glob": "^4.0.1" - - } - - }, - - "globby": { - - "version": "13.1.2", - - "resolved": "https://registry.npmjs.org/globby/-/globby-13.1.2.tgz", - - "integrity": "sha512-LKSDZXToac40u8Q1PQtZihbNdTYSNMuWe+K5l+oa6KgDzSvVrHXlJy40hUP522RjAIoNLJYBJi7ow+rbFpIhHQ==", - - "dev": true, - - "requires": { - - "dir-glob": "^3.0.1", - - "fast-glob": "^3.2.11", - - "ignore": "^5.2.0", - - "merge2": "^1.4.1", - - "slash": "^4.0.0" - - } - - }, - - "graceful-fs": { - - "version": "4.2.10", - - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", - - "dev": true - - }, - - "hugo-extended": { - - "version": "0.155.3", - - "resolved": "https://registry.npmjs.org/hugo-extended/-/hugo-extended-0.155.3.tgz", - - "integrity": "sha512-nzGmsgnOdeOGDgtpPHEPZ1PVizDHPU3240UdRmxu0b9vT+A7iB9toaUGcUQXQ/PVURq6y8lIuoTFsI4xfDoLLA==", - - "dev": true, - - "requires": { - - "adm-zip": "^0.5.16", - - "tar": "^7.5.7" - - } - - }, - - "ignore": { - - "version": "5.2.0", - - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", - - "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", - - "dev": true - - }, - - "is-binary-path": { - - "version": "2.1.0", - - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - - "dev": true, - - "requires": { - - "binary-extensions": "^2.0.0" - - } - - }, - - "is-extglob": { - - "version": "2.1.1", - - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - - "dev": true - - }, - - "is-fullwidth-code-point": { - - "version": "3.0.0", - - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - - "dev": true - - }, - - "is-glob": { - - "version": "4.0.3", - - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - - "dev": true, - - "requires": { - - "is-extglob": "^2.1.1" - - } - - }, - - "is-number": { - - "version": "7.0.0", - - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - - "dev": true - - }, - - "jquery": { - - "version": "3.7.1", - - "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.7.1.tgz", - - "integrity": "sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg==" - - }, - - "jsonfile": { - - "version": "6.1.0", - - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", - - "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - - "dev": true, - - "requires": { - - "graceful-fs": "^4.1.6", - - "universalify": "^2.0.0" - - } - - }, - - "lilconfig": { - - "version": "2.0.6", - - "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.0.6.tgz", - - "integrity": "sha512-9JROoBW7pobfsx+Sq2JsASvCo6Pfo6WWoUW79HuB1BCoBXD4PLWJPqDF6fNj67pqBYTbAHkE57M1kS/+L1neOg==", - - "dev": true - - }, - - "merge2": { - - "version": "1.4.1", - - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - - "dev": true - - }, - - "micromatch": { - - "version": "4.0.8", - - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - - "dev": true, - - "requires": { - - "braces": "^3.0.3", - - "picomatch": "^2.3.1" - - } - - }, - - "minipass": { - - "version": "7.1.2", - - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - - "dev": true - - }, - - "minizlib": { - - "version": "3.1.0", - - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", - - "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", - - "dev": true, - - "requires": { - - "minipass": "^7.1.2" - - } - - }, - - "nanoid": { - - "version": "3.3.11", - - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", - - "dev": true - - }, - - "node-releases": { - - "version": "2.0.6", - - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz", - - "integrity": "sha512-PiVXnNuFm5+iYkLBNeq5211hvO38y63T0i2KKh2KnUs3RpzJ+JtODFjkD8yjLwnDkTYF1eKXheUwdssR+NRZdg==", - - "dev": true - - }, - - "normalize-path": { - - "version": "3.0.0", - - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - - "dev": true - - }, - - "normalize-range": { - - "version": "0.1.2", - - "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz", - - "integrity": "sha1-LRDAa9/TEuqXd2laTShDlFa3WUI=", - - "dev": true - - }, - - "path-type": { - - "version": "4.0.0", - - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - - "dev": true - - }, - - "picocolors": { - - "version": "1.1.1", - - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - - "dev": true - - }, - - "picomatch": { - - "version": "2.3.1", - - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - - "dev": true - - }, - - "pify": { - - "version": "2.3.0", - - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - - "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", - - "dev": true - - }, - - "postcss": { - - "version": "8.5.6", - - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", - - "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", - - "dev": true, - - "requires": { - - "nanoid": "^3.3.11", - - "picocolors": "^1.1.1", - - "source-map-js": "^1.2.1" - - } - - }, - - "postcss-cli": { - - "version": "10.0.0", - - "resolved": "https://registry.npmjs.org/postcss-cli/-/postcss-cli-10.0.0.tgz", - - "integrity": "sha512-Wjy/00wBBEgQqnSToznxLWDnATznokFGXsHtF/3G8glRZpz5KYlfHcBW/VMJmWAeF2x49zjgy4izjM3/Wx1dKA==", - - "dev": true, - - "requires": { - - "chokidar": "^3.3.0", - - "dependency-graph": "^0.11.0", - - "fs-extra": "^10.0.0", - - "get-stdin": "^9.0.0", - - "globby": "^13.0.0", - - "picocolors": "^1.0.0", - - "postcss-load-config": "^4.0.0", - - "postcss-reporter": "^7.0.0", - - "pretty-hrtime": "^1.0.3", - - "read-cache": "^1.0.0", - - "slash": "^4.0.0", - - "yargs": "^17.0.0" - - } - - }, - - "postcss-load-config": { - - "version": "4.0.1", - - "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.1.tgz", - - "integrity": "sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA==", - - "dev": true, - - "requires": { - - "lilconfig": "^2.0.5", - - "yaml": "^2.1.1" - - } - - }, - - "postcss-reporter": { - - "version": "7.0.5", - - "resolved": "https://registry.npmjs.org/postcss-reporter/-/postcss-reporter-7.0.5.tgz", - - "integrity": "sha512-glWg7VZBilooZGOFPhN9msJ3FQs19Hie7l5a/eE6WglzYqVeH3ong3ShFcp9kDWJT1g2Y/wd59cocf9XxBtkWA==", - - "dev": true, - - "requires": { - - "picocolors": "^1.0.0", - - "thenby": "^1.3.4" - - } - - }, - - "postcss-value-parser": { - - "version": "4.2.0", - - "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", - - "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", - - "dev": true - - }, - - "pretty-hrtime": { - - "version": "1.0.3", - - "resolved": "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz", - - "integrity": "sha512-66hKPCr+72mlfiSjlEB1+45IjXSqvVAIy6mocupoww4tBFE9R9IhwwUGoI4G++Tc9Aq+2rxOt0RFU6gPcrte0A==", - - "dev": true - - }, - - "queue-microtask": { - - "version": "1.2.3", - - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - - "dev": true - - }, - - "read-cache": { - - "version": "1.0.0", - - "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", - - "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", - - "dev": true, - - "requires": { - - "pify": "^2.3.0" - - } - - }, - - "readdirp": { - - "version": "3.6.0", - - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - - "dev": true, - - "requires": { - - "picomatch": "^2.2.1" - - } - - }, - - "require-directory": { - - "version": "2.1.1", - - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - - "dev": true - - }, - - "reusify": { - - "version": "1.0.4", - - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - - "dev": true - - }, - - "run-parallel": { - - "version": "1.2.0", - - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - - "dev": true, - - "requires": { - - "queue-microtask": "^1.2.2" - - } - - }, - - "slash": { - - "version": "4.0.0", - - "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", - - "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", - - "dev": true - - }, - - "source-map-js": { - - "version": "1.2.1", - - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - - "dev": true - - }, - - "string-width": { - - "version": "4.2.3", - - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - - "dev": true, - - "requires": { - - "emoji-regex": "^8.0.0", - - "is-fullwidth-code-point": "^3.0.0", - - "strip-ansi": "^6.0.1" - - } - - }, - - "strip-ansi": { - - "version": "6.0.1", - - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - - "dev": true, - - "requires": { + "dependencies": { "ansi-regex": "^5.0.1" - } - - }, - - "tabpanel": { - - "version": "0.2.0", - - "resolved": "https://registry.npmjs.org/tabpanel/-/tabpanel-0.2.0.tgz", - - "integrity": "sha512-tS6UG1L/QfZZ0GdqMKkPXSYtErQNE0qDxNpo0xJh0XZT9TYjZ0lLHwIJUhhyV+u01h1iSswrw6vCNYqwD/R+EQ==" - - }, - - "tar": { - - "version": "7.5.7", - - "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.7.tgz", - - "integrity": "sha512-fov56fJiRuThVFXD6o6/Q354S7pnWMJIVlDBYijsTNx6jKSE4pvrDTs6lUnmGvNyfJwFQQwWy3owKz1ucIhveQ==", - - "dev": true, - - "requires": { - - "@isaacs/fs-minipass": "^4.0.0", - - "chownr": "^3.0.0", - - "minipass": "^7.1.2", - - "minizlib": "^3.1.0", - - "yallist": "^5.0.0" - - } - - }, - - "thenby": { - - "version": "1.3.4", - - "resolved": "https://registry.npmjs.org/thenby/-/thenby-1.3.4.tgz", - - "integrity": "sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ==", - - "dev": true - - }, - - "to-regex-range": { - - "version": "5.0.1", - - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - - "dev": true, - - "requires": { - - "is-number": "^7.0.0" - - } - - }, - - "universalify": { - - "version": "2.0.0", - - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", - - "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", - - "dev": true - - }, - - "update-browserslist-db": { - - "version": "1.0.5", - - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.5.tgz", - - "integrity": "sha512-dteFFpCyvuDdr9S/ff1ISkKt/9YZxKjI9WlRR99c180GaztJtRa/fn18FdxGVKVsnPY7/a/FDN68mcvUmP4U7Q==", - - "dev": true, - - "requires": { - - "escalade": "^3.1.1", - - "picocolors": "^1.0.0" - - } - - }, - - "wrap-ansi": { - - "version": "7.0.0", - - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - - "dev": true, - - "requires": { - - "ansi-styles": "^4.0.0", - - "string-width": "^4.1.0", - - "strip-ansi": "^6.0.0" - - } - - }, - - "y18n": { - - "version": "5.0.8", - - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - - "dev": true - - }, - - "yallist": { - - "version": "5.0.0", - - "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", - - "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", - - "dev": true - - }, - - "yaml": { - - "version": "2.8.2", - - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", - - "dev": true - - }, - - "yargs": { - - "version": "17.5.1", - - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.5.1.tgz", - - "integrity": "sha512-t6YAJcxDkNX7NFYiVtKvWUz8l+PaKTLiL63mJYWR2GnHq2gjEWISzsLp9wg3aY36dY1j+gfIEL3pIF+XlJJfbA==", - - "dev": true, - - "requires": { - - "cliui": "^7.0.2", - - "escalade": "^3.1.1", - - "get-caller-file": "^2.0.5", - - "require-directory": "^2.1.1", - - "string-width": "^4.2.3", + }, - "y18n": "^5.0.5", + "engines": { - "yargs-parser": "^21.0.0" + "node": ">=8" } - }, - - "yargs-parser": { - - "version": "21.1.1", - - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - - "dev": true - } } diff --git a/package.json b/package.json index 7acc342b..cd73ea02 100644 --- a/package.json +++ b/package.json @@ -14,8 +14,12 @@ "dependencies": { + "dompurify": "^3.3.1", + "jquery": "^3.7.1", + "marked": "^15.0.12", + "tabpanel": "^0.2.0" } diff --git a/scripts/github-index.js b/scripts/github-index.js new file mode 100644 index 00000000..91bec410 --- /dev/null +++ b/scripts/github-index.js @@ -0,0 +1,672 @@ +const fs = require('fs'); +const path = require('path'); + +const GITHUB_TOKEN = process.env.GITHUB_TOKEN || process.env.GITHUB_PAT || process.env.GH_TOKEN || ''; +const ORG = 'Interlisp'; +const OUTPUT_FILE = path.join(__dirname, '../github-index.jsonl'); +const DEBUG = process.env.DEBUG_GITHUB_INDEX === '1'; +const START_TIME = Date.now(); + +if (!GITHUB_TOKEN) { + throw new Error('Missing GitHub token. Set GITHUB_TOKEN, GITHUB_PAT, or GH_TOKEN in the environment.'); +} + +const headers = { + 'Authorization': `Bearer ${GITHUB_TOKEN}`, + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28' +}; + +const FETCH_TIMEOUT_MS = 30000; +const FETCH_RETRIES = 2; +const MAX_PAGES_PER_PAGINATION = Number.parseInt(process.env.GITHUB_INDEX_MAX_PAGES || '', 10); +const MAX_RATE_LIMIT_RETRIES = Number.parseInt(process.env.GITHUB_RATE_LIMIT_RETRIES || '6', 10); +const SECONDARY_LIMIT_BASE_WAIT_MS = Number.parseInt(process.env.GITHUB_SECONDARY_LIMIT_BASE_WAIT_MS || '60000', 10); +const SECONDARY_LIMIT_MAX_WAIT_MS = Number.parseInt(process.env.GITHUB_SECONDARY_LIMIT_MAX_WAIT_MS || '900000', 10); + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function fetchWithTimeout(url, options = {}, timeoutMs = FETCH_TIMEOUT_MS) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + + try { + return await fetch(url, { ...options, signal: controller.signal }); + } finally { + clearTimeout(timer); + } +} + +async function githubRequest(url, options = {}, label = 'request') { + let lastError; + + for (let attempt = 0; attempt <= MAX_RATE_LIMIT_RETRIES; attempt += 1) { + try { + const response = await fetchWithTimeout(url, options); + if (response.ok) { + return response; + } + + const status = response.status; + const retryAfter = response.headers.get('retry-after'); + const remainingRaw = response.headers.get('x-ratelimit-remaining'); + const resetRaw = response.headers.get('x-ratelimit-reset'); + const remaining = Number.parseInt(remainingRaw || '', 10); + const resetEpoch = Number.parseInt(resetRaw || '', 10); + + let waitMs = null; + let reason = ''; + + if (retryAfter) { + const retryAfterSec = Number.parseInt(retryAfter, 10); + if (Number.isFinite(retryAfterSec) && retryAfterSec > 0) { + waitMs = retryAfterSec * 1000; + reason = `retry-after=${retryAfterSec}s`; + } + } + + if (waitMs === null && remaining === 0 && Number.isFinite(resetEpoch)) { + waitMs = Math.max((resetEpoch * 1000) - Date.now(), 1000); + reason = `x-ratelimit-reset=${resetEpoch}`; + } + + if (waitMs === null && (status === 403 || status === 429)) { + const bodyText = await response.clone().text(); + const mentionsRateLimit = /rate limit|secondary rate limit|abuse/i.test(bodyText); + if (mentionsRateLimit) { + // Secondary rate limiting guidance: start at 1 minute, then exponential backoff. + waitMs = Math.min( + SECONDARY_LIMIT_BASE_WAIT_MS * (2 ** attempt), + SECONDARY_LIMIT_MAX_WAIT_MS + ); + reason = 'secondary-rate-limit-backoff'; + } + } + + if (waitMs !== null) { + if (attempt >= MAX_RATE_LIMIT_RETRIES) { + const exhaustedError = new Error( + `Rate limit retries exhausted for ${label} (status ${status})` + ); + exhaustedError.status = status; + exhaustedError.url = url; + throw exhaustedError; + } + + const jitterMs = Math.floor(Math.random() * 1000); + const totalWaitMs = waitMs + jitterMs; + console.warn( + `Rate limited for ${label} (status ${status}, ${reason}). ` + + `Retrying in ${(totalWaitMs / 1000).toFixed(1)}s ` + + `(attempt ${attempt + 1}/${MAX_RATE_LIMIT_RETRIES})` + ); + await sleep(totalWaitMs); + continue; + } + + return response; + } catch (error) { + lastError = error; + if (attempt >= MAX_RATE_LIMIT_RETRIES) break; + + const backoffMs = Math.min(1000 * (2 ** attempt), 10000); + if (DEBUG) { + console.warn( + `Retrying ${label} (${attempt + 1}/${MAX_RATE_LIMIT_RETRIES}) ` + + `after error: ${error.message}` + ); + } + await sleep(backoffMs); + } + } + + throw lastError; +} + +function toDateTime(value, fallback) { + if (!value) return fallback; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? fallback : parsed.toISOString(); +} + +function logProgress(message) { + const elapsedSeconds = ((Date.now() - START_TIME) / 1000).toFixed(1); + console.log(`[+${elapsedSeconds}s] ${message}`); +} + +function sanitizeId(id) { + // Vertex AI requires IDs to match [a-zA-Z0-9-_]* + // Replace dots and slashes with hyphens + return id.replace(/[^a-zA-Z0-9-_]/g, '-'); +} + +function makeDoc(id, data) { + const sanitized = sanitizeId(id); + return { + id: sanitized, + structData: { + id: sanitized, + ...data + } + }; +} + +const ID_PATTERN = /^[a-zA-Z0-9-_]*$/; + +function isIsoDateTime(value) { + return typeof value === 'string' && !Number.isNaN(Date.parse(value)); +} + +function normalizeLabel(label) { + if (typeof label === 'string') return label; + if (label && typeof label.name === 'string') return label.name; + return ''; +} + +function extractReferences(content, repo, defaultBranch) { + const text = String(content || ''); + const refs = []; + const seen = new Set(); + + function addRef(rawText, targetUrl) { + if (!rawText || !targetUrl) return; + const key = `${rawText}::${targetUrl}`; + if (seen.has(key)) return; + seen.add(key); + refs.push({ raw_text: rawText, target_url: targetUrl }); + } + + const discussionPattern = /\bdiscussion\s+#(\d+)\b/gi; + for (const match of text.matchAll(discussionPattern)) { + addRef(`#${match[1]}`, `https://github.com/${ORG}/${repo}/discussions/${match[1]}`); + } + + const issuePattern = /\bissues?\s+#(\d+)\b/gi; + for (const match of text.matchAll(issuePattern)) { + addRef(`#${match[1]}`, `https://github.com/${ORG}/${repo}/issues/${match[1]}`); + } + + const prPattern = /\b(?:pr|pull\s*request)\s+#(\d+)\b/gi; + for (const match of text.matchAll(prPattern)) { + addRef(`#${match[1]}`, `https://github.com/${ORG}/${repo}/pull/${match[1]}`); + } + + const blobPathPattern = /\b([\w.-]+\.(?:md|markdown|mdx))\b/gi; + for (const match of text.matchAll(blobPathPattern)) { + addRef(match[1], `https://github.com/${ORG}/${repo}/blob/${defaultBranch}/${match[1]}`); + } + + return refs; +} + +function validateDoc(doc) { + const errors = []; + const data = doc?.structData; + + if (typeof doc.id !== 'string' || doc.id.length === 0 || !ID_PATTERN.test(doc.id)) { + errors.push('invalid id'); + } + + if (!data || typeof data !== 'object' || Array.isArray(data)) { + errors.push('missing structData'); + return { valid: false, errors }; + } + + if (typeof data.id !== 'string' || data.id !== doc.id) { + errors.push('invalid structData.id'); + } + if (typeof data.title !== 'string') errors.push('invalid title'); + if (typeof data.content !== 'string') errors.push('invalid content'); + if (typeof data.url !== 'string') errors.push('invalid url'); + if (typeof data.type !== 'string') errors.push('invalid type'); + if (typeof data.repo !== 'string') errors.push('invalid repo'); + if (typeof data.state !== 'string') errors.push('invalid state'); + if (typeof data.author !== 'string') errors.push('invalid author'); + if (!Array.isArray(data.labels) || data.labels.some(label => typeof label !== 'string')) { + errors.push('invalid labels'); + } + if (!isIsoDateTime(data.created_at)) errors.push('invalid created_at'); + if (!isIsoDateTime(data.updated_at)) errors.push('invalid updated_at'); + if (!Array.isArray(data.references)) { + errors.push('invalid references'); + } else { + for (const ref of data.references) { + if (!ref || typeof ref !== 'object') { + errors.push('invalid reference object'); + break; + } + if (typeof ref.raw_text !== 'string') errors.push('invalid reference raw_text'); + if (typeof ref.target_url !== 'string') errors.push('invalid reference target_url'); + } + } + if (data.number !== undefined && !Number.isInteger(data.number)) { + errors.push('invalid number'); + } + + return { + valid: errors.length === 0, + errors + }; +} + +// Repos to index — add/remove as needed +const REPOS = [ + 'medley', + 'Interlisp.github.io', + 'online', + 'maiko', + 'loops', + 'notecards', + // add others here +]; + +async function githubFetch(url) { + const resp = await githubRequest(url, { headers }, url); + if (!resp.ok) { + const body = await resp.text(); + const error = new Error(`GitHub API error ${resp.status}: ${url} :: ${body.slice(0, 300)}`); + error.status = resp.status; + error.url = url; + throw error; + } + return resp.json(); +} + +function isNotFoundError(error) { + return error && Number(error.status) === 404; +} + +async function getDefaultBranch(repo) { + const meta = await githubFetch(`https://api.github.com/repos/${ORG}/${repo}`); + return { + defaultBranch: meta.default_branch || 'main', + repoUpdatedAt: toDateTime(meta.updated_at, new Date().toISOString()) + }; +} + +async function indexMarkdownFiles(repo, defaultBranch, repoUpdatedAt) { + const docs = []; + const treeResp = await githubFetch( + `https://api.github.com/repos/${ORG}/${repo}/git/trees/${defaultBranch}?recursive=1` + ); + const tree = treeResp.tree || []; + const markdownFiles = tree.filter(entry => + entry.type === 'blob' && + /\.(md|markdown|mdx)$/i.test(entry.path) && + !/^README(\.[^/]+)?$/i.test(path.basename(entry.path)) + ); + + for (const file of markdownFiles) { + try { + const blob = await githubFetch( + `https://api.github.com/repos/${ORG}/${repo}/git/blobs/${file.sha}` + ); + const content = Buffer.from(blob.content || '', 'base64').toString('utf-8'); + const trimmed = content.slice(0, 10000); + const url = `https://github.com/${ORG}/${repo}/blob/${defaultBranch}/${file.path}`; + docs.push(makeDoc( + `gh-markdown-${repo}-${file.path.replace(/\//g, '-')}`, + { + title: `[${repo}] ${file.path}`, + content: trimmed, + url, + references: extractReferences(trimmed, repo, defaultBranch), + type: 'markdown', + labels: [], + repo, + state: 'active', + author: '', + created_at: repoUpdatedAt, + updated_at: repoUpdatedAt + } + )); + } catch (e) { + if (DEBUG) { + console.warn(`Could not index markdown file ${repo}:${file.path}: ${e.message}`); + } + } + } + + return docs; +} + +async function* paginate(url, label = url) { + let nextUrl = url; + let page = 0; + while (nextUrl) { + page += 1; + if (Number.isInteger(MAX_PAGES_PER_PAGINATION) && MAX_PAGES_PER_PAGINATION > 0 && page > MAX_PAGES_PER_PAGINATION) { + console.warn(`Pagination capped for ${label} at ${MAX_PAGES_PER_PAGINATION} pages`); + break; + } + if (DEBUG) { + logProgress(`Fetching ${label} page ${page}`); + } + let resp; + try { + resp = await githubRequest(nextUrl, { headers }, `${label} page ${page}`); + } catch (error) { + console.warn(`Pagination stopped for ${label} at page ${page}: ${error.message}`); + break; + } + if (!resp.ok) { + const body = await resp.text(); + console.warn(`Pagination stopped for ${label} at page ${page}: HTTP ${resp.status} ${body.slice(0, 200)}`); + break; + } + const data = await resp.json(); + yield* data; + + // Parse Link header for next page + const link = resp.headers.get('Link') || ''; + const match = link.match(/<([^>]+)>;\s*rel="next"/); + nextUrl = match ? match[1] : null; + } +} + +function truncateByChars(parts, maxChars) { + const output = []; + let used = 0; + + for (const part of parts) { + if (!part) continue; + const chunk = String(part).trim(); + if (!chunk) continue; + if (used >= maxChars) break; + + const remaining = maxChars - used; + if (chunk.length <= remaining) { + output.push(chunk); + used += chunk.length; + continue; + } + + output.push(chunk.slice(0, remaining)); + used = maxChars; + break; + } + + return output.join('\n\n'); +} + +function promiseWithTimeout(promise, timeoutMs, message) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(message)), timeoutMs); + promise + .then(value => { + clearTimeout(timer); + resolve(value); + }) + .catch(error => { + clearTimeout(timer); + reject(error); + }); + }); +} + +async function fetchPullRequestSearchSignals(repo, prNumber) { + const changedFiles = []; + const commitMessages = []; + + for await (const file of paginate( + `https://api.github.com/repos/${ORG}/${repo}/pulls/${prNumber}/files?per_page=100` + )) { + if (typeof file.filename === 'string' && file.filename.length > 0) { + changedFiles.push(file.filename); + } + } + + for await (const commit of paginate( + `https://api.github.com/repos/${ORG}/${repo}/pulls/${prNumber}/commits?per_page=100` + )) { + const msg = commit.commit?.message; + if (typeof msg === 'string' && msg.length > 0) { + commitMessages.push(msg); + } + } + + return { + changedFiles, + commitMessages + }; +} + +async function indexRepo(repo) { + const docs = []; + logProgress(`Indexing ${ORG}/${repo}...`); + const { defaultBranch, repoUpdatedAt } = await getDefaultBranch(repo); + + logProgress(`${ORG}/${repo}: default branch is ${defaultBranch}`); + + // Index README + try { + logProgress(`${ORG}/${repo}: fetching README`); + const readme = await githubFetch( + `https://api.github.com/repos/${ORG}/${repo}/readme` + ); + const content = Buffer.from(readme.content, 'base64').toString('utf-8'); + const trimmed = content.slice(0, 10000); + const url = `https://github.com/${ORG}/${repo}/blob/${defaultBranch}/${readme.path || 'README.md'}`; + docs.push(makeDoc( + `gh-markdown-${repo}-readme`, + { + title: `${repo} README`, + content: trimmed, + url, + references: extractReferences(trimmed, repo, defaultBranch), + type: 'markdown', + labels: [], + repo, + state: 'active', + author: '', + created_at: repoUpdatedAt, + updated_at: repoUpdatedAt + } + )); + } catch (e) { + if (isNotFoundError(e)) { + console.warn(`No README for ${repo}`); + } else { + console.warn(`README lookup failed for ${repo}: ${e.message}`); + } + } + + // Index all markdown files in the repository (excluding README files). + try { + logProgress(`${ORG}/${repo}: indexing markdown files`); + const markdownDocs = await indexMarkdownFiles(repo, defaultBranch, repoUpdatedAt); + docs.push(...markdownDocs); + logProgress(`${ORG}/${repo}: indexed ${markdownDocs.length} markdown files`); + } catch (e) { + console.warn(`Could not index markdown files for ${repo}: ${e.message}`); + } + + // Index Issues + let issueCount = 0; + for await (const issue of paginate( + `https://api.github.com/repos/${ORG}/${repo}/issues?state=all&per_page=100`, + `${ORG}/${repo} issues` + )) { + if (issue.pull_request) continue; // skip PRs here, handle separately + issueCount += 1; + const content = (issue.body || '').slice(0, 10000); + docs.push(makeDoc( + `gh-issue-${repo}-${issue.number}`, + { + title: issue.title, + content, + url: issue.html_url, + references: extractReferences(content, repo, defaultBranch), + type: 'issue', + labels: (issue.labels || []).map(normalizeLabel).filter(Boolean), + repo, + state: issue.state, + author: issue.user?.login || '', + created_at: toDateTime(issue.created_at, repoUpdatedAt), + updated_at: toDateTime(issue.updated_at, repoUpdatedAt), + number: issue.number + } + )); + + if (issueCount === 1 || issueCount % 25 === 0) { + logProgress(`${ORG}/${repo}: indexed ${issueCount} issues`); + } + } + + logProgress(`${ORG}/${repo}: finished issues (${issueCount})`); + + // Index Pull Requests + let prCount = 0; + for await (const pr of paginate( + `https://api.github.com/repos/${ORG}/${repo}/pulls?state=all&per_page=100`, + `${ORG}/${repo} pull requests` + )) { + prCount += 1; + if (prCount === 1 || prCount % 10 === 0) { + logProgress(`${ORG}/${repo}: processing PR ${prCount} (#${pr.number})`); + } + + let prSignals = { changedFiles: [], commitMessages: [] }; + + try { + prSignals = await promiseWithTimeout( + fetchPullRequestSearchSignals(repo, pr.number), + 20000, + `Timed out fetching PR details for ${repo}#${pr.number}` + ); + } catch (e) { + if (DEBUG) { + console.warn(`Could not fetch PR details ${repo}#${pr.number}: ${e.message}`); + } + } + + const content = truncateByChars([ + pr.body || '', + prSignals.changedFiles.length > 0 + ? `Changed files:\n${prSignals.changedFiles.join('\n')}` + : '', + prSignals.commitMessages.length > 0 + ? `Commit messages:\n${prSignals.commitMessages.join('\n---\n')}` + : '' + ], 10000); + + docs.push(makeDoc( + `gh-pr-${repo}-${pr.number}`, + { + title: pr.title, + content, + url: pr.html_url, + references: extractReferences(content, repo, defaultBranch), + type: 'pull_request', + labels: (pr.labels || []).map(normalizeLabel).filter(Boolean), + repo, + state: pr.state, + author: pr.user?.login || '', + created_at: toDateTime(pr.created_at, repoUpdatedAt), + updated_at: toDateTime(pr.updated_at, repoUpdatedAt), + number: pr.number + } + )); + } + + logProgress(`${ORG}/${repo}: finished pull requests (${prCount})`); + + // Index Discussions (requires GraphQL API) + try { + logProgress(`${ORG}/${repo}: fetching discussions`); + const discussionsResp = await githubRequest('https://api.github.com/graphql', { + method: 'POST', + headers: { ...headers, 'Content-Type': 'application/json' }, + body: JSON.stringify({ query: ` + query { + repository(owner: "${ORG}", name: "${repo}") { + discussions(first: 100) { + nodes { + number + title + body + url + createdAt + updatedAt + author { login } + category { name } + } + } + } + } + `}) + }); + + const discussionsData = await discussionsResp.json(); + const discussions = discussionsData.data?.repository?.discussions?.nodes || []; + logProgress(`${ORG}/${repo}: found ${discussions.length} discussions`); + + discussions.forEach(d => { + const content = (d.body || '').slice(0, 10000); + docs.push(makeDoc( + `gh-discussion-${repo}-${d.number}`, + { + title: d.title, + content, + url: d.url, + references: extractReferences(content, repo, defaultBranch), + type: 'discussion', + labels: d.category?.name ? [d.category.name] : [], + repo, + state: 'open', + author: d.author?.login || '', + created_at: toDateTime(d.createdAt, repoUpdatedAt), + updated_at: toDateTime(d.updatedAt, repoUpdatedAt), + number: d.number + } + )); + }); + } catch (e) { + console.warn(`Could not fetch discussions for ${repo}:`, e.message); + } + + return docs; +} + +async function main() { + const allDocs = []; + + for (const repo of REPOS) { + try { + logProgress(`Starting repo ${ORG}/${repo}`); + const docs = await indexRepo(repo); + allDocs.push(...docs); + logProgress(`${ORG}/${repo}: produced ${docs.length} documents`); + } catch (e) { + console.error(`Failed to index ${repo}:`, e.message); + } + } + + const validDocs = []; + let invalidCount = 0; + + for (const doc of allDocs) { + const result = validateDoc(doc); + if (result.valid) { + validDocs.push(doc); + continue; + } + + invalidCount += 1; + console.warn( + `Skipping invalid document ${doc.id || ''}: ${result.errors.join(', ')}` + ); + } + + const jsonl = validDocs.map(d => JSON.stringify(d)).join('\n'); + fs.writeFileSync(OUTPUT_FILE, jsonl); + logProgress(`Wrote ${validDocs.length} valid documents to ${OUTPUT_FILE}`); + if (invalidCount > 0) { + console.warn(`Skipped ${invalidCount} invalid documents during pre-write validation`); + } +} + +//main().catch(console.error); +main(); diff --git a/search-function/.gcloudignore b/search-function/.gcloudignore new file mode 100644 index 00000000..1c2b3137 --- /dev/null +++ b/search-function/.gcloudignore @@ -0,0 +1,3 @@ +node_modules/ +.git/ +*.md \ No newline at end of file diff --git a/search-function/deploy-search-function.sh b/search-function/deploy-search-function.sh new file mode 100755 index 00000000..3c7eba23 --- /dev/null +++ b/search-function/deploy-search-function.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# File: deploy-search-function.sh + +PROJECT_ID="interlispsearch" +REGION="us-central1" +WEBSITE_ENGINE_ID="interlisp-website-only" # [interlisp-web-sites_1741606671710] — proven www history/medley (SUCCEEDED) +GITHUB_ENGINE_ID="interlisp-github-only" # [interlisp-github-v2] — issues/PRs/discussions +# Fallback unified engine retained for single-engine callers but not used by dual-fetch: +FALLBACK_ENGINE_ID="interlisp-search-unified" # [interlisp-web-sites_1741606671710, interlisp-github-v2] +# Verify engines exist (REST, no gcloud discoveryengine): +# TOKEN=$(gcloud auth application-default print-access-token); curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines" | python3 -m json.tool | grep -E '"name"|interlisp-website-only|interlisp-github-only' + +echo "Deploying search function to $REGION..." +echo "WEBSITE_ENGINE_ID=$WEBSITE_ENGINE_ID GITHUB_ENGINE_ID=$GITHUB_ENGINE_ID (fallback $FALLBACK_ENGINE_ID)" +echo "" + +gcloud functions deploy search \ + --gen2 \ + --runtime=nodejs24 \ + --region=$REGION \ + --source=. \ + --entry-point=search \ + --trigger-http \ + --allow-unauthenticated \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --set-env-vars PROJECT_ID=$PROJECT_ID,WEBSITE_ENGINE_ID=$WEBSITE_ENGINE_ID,GITHUB_ENGINE_ID=$GITHUB_ENGINE_ID,ENGINE_ID=$WEBSITE_ENGINE_ID \ + --memory=256Mi \ + --timeout=30s \ + --project=$PROJECT_ID + +echo "" +echo "Deployed: https://$REGION-$PROJECT_ID.cloudfunctions.net/search" +echo "Env: WEBSITE_ENGINE_ID=$WEBSITE_ENGINE_ID GITHUB_ENGINE_ID=$GITHUB_ENGINE_ID ENGINE_ID=$WEBSITE_ENGINE_ID (fallback $FALLBACK_ENGINE_ID kept as engine)" diff --git a/search-function/github-reimport.js b/search-function/github-reimport.js new file mode 100644 index 00000000..35b0b8da --- /dev/null +++ b/search-function/github-reimport.js @@ -0,0 +1,278 @@ +'use strict'; + +const { GoogleAuth } = require('google-auth-library'); +const { Storage } = require('@google-cloud/storage'); + +const PROJECT_ID = 'interlispsearch'; +const LOCATION = 'global'; +// Structured GitHub store — NOT the website store. Keeps website crawl separate +// and is consumed via interlisp-github-only engine (blended 70/30 in search function). +// Fallback engine interlisp-search-unified ([interlisp-web-sites_1741606671710, interlisp-github-v2]) retained. +const DATA_STORE_ID = 'interlisp-github-v2'; +const GITHUB_TOKEN = process.env.GITHUB_TOKEN; +const GITHUB_ORG = 'Interlisp'; +const GITHUB_REPOS = ['medley', 'maiko', 'Interlisp.github.io']; + +const auth = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'] +}); + +const storage = new Storage({ projectId: PROJECT_ID }); + +exports.reimportGithub = async (message, context) => { + console.log(`[${new Date().toISOString()}] Starting GitHub reimport...`); + + try { + // 1. Fetch GitHub content + const gitHubDocuments = []; + + console.log('Fetching GitHub Issues...'); + const issues = await fetchGitHubIssues(); + gitHubDocuments.push(...issues); + console.log(`Fetched ${issues.length} GitHub issues`); + + console.log('Fetching GitHub Pull Requests...'); + const prs = await fetchGitHubPullRequests(); + gitHubDocuments.push(...prs); + console.log(`Fetched ${prs.length} GitHub pull requests`); + + console.log('Fetching GitHub Markdown files...'); + const markdownFiles = await fetchGitHubMarkdown(); + gitHubDocuments.push(...markdownFiles); + console.log(`Fetched ${markdownFiles.length} GitHub markdown files`); + + if (gitHubDocuments.length === 0) { + console.warn('No GitHub documents fetched. Check GitHub token and repo access.'); + return { status: 'NO_DOCUMENTS', count: 0 }; + } + + // 2. Convert to JSONL format — interlisp-github-v2 is NO_CONTENT: each line is + // {id, structData:{...}} with sanitized id ^[a-zA-Z0-9-_]+$ on branches/0 (see guide 1.6). + const sanitizeId = (id) => id.replace(/\./g, '-').replace(/[^a-zA-Z0-9-_]/g, '-'); + console.log(`Converting ${gitHubDocuments.length} documents to JSONL (structData, branches/0, sanitized ids)...`); + const jsonlLines = gitHubDocuments.map(doc => { + const rawId = doc.id; + const sid = sanitizeId(rawId); + // Flatten metadata into structData as expected by the structured store + const m = doc.metadata || {}; + const structData = { + id: sid, + title: doc.title || '', + url: doc.url || '', + content: doc.content || '', + source: m.source || 'github', + priority: m.priority || 3, + type: m.content_type || m.type || null, + content_type: m.content_type || null, + repo: m.repo || null, + state: m.state || null, + number: m.issue_number || m.pr_number || m.number || null, + issue_number: m.issue_number || null, + pr_number: m.pr_number || null, + file: m.file || null, + created_at: m.created_date || null, + updated_at: m.updated_date || null, + created_date: m.created_date || null, + updated_date: m.updated_date || null, + last_indexed: m.last_indexed || new Date().toISOString() + }; + // Remove nulls to keep JSONL clean + Object.keys(structData).forEach(k => structData[k] == null && delete structData[k]); + return JSON.stringify({ id: sid, structData }); + }); + const jsonlContent = jsonlLines.join('\n'); + + // 3. Upload JSONL to Cloud Storage + const bucketName = `${PROJECT_ID}-search-imports`; + const fileName = `github-import-${Date.now()}.jsonl`; + + console.log(`Creating Cloud Storage bucket if needed: ${bucketName}`); + const bucket = storage.bucket(bucketName); + + try { + await bucket.exists(); + } catch (e) { + console.log(`Creating bucket: ${bucketName}`); + await storage.createBucket(bucketName, { location: 'US' }); + } + + console.log(`Uploading JSONL to gs://${bucketName}/${fileName}`); + const file = bucket.file(fileName); + await file.save(jsonlContent); + + // 4. Call Vertex AI importDocuments API + const client = await auth.getClient(); + const token = await client.getAccessToken(); + + const importUrl = `https://discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/collections/default_collection/dataStores/${DATA_STORE_ID}/branches/0/documents:import`; + + console.log(`Calling Vertex AI import API: ${importUrl}`); + + const importResponse = await fetch(importUrl, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token.token}`, + 'Content-Type': 'application/json', + 'x-goog-user-project': PROJECT_ID + }, + body: JSON.stringify({ + gcsSource: { + inputUris: [`gs://${bucketName}/${fileName}`] + } + }) + }); + + if (!importResponse.ok) { + const errorText = await importResponse.text(); + throw new Error(`Import failed: ${importResponse.statusText} - ${errorText}`); + } + + const importData = await importResponse.json(); + console.log(`Import operation started: ${importData.name}`); + console.log(`[${new Date().toISOString()}] GitHub reimport completed successfully`); + + return { + status: 'SUCCESS', + documentsImported: gitHubDocuments.length, + operation: importData.name, + gcsLocation: `gs://${bucketName}/${fileName}` + }; + + } catch (err) { + console.error(`[${new Date().toISOString()}] GitHub reimport error:`, err); + throw err; + } +}; + +async function fetchGitHubIssues() { + const issues = []; + + for (const repo of GITHUB_REPOS) { + try { + const url = `https://api.github.com/repos/${GITHUB_ORG}/${repo}/issues?state=all&per_page=100`; + const response = await fetch(url, { + headers: { + 'Authorization': `token ${GITHUB_TOKEN}`, + 'Accept': 'application/vnd.github.v3+json' + } + }); + + if (!response.ok) { + console.warn(`Failed to fetch issues from ${GITHUB_ORG}/${repo}: ${response.statusText}`); + continue; + } + + const repoIssues = await response.json(); + + for (const issue of repoIssues) { + issues.push({ + id: `github-issue-${repo}-${issue.number}`, + title: issue.title, + url: issue.html_url, + content: issue.body || '', + metadata: { + source: 'github', + priority: 3, + content_type: 'issue', + repo: `${GITHUB_ORG}/${repo}`, + issue_number: issue.number, + state: issue.state, + created_date: issue.created_at, + updated_date: issue.updated_at, + last_indexed: new Date().toISOString() + } + }); + } + } catch (err) { + console.error(`Error fetching issues from ${GITHUB_ORG}/${repo}:`, err.message); + } + } + + return issues; +} + +async function fetchGitHubPullRequests() { + const prs = []; + + for (const repo of GITHUB_REPOS) { + try { + const url = `https://api.github.com/repos/${GITHUB_ORG}/${repo}/pulls?state=all&per_page=100`; + const response = await fetch(url, { + headers: { + 'Authorization': `token ${GITHUB_TOKEN}`, + 'Accept': 'application/vnd.github.v3+json' + } + }); + + if (!response.ok) { + console.warn(`Failed to fetch PRs from ${GITHUB_ORG}/${repo}: ${response.statusText}`); + continue; + } + + const repoPrs = await response.json(); + + for (const pr of repoPrs) { + prs.push({ + id: `github-pr-${repo}-${pr.number}`, + title: pr.title, + url: pr.html_url, + content: (pr.body || '') + '\n' + (pr.title || ''), + metadata: { + source: 'github', + priority: 3, + content_type: 'pull_request', + repo: `${GITHUB_ORG}/${repo}`, + pr_number: pr.number, + state: pr.state, + created_date: pr.created_at, + updated_date: pr.updated_at, + last_indexed: new Date().toISOString() + } + }); + } + } catch (err) { + console.error(`Error fetching PRs from ${GITHUB_ORG}/${repo}:`, err.message); + } + } + + return prs; +} + +async function fetchGitHubMarkdown() { + const markdownFiles = []; + + for (const repo of GITHUB_REPOS) { + try { + // Fetch README + const readmeUrl = `https://api.github.com/repos/${GITHUB_ORG}/${repo}/readme`; + const readmeResponse = await fetch(readmeUrl, { + headers: { + 'Authorization': `token ${GITHUB_TOKEN}`, + 'Accept': 'application/vnd.github.v3.raw' + } + }); + + if (readmeResponse.ok) { + const content = await readmeResponse.text(); + markdownFiles.push({ + id: `github-markdown-${repo}-README`, + title: `${repo} README`, + url: `https://github.com/${GITHUB_ORG}/${repo}/blob/main/README.md`, + content: content, + metadata: { + source: 'github', + priority: 3, + content_type: 'markdown', + repo: `${GITHUB_ORG}/${repo}`, + file: 'README.md', + last_indexed: new Date().toISOString() + } + }); + } + } catch (err) { + console.warn(`No README found in ${GITHUB_ORG}/${repo}`); + } + } + + return markdownFiles; +} diff --git a/search-function/index.js b/search-function/index.js new file mode 100644 index 00000000..76e20b3e --- /dev/null +++ b/search-function/index.js @@ -0,0 +1,267 @@ +'use strict'; + +const { GoogleAuth } = require('google-auth-library'); +const { isRateLimited } = require('./rateLimiter'); + +const PROJECT_ID = process.env.PROJECT_ID; +const WEBSITE_ENGINE_ID = process.env.WEBSITE_ENGINE_ID || process.env.ENGINE_ID; +const GITHUB_ENGINE_ID = process.env.GITHUB_ENGINE_ID || null; +const LOCATION = 'global'; + +const auth = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'] +}); + +exports.search = async (req, res) => { + + const allowedOrigins = [ + 'https://interlisp.org', + 'https://www.interlisp.org', + ]; + + const origin = req.headers.origin || ''; + const allowedOrigin = allowedOrigins.includes(origin) + ? origin + : 'https://interlisp.org'; + + res.set('Access-Control-Allow-Origin', allowedOrigin); + res.set('Access-Control-Allow-Methods', 'GET, OPTIONS'); + res.set('Access-Control-Allow-Headers', 'Content-Type'); + res.set('Access-Control-Max-Age', '3600'); + + if (req.method === 'OPTIONS') { + res.status(204).send(''); + return; + } + + // Rate limiting + const rateLimitResult = await isRateLimited(req); + if (rateLimitResult.limited) { + res.set('Retry-After', String(rateLimitResult.retryAfter)); + res.status(429).json({ + error: 'Rate limit exceeded', + message: rateLimitResult.reason, + retryAfter: rateLimitResult.retryAfter + }); + return; + } + + const query = req.query.q || req.body?.q || ''; + const context = req.query.context || req.body?.context || ''; + const pageSize = parseInt(req.query.pageSize, 10) || 10; + + if (!query.trim()) { + res.status(400).json({ error: 'Missing query parameter q' }); + return; + } + + try { + const client = await auth.getClient(); + const token = await client.getAccessToken(); + + const stripHtml = str => str ? str.replace(/<[^>]*>/g, '') : null; + + const getSource = (url) => { + if (!url) return 'unknown'; + try { + const u = new URL(url); + const host = u.hostname.toLowerCase(); + if (host === 'www.interlisp.org' || host === 'interlisp.org') return 'website-primary'; + if (host.endsWith('.interlisp.org')) return 'website-secondary'; + if (host.includes('github.com') || url.includes('github.com')) return 'github'; + return 'other'; + } catch (_) { + if (url.includes('github.com')) return 'github'; + if (url.includes('interlisp.org')) return 'website-secondary'; + return 'other'; + } + }; + + const sourcePriorityMap = { + 'website-primary': 1, + 'website-secondary': 2, + 'github': 3, + 'other': 4, + 'unknown': 999 + }; + + const fetchPageSize = 20; + + const searchEngine = async (engineId, withSummary) => { + const endpoint = `https://discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/collections/default_collection/engines/${engineId}/servingConfigs/default_config:search`; + const body = { + query, + pageSize: fetchPageSize, + contentSearchSpec: { + snippetSpec: { returnSnippet: true }, + extractiveContentSpec: { maxExtractiveAnswerCount: 3 } + } + }; + if (withSummary) { + body.contentSearchSpec.summarySpec = { + summaryResultCount: 5, + includeCitations: true, + useSemanticChunks: true, + languageCode: 'en-US', + modelPromptSpec: { preamble: buildPreamble(context) }, + modelSpec: { version: 'stable' } + }; + } + const resp = await fetch(endpoint, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token.token}`, + 'Content-Type': 'application/json', + 'x-goog-user-project': PROJECT_ID + }, + body: JSON.stringify(body) + }); + if (!resp.ok) { + const errText = await resp.text(); + throw new Error(`Vertex API error ${resp.status} on ${engineId}: ${errText}`); + } + return resp.json(); + }; + + let websiteData; + let githubData; + if (GITHUB_ENGINE_ID) { + [websiteData, githubData] = await Promise.all([ + searchEngine(WEBSITE_ENGINE_ID, true), + searchEngine(GITHUB_ENGINE_ID, false) + ]); + } else { + websiteData = await searchEngine(WEBSITE_ENGINE_ID, true); + githubData = null; + } + + // Blended merge: guarantee GitHub despite website priority (70/30, 50/50 for GitHub queries) + const isGithubQuery = /github|issue|pull\s*request|\bpr\b|discussion/i.test(query); + const websiteTake = isGithubQuery ? Math.ceil(pageSize * 0.5) : Math.ceil(pageSize * 0.7); + const githubTake = pageSize - websiteTake; + + const toItems = (data, hint) => (data?.results || []).map(r => { + const d = r.document?.derivedStructData; + const s = r.document?.structData; + const url = d?.link || d?.url || s?.url || s?.link || null; + const src = hint || getSource(url); + return { + raw: r, + url, + source: src, + priority: sourcePriorityMap[src] || 999, + derived: d, + structData: s, + score: r.retrievalSignals?.semanticRelevanceScore || 0 + }; + }).filter(i => i.url); + + let websiteItems = toItems(websiteData, null).sort((a, b) => a.priority - b.priority || b.score - a.score); + let githubItems = toItems(githubData, 'github').sort((a, b) => b.score - a.score); + + // Dedup by URL + const seen = new Set(); + websiteItems = websiteItems.filter(i => !seen.has(i.url) && seen.add(i.url)); + githubItems = githubItems.filter(i => !seen.has(i.url) && seen.add(i.url)); + + let allResults = [...websiteItems.slice(0, websiteTake), ...githubItems.slice(0, githubTake)]; + if (allResults.length < pageSize) { + const rem = [...websiteItems.slice(websiteTake), ...githubItems.slice(githubTake)].sort((a, b) => a.priority - b.priority || b.score - a.score); + allResults = [...allResults, ...rem].slice(0, pageSize); + } + allResults.sort((a, b) => a.priority - b.priority || b.score - a.score); + + // docId → URL map for citations (from both engines) + const docIdToUrl = {}; + [...websiteItems, ...githubItems].forEach(i => { + if (i.raw.document?.id) docIdToUrl[i.raw.document.id] = i.url; + }); + + let references = (websiteData?.summary?.summaryWithMetadata?.references || []).map(ref => { + const docId = ref.document?.split('/').pop(); + const uri = docIdToUrl[docId] || null; + const src = getSource(uri); + return { + title: ref.title, + uri, + docId, + source: src, + priority: sourcePriorityMap[src] || 999 + }; + }).sort((a, b) => a.priority - b.priority); + + const results = allResults.map(i => ({ + id: i.raw.document?.id, + title: i.derived?.title || i.structData?.title || null, + url: i.url, + snippet: stripHtml(i.derived?.snippets?.[0]?.snippet || i.structData?.content?.slice(0, 300) || null), + source: i.source, + priority: i.priority, + section: i.url.replace(/^https:\/\/[^/]+\//, '').split('/')?.[0] || '', + type: i.structData?.type || null, + repo: i.structData?.repo || null, + state: i.structData?.state || null + })).slice(0, pageSize); + + res.json({ + summary: (websiteData?.summary?.summaryText ? { + summaryText: websiteData.summary.summaryText, + citations: references + } : null), + results + }); + + } catch (err) { + console.error('Search error:', err); + res.status(500).json({ error: 'Search failed', detail: err.message }); + } +}; + +// Re-export for github-reimport Cloud Function (shares same source dir but different entry-point) +try { + exports.reimportGithub = require('./github-reimport').reimportGithub; +} catch (e) { + // ignore if github-reimport not available during search deploy + void e; +} + +function buildPreamble(context) { + const base = `You are a search assistant for Interlisp.org. You answer questions about the Interlisp project using indexed documentation, code examples, historical information, and GitHub content (issues, PRs, discussions). + +## Core Principles +- Answer clearly and concisely. Always cite sources. +- If unsure, say "I don't know" — never guess. +- Prefer authoritative primary sources (Interlisp Reference Manual, official docs) over secondary sources. +- When multiple sources cover the same topic, cite the most authoritative one first. + +## Citation Priorities (in order) +1. **Interlisp.org website** (documentation, guides, examples) — preferred for general questions +2. **Official PDFs** (files.interlisp.org) — Medley documentation, release notes, reference manuals +3. **GitHub content** (issues, PRs, discussions, project status) — use when discussing development, maintenance, or community feedback +4. GitHub repository source files — use only if no better source exists + +When a topic is covered by both the website AND GitHub (e.g., release notes on interlisp.org vs GitHub issues), cite the website version. + +## Content Guidance +- **General Interlisp questions:** Use interlisp.org docs and PDFs as primary sources +- **Code examples:** Provide Interlisp examples; note Common Lisp differences when relevant; cite source docs +- **Development/maintenance:** Include GitHub issue/PR references; cite discussions that show community input +- **Historical context:** Use interlisp.org history section and official documents + +## Response Format (always use this structure) +- Opening paragraph (1-2 sentences) +- "## Key Points" with bullet points +- "## Details" for additional context (only if needed) +- "## Caveats" (only when needed for safety/accuracy) +- Citations at the end as [1], [2], [3] + +## Critical Notes +- Answer in Markdown only — no HTML +- When citing sources, include the direct link, not GitHub repo paths +- If the user is browsing a specific website section, prioritize results from that section`; + + if (context) { + return `${base}\n\n## Current Context\nThe user is browsing the "${context}" section of interlisp.org — prioritize results from that section when relevant.`; + } + return base; +} diff --git a/search-function/long_term_search_arch_proposal.md b/search-function/long_term_search_arch_proposal.md new file mode 100644 index 00000000..fcb92ebe --- /dev/null +++ b/search-function/long_term_search_arch_proposal.md @@ -0,0 +1,1100 @@ +# Long-Term Architecture Plan: Multi-Source Interlisp Search with Automated Refresh + +**Document Created:** August 26, 2024 +**Last Updated:** September 02, 2026 +**Status:** Implemented — Dual-Engine Blended (verified live 2026-09-02) +**Author:** OpenCode Architecture Planning Session +**Deployed:** `search-00037-xas` (`WEBSITE_ENGINE_ID=interlisp-website-only`, `GITHUB_ENGINE_ID=interlisp-github-only`) + +--- + +## Table of Contents + +1. [Executive Summary](#executive-summary) +2. [Current State Analysis](#current-state-analysis) +3. [Architecture Overview](#architecture-overview) +4. [Phase 1: Research Vertex AI Search Capabilities](#phase-1-research-vertex-ai-search-capabilities) +5. [Phase 2: Optimal Data Store Configuration](#phase-2-optimal-data-store-configuration) +6. [Phase 3: Source Weighting & Ranking Strategy](#phase-3-source-weighting--ranking-strategy) +7. [Phase 4: Automated Refresh Pipeline](#phase-4-automated-refresh-pipeline) +8. [Phase 5: Reference Manual Structuring](#phase-5-reference-manual-structuring) +9. [Phase 6: Updated Cloud Function](#phase-6-updated-cloud-function) +10. [Implementation Sequence](#implementation-sequence) +11. [Decision Points](#decision-points) + +--- + +## Executive Summary + +This document proposes a long-term architecture for the Interlisp.org AI-assisted search system that: + +- **Unifies** website, GitHub, and reference materials via two engines blended in one Cloud Function +- **Prioritizes** `www.interlisp.org` / `interlisp.org` (primary) over `*.interlisp.org` secondary subdomains (`files`, `primer`, `online`) over GitHub, while guaranteeing GitHub representation +- **Automates** content refresh (website crawl, GitHub `interlisp-github-v2` import) on a recurring schedule +- **Prepares** for future structuring of reference materials (PDFs, documentation) +- **Maintains** backward compatibility with current Cloud Function architecture + +### Key Improvements + +| Aspect | Current State | Proposed State | +|--------|---------------|-----------------| +| **Data Organization** | Two separate engines (`v3` website+GitHub, `176886...` website+bucket) | Two focused engines (`interlisp-website-only` [`interlisp-web-sites_1741606671710`], `interlisp-github-only` [`interlisp-github-v2`]) blended; `interlisp-search-unified` combines both for fallback | +| **Website Prioritization** | GitHub dominates or secondary subdomains dominate | `website-primary` (apex/`www`) → `website-secondary` (`files`/`primer`/`online`) → `github`; blended 70/30 (50/50 for GitHub queries) guarantees both | +| **Content Freshness** | Manual recrawl/reimport required | Automated daily website (via existing `*.interlisp.org` crawl), weekly GitHub JSONL import to `interlisp-github-v2` branches/0 | +| **Maintainability** | Two Cloud Functions, manual updates | Single `search` function dual-fetching both engines, automated refresh pipeline | +| **Future-Ready** | Limited structure for reference materials | `source`/`priority`/`last_indexed` metadata additive to GitHub `repo`/`type`/`created_date`; framework ready for structured imports | + +--- + +## Current State Analysis + +### Existing Implementations + +#### `interlisp-org-search_1768860477660` (Unstructured) +- **Contents:** interlisp.org website + files.interlisp.org (PDFs, documentation) +- **Strengths:** ✅ Website content surfaces first, clean results +- **Weaknesses:** ❌ Reference Manual PDFs are unstructured, not searchable at section/chapter level + +#### `interlisp-search-v3` (Structured) +- **Contents:** interlisp.org website + files.interlisp.org + GitHub structured data +- **GitHub Data:** Issues, PRs, Markdown files +- **Strengths:** ✅ GitHub content available as context +- **Weaknesses:** ❌ GitHub content dominates results, website is deprioritized + +### Root Cause: No Source-Level Weighting + +Vertex AI Search has **limited native support** for source-level prioritization at the data store configuration level. The system returns results based on semantic relevance and ranking, without explicit mechanisms to say "always rank source A above source B." + +**Solution:** Combine request-time filtering + post-processing logic in the Cloud Function to enforce source priority. + +--- + +## Architecture Overview + +### System Diagram — Implemented (dual-engine blended) + +``` +┌──────────────────────────────┐ ┌──────────────────────────────┐ +│ DATA STORE │ │ DATA STORE │ +│ interlisp-web-sites_ │ │ interlisp-github-v2 │ +│ 1741606671710 │ │ (NO_CONTENT, GENERIC) │ +│ PUBLIC_WEBSITE, *.interlisp. │ │ structData: title/url/content│ +│ org/*, SUCCEEDED │ │ + source/priority/last_indexed│ +└──────────────┬───────────────┘ └──────────────┬───────────────┘ + │ │ + ▼ ▼ +┌──────────────────────────────┐ ┌──────────────────────────────┐ +│ ENGINE │ │ ENGINE │ +│ interlisp-website-only │ │ interlisp-github-only │ +│ [interlisp-web-sites_...] │ │ [interlisp-github-v2] │ +│ GENERIC, ENTERPRISE+LLM │ │ GENERIC, ENTERPRISE+LLM │ +└──────────────┬───────────────┘ └──────────────┬───────────────┘ + │ │ + └──────────────┬──────────────────────┘ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ CLOUD FUNCTION `search` (Gen2, us-central1) │ +│ WEBSITE_ENGINE_ID=interlisp-website-only │ +│ GITHUB_ENGINE_ID=interlisp-github-only │ +│ 1. Parallel search both engines (fetch 20 each) │ +│ 2. getSource(): website-primary (interlisp.org/www) →1, │ +│ website-secondary (*.interlisp.org) →2, github →3 │ +│ 3. Blended merge: 70/30 (50/50 if /github|issue|pr/i) │ +│ guarantees GitHub despite website priority │ +│ 4. Summary from website engine, preamble prefers www, │ +│ citations sorted primary→secondary→github │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ AUTOMATED REFRESH PIPELINE (Cloud Scheduler + Functions) │ +│ Trigger 1: Weekly GitHub JSONL → interlisp-github-v2 │ +│ branches/0 (source/priority/last_indexed) │ +│ Trigger 2: Website crawl is managed by existing *.interlisp │ +│ targetSites (SUCCEEDED) — no recreation needed │ +└─────────────────────────────────────────────────────────────┘ + Fallback engine interlisp-search-unified [interlisp-web-sites, interlisp-github-v2] retained for single-engine callers. +``` + +### Data Flow — Implemented + +1. **User Query** → Cloud Function `search` +2. **Parallel** `POST .../engines/interlisp-website-only/servingConfigs/default_config:search` (with `summarySpec` + preamble) + `POST .../engines/interlisp-github-only/...` (results only), each `pageSize:20` +3. **Merge:** dedup by `url`, `getSource()` → `website-primary` (1) / `website-secondary` (2) / `github` (3), blended 70/30 (50/50 for GitHub queries) so GitHub never starved, then final priority + `semanticRelevanceScore` sort, slice to `pageSize` +4. **Summary** from website engine (LLM preamble: `www.interlisp.org` → `*.interlisp.org` → GitHub), citations resolved via `docId→url` map and sorted same priority +5. **Response** → `results[]` + `summary {summaryText, citations[]}` with `source`/`priority` for observability + +### Refresh Flow + +1. **Daily @ 2 AM UTC** → Cloud Scheduler → `trigger-website-recrawl()` Cloud Function + - Initiates Vertex AI recrawl of `interlisp.org/*` and `files.interlisp.org/*` patterns + - Updates `last_indexed` metadata + +2. **Weekly (Monday) @ 3 AM UTC** → Cloud Scheduler → `github-reimport()` Cloud Function + - Fetches latest GitHub Issues, PRs, Markdown files + - Transforms to JSONL with metadata (`source: github`, `priority: 3`) + - Uploads to Cloud Storage + - Calls Vertex AI `importDocuments()` API + - Clears old GitHub documents before import + +3. **Manual Triggers** → On-demand via Cloud Scheduler UI or `gcloud` CLI + - For urgent updates, testing, or manual validation + +--- + +## Phase 1: Research Vertex AI Search Capabilities + +### Findings + +Vertex AI Search (Discovery Engine) supports the following relevant features: + +#### ✅ Supported + +1. **Metadata-based filtering & boosting** via search request parameters + - `filter` parameter in `contentSearchSpec` + - Can filter by custom metadata fields added during import + - Example: `filter: "metadata.source:website"` + +2. **Multiple target sites** in a single data store + - Different URL patterns (`interlisp.org/*`, `files.interlisp.org/*`) + - Both crawled automatically + - Can specify recrawl triggers per target + +3. **Structured document imports** (JSONL) alongside crawled content + - GitHub Issues, PRs, Markdown as JSONL documents + - Custom metadata fields included in import schema + - Can be re-imported periodically to update content + +4. **Custom metadata fields** on imported documents + - Fields like `source`, `priority`, `last_updated`, `content_type` + - Available for filtering, sorting, and post-processing + +5. **Search-time parameters** to influence ranking + - `pageSize`, `offset` for pagination + - `filter` for inclusion/exclusion criteria + - `orderBy` for explicit sorting (limited) + +#### ⚠️ Limited + +1. **Source-level weighting at config time** + - No built-in "prioritize collection A over B" setting + - Must implement via request-time filtering or post-processing + +2. **Automatic deduplication across sources** + - Same content from website + GitHub may appear as separate results + - Requires post-processing to deduplicate if needed + +### Implication — Implemented + +**Implemented approach:** **Two focused data stores + dual-engine blended Cloud Function** (replaces the earlier “single unified store” proposal). + +* Website store `interlisp-web-sites_1741606671710` (`PUBLIC_WEBSITE`, `*.interlisp.org/*`, `SUCCEEDED`) — proven to return `https://interlisp.org/...` for `history`/`software`/`Medley` (the freshly created `interlisp-search-unified` website store `PUBLIC_WEBSITE` `interlisp.org/*` + `files.interlisp.org/*` returned only `files`/`online`/`primer` and is retained but not used for serving). +* GitHub store `interlisp-github-v2` (`NO_CONTENT`, `GENERIC`, structured `structData` on `branches/0`) — GitHub issues/PRs/markdown with future-proof `retrievable:true` schema (title/url/repo/type/state/author/number etc., see Phase 2). +* Fallback engine `interlisp-search-unified` (`[interlisp-web-sites_1741606671710, interlisp-github-v2]`) retained for single-engine callers. + +This dual-store blending avoids GitHub starvation (guaranteed 70/30, 50/50 for GitHub-intent queries) while keeping website-primary strictly first. + +--- + +## Phase 2: Optimal Data Store Configuration — Implemented (dual focused stores) + +### Recommendation: Two Focused Stores Blended in Cloud Function (Implemented) + +| Approach | Pros | Cons | Implemented | +|----------|------|------|-------------| +| **Unified** (`interlisp-search-unified` website store) | Single query | Requires careful filtering; fresh `interlisp-search-unified` crawl returned only `files`/`online`/`primer` for `history`/`Medley` — not suitable for serving | — (retained, not used) | +| **Split (implemented)** | Proven website crawl `interlisp-web-sites_1741606671710` (`*.interlisp.org/*` `SUCCEEDED`) + structured `interlisp-github-v2` (`NO_CONTENT` `branches/0`) blended 70/30; fallback `interlisp-search-unified` engine combines both | Slightly more Cloud Function logic (dual fetch) | ✓ | + +**Decision:** **Dual focused stores** — `interlisp-web-sites_1741606671710` + `interlisp-github-v2` (+ fallback engine `interlisp-search-unified`). + +### Implemented Data Store Configuration + +#### Website Store (proven, primary) + +```yaml +Data Store ID: interlisp-web-sites_1741606671710 +Display Name: Interlisp Web Sites +Project: interlispsearch +Location: global +Industry Vertical: GENERIC +Solution Types: SOLUTION_TYPE_SEARCH +Content Config: PUBLIC_WEBSITE +Target Site: + URI Pattern: "*.interlisp.org/*" + Type: INCLUDE + Indexing Status: SUCCEEDED # proven for history/software/Medley queries +Advanced Site Search: disableInitialIndex:true, disableAutomaticRefresh:true +``` + +*Note:* Fresh store `interlisp-search-unified` (`PUBLIC_WEBSITE` `interlisp.org/*` + `files.interlisp.org/*`) was created per guide Steps 1.2–1.5 and is `SUCCEEDED`, but its crawl was not used for serving; it is retained for inventory. + +#### GitHub Structured Store (`NO_CONTENT`, branches/0) + +```yaml +Data Store ID: interlisp-github-v2 +Display Name: Interlisp GitHub Content +Project: interlispsearch +Location: global +Industry Vertical: GENERIC +Solution Types: SOLUTION_TYPE_SEARCH +Content Config: NO_CONTENT +Branches: 0 # NOT default/branches/default +Schema: default_schema # future-proof, see below +``` + +**Schema `default_schema` (future-proof, patched 2026-09-02):** + +Future-proof patch made all display fields `retrievable:true` so Search API returns them in both `structData` and fallback `derivedStructData`: + +```yaml +title: {keyPropertyMapping: title, retrievable: true} +url: {keyPropertyMapping: uri, retrievable: true} +content: {keyPropertyMapping: description, retrievable: true} +repo: {retrievable: true, searchable: true, indexable: true} +type: {retrievable: true, searchable: true, indexable: true} +state: {retrievable: true, searchable: true, indexable: true} +author/number/labels/references/created_at/updated_at/created_date/priority/last_indexed/file/issue_number/pr_number: retrievable: true (as applicable) +``` + +Without this patch, GitHub results were `title:null` → frontend `Untitled` (fixed 2026-09-02; `github-only` engine now returns `structData.title` and Cloud Function `index.js:193-195` `derived.title || structData.title` resolves). + +**Patch command (idempotent):** +```bash +TOKEN=$(gcloud auth application-default print-access-token) +curl -X PATCH -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + -H "Content-Type: application/json" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores/interlisp-github-v2/schemas/default_schema" \ + -d @/tmp/patch_schema4.json # see Implementation Guide Phase 1.6 for JSON +``` + +#### Structured Data Imports (JSONL) — Actual Shape + +```json +{"id":"github-issue-medley-2741","structData":{"id":"github-issue-medley-2741","title":"FONTSAVAILABLE…","url":"https://github.com/Interlisp/medley/issues/2741","content":"…","source":"github","priority":3,"type":"issue","repo":"Interlisp/medley","state":"open","number":2741,"created_at":"2026-08-26T01:21:42Z","last_indexed":"2026-09-02T12:54:55Z"}} +``` +Sanitized `id` (`Interlisp.github.io` → `Interlisp-github-io`), uploaded to `gs://interlispsearch-search-imports/github-import-*.jsonl`, imported via `POST …/branches/0/documents:import`. + +#### Indexing Configuration — Implemented + +```yaml +Website: Existing *.interlisp.org crawl managed by siteSearchEngine (no recreation needed) +GitHub: Weekly JSONL import to interlisp-github-v2 branches/0 (local one-off until scheduled reimport enabled) +``` + +Initial GitHub Seed (implemented 2026-09-02 via local one-off `GITHUB_TOKEN=$(gh auth token) node /tmp/local_import_simple.js` → 350 docs `successCount 350/350`); scheduled weekly `github-reimport` Pub/Sub requires `GITHUB_TOKEN` env and is deferred until source-dir isolation fixed (search function already re-exports `reimportGithub`). + +Fallback engine `interlisp-search-unified` (`[interlisp-web-sites_1741606671710, interlisp-github-v2]`) retained for single-engine callers; live `search` uses dual fetch. + +--- + +## Phase 3: Source Weighting & Ranking Strategy — Implemented (dual-fetch blended) + +### Implemented: URL-based `getSource()` + Priority + Guaranteed GitHub Slice + +Live `search-function/index.js:64-86,138-172` (deployed `search-00037-xas`) implements: + +* **No `filter` param** (proposal filtered by `metadata.source` proved unnecessary and would starve GitHub). Ranking is pure post-processing on URL + blended slice. + +* **Source detection (`getSource(url)`):** + ```javascript + host === 'www.interlisp.org' || host === 'interlisp.org' → 'website-primary' (1) + host.endsWith('.interlisp.org') → 'website-secondary' (2) // files/primer/online — strictly after primary, not tiered further + host.includes('github.com') → 'github' (3) + other → 4, unknown → 999 + ``` + +* **Parallel fetch:** `Promise.all([searchEngine(WEBSITE_ENGINE_ID, withSummary:true), searchEngine(GITHUB_ENGINE_ID, withSummary:false)])` each `fetchPageSize=20` (website engine provides `summarySpec` + preamble; github engine results-only). + +* **Blended merge (guaranteed GitHub, never starved):** + ```javascript + isGithubQuery = /github|issue|pull\s*request|\bpr\b|discussion/i.test(query) + websiteTake = ceil(pageSize * (isGithubQuery ? 0.5 : 0.7)) + githubTake = pageSize - websiteTake + // sort: websiteItems by priority→semanticRelevanceScore, githubItems by score + // dedup by url, take slices, fill remainder sorted priority→score, final sort priority→score + ``` + +* **Citations & results:** `docId→url` map across both engines, `references` sorted `website-primary → website-secondary → github`; `results` mapped with `title: derived.title || structData.title` (fixed by schema `retrievable:true`) and `type/repo/state` from `structData`. + +#### Tier 1: Search Request-Level Filtering (Not Used — See Implemented Above) + +Proposal `filter: buildSourceFilter(context)` was superseded by URL-based blended slice; kept only as fallback if `GITHUB_ENGINE_ID` unset (`index.js:128-136`). + +#### Tier 2: Post-Processing Reordering (Guardrail) — Implemented + +**Source Priority Order (implemented, strictly after, not tiered within secondary):** +1. `website-primary` (1) — `interlisp.org` / `www.interlisp.org` +2. `website-secondary` (2) — `*.interlisp.org` (files, primer, online) +3. `github` (3) — issues/PRs/discussions/markdown + +**Snippet from live `index.js:143-204`:** +```javascript +const toItems = (data, hint) => (data?.results||[]).map(r=>{... url = d.link||d.url||s.url ...; src=hint||getSource(url); return {raw:r, url, source:src, priority:sourcePriorityMap[src], score:retrievalSignals.semanticRelevanceScore}}).filter(i=>i.url); +let websiteItems = toItems(websiteData,null).sort((a,b)=>a.priority-b.priority||b.score-a.score); +let githubItems = toItems(githubData,'github').sort((a,b)=>b.score-a.score); +// dedup, blended slice, final priority→score sort +``` + +### Ranking Behavior — Verified Live + +| Query Type | Verified Behavior (live 2026-09-02) | +|------------|-------------------| +| General `interlisp/medley/history/software` | First 7 website-primary, last 3 github; citations website-primary first | +| GitHub-intent `github` / `github issue` | 5+5 split (50/50) — `isGithubQuery` triggers | +| `pull request` | 1 website + 9 github (still priority→score, githubTake 5 plus remainder fill) | +| Code/doc overlap | Website cited first per preamble; GitHub never starved | + +**Frontend:** `assets/js/vertex-search.js:220` `r.title || 'Untitled'` now resolves because schema patch makes `title` retrievable; verified 0 untitled across `github/medley/pull request/issue/screenshots/FONTSAVAILABLE` matrix. + +--- + +## Phase 4: Automated Refresh Pipeline + +### Goal + +Keep content fresh without manual intervention. Automate: +- **Website recrawl** (daily) — captures updated docs, blog posts, etc. +- **GitHub reimport** (weekly) — refreshes issues, PRs, discussions +- **Manual triggers** — for urgent updates or testing + +### Architecture + +``` +Cloud Scheduler (GCP) + │ + ├─ Trigger 1: Daily @ 2 AM UTC + │ └─ Pub/Sub Topic: "recrawl-website" + │ └─ Cloud Function: trigger-website-recrawl() + │ └─ Calls: Vertex AI API to recrawl target sites + │ + ├─ Trigger 2: Weekly (Monday @ 3 AM UTC) + │ └─ Pub/Sub Topic: "reimport-github" + │ └─ Cloud Function: github-reimport() + │ ├─ Fetch: GitHub API (issues, PRs, markdown) + │ ├─ Transform: to JSONL with metadata + │ ├─ Upload: to Cloud Storage + │ └─ Import: Vertex AI importDocuments() + │ + └─ Trigger 3: Manual (gcloud CLI or UI) + └─ Force recrawl/reimport on demand +``` + +### Step 1: Create `trigger-website-recrawl()` Cloud Function + +**Purpose:** Initiate Vertex AI recrawl of target sites + +**Deployment:** + +```bash +gcloud functions deploy trigger-website-recrawl \ + --gen2 \ + --runtime=nodejs24 \ + --region=us-central1 \ + --trigger-topic=recrawl-website \ + --entry-point=recrawlWebsite \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --memory=256Mi \ + --timeout=60s \ + --project=interlispsearch +``` + +**Code:** `search-function/trigger-website-recrawl.js` (targets proven website store) + +```javascript +'use strict'; + +const { GoogleAuth } = require('google-auth-library'); + +const PROJECT_ID = 'interlispsearch'; +const LOCATION = 'global'; +const DATA_STORE_ID = 'interlisp-web-sites_1741606671710'; // NOT interlisp-search-unified (its crawl returned only files) + +const auth = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'] +}); + +exports.recrawlWebsite = async (message, context) => { + console.log('Starting website recrawl...'); + + try { + const client = await auth.getClient(); + const token = await client.getAccessToken(); + + // Get target sites + const targetSitesUrl = `https://discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/collections/default_collection/dataStores/${DATA_STORE_ID}/siteSearchEngine/targetSites`; + + const targetSitesResponse = await fetch(targetSitesUrl, { + method: 'GET', + headers: { + 'Authorization': `Bearer ${token.token}`, + 'x-goog-user-project': PROJECT_ID + } + }); + + if (!targetSitesResponse.ok) { + throw new Error(`Failed to fetch target sites: ${targetSitesResponse.statusText}`); + } + + const targetSitesData = await targetSitesResponse.json(); + const targetSites = targetSitesData.targetSites || []; + + console.log(`Found ${targetSites.length} target sites. Starting recrawl...`); + + // Trigger recrawl for each target site + for (const site of targetSites) { + const recrawlUrl = `https://discoveryengine.googleapis.com/v1/${site.name}:recrawl`; + + const recrawlResponse = await fetch(recrawlUrl, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token.token}`, + 'Content-Type': 'application/json', + 'x-goog-user-project': PROJECT_ID + }, + body: JSON.stringify({}) + }); + + if (!recrawlResponse.ok) { + console.error(`Recrawl failed for ${site.uri_pattern}: ${recrawlResponse.statusText}`); + } else { + console.log(`Recrawl triggered for ${site.uri_pattern}`); + } + } + + console.log('Website recrawl completed'); + + } catch (err) { + console.error('Recrawl error:', err); + throw err; + } +}; +``` + +### Step 2: Create `github-reimport()` Cloud Function + +**Purpose:** Fetch latest GitHub content and reimport as structured documents + +**Deployment:** + +```bash +gcloud functions deploy github-reimport \ + --gen2 \ + --runtime=nodejs24 \ + --region=us-central1 \ + --trigger-topic=reimport-github \ + --entry-point=reimportGithub \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --memory=512Mi \ + --timeout=300s \ + --set-env-vars GITHUB_TOKEN=your_github_token_here \ + --project=interlispsearch +``` + +**Code:** `search-function/github-reimport.js` (pseudocode) + +```javascript +'use strict'; + +const { GoogleAuth } = require('google-auth-library'); +const { Storage } = require('@google-cloud/storage'); +const https = require('https'); + +const PROJECT_ID = 'interlispsearch'; +const LOCATION = 'global'; +const DATA_STORE_ID = 'interlisp-github-v2'; // NOT interlisp-search-unified; structured GitHub store on branches/0 +const GITHUB_TOKEN = process.env.GITHUB_TOKEN; +const GITHUB_ORG = 'Interlisp'; + +const auth = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'] +}); + +const storage = new Storage({ projectId: PROJECT_ID }); + +exports.reimportGithub = async (message, context) => { + console.log('Starting GitHub reimport...'); + + try { + // 1. Fetch GitHub issues, PRs, markdown files + const gitHubDocuments = []; + + // Fetch Issues + const issues = await fetchGitHubIssues(); + gitHubDocuments.push(...issues); + console.log(`Fetched ${issues.length} GitHub issues`); + + // Fetch Pull Requests + const prs = await fetchGitHubPullRequests(); + gitHubDocuments.push(...prs); + console.log(`Fetched ${prs.length} GitHub pull requests`); + + // Fetch Markdown files from repos + const markdownFiles = await fetchGitHubMarkdown(); + gitHubDocuments.push(...markdownFiles); + console.log(`Fetched ${markdownFiles.length} GitHub markdown files`); + + // 2. Convert to JSONL format with metadata + const jsonlContent = gitHubDocuments + .map(doc => JSON.stringify(doc)) + .join('\n'); + + // 3. Upload JSONL to Cloud Storage + const bucket = storage.bucket(`${PROJECT_ID}-search-imports`); + const file = bucket.file(`github-import-${Date.now()}.jsonl`); + + await file.save(jsonlContent); + console.log(`Uploaded JSONL to gs://${bucket.name}/${file.name}`); + + // 4. Call Vertex AI importDocuments API + const client = await auth.getClient(); + const token = await client.getAccessToken(); + + const importUrl = `https://discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/collections/default_collection/dataStores/${DATA_STORE_ID}/branches/0/documents:import`; // branches/0, not default + + // JSONL shape: {"id": sanitized, "structData":{id,title,url,content,source,priority,repo,type,state,…}} — NOT top-level title/url + const importResponse = await fetch(importUrl, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token.token}`, + 'Content-Type': 'application/json', + 'x-goog-user-project': PROJECT_ID + }, + body: JSON.stringify({ + gcsSource: { + inputUris: [`gs://${bucket.name}/${file.name}`] + } + }) + }); + + if (!importResponse.ok) { + throw new Error(`Import failed: ${importResponse.statusText}`); + } + + const importData = await importResponse.json(); + console.log(`Import operation started: ${importData.name}`); + console.log('GitHub reimport completed'); + + } catch (err) { + console.error('GitHub reimport error:', err); + throw err; + } +}; + +async function fetchGitHubIssues() { + // Fetch issues from Interlisp/medley, Interlisp/maiko, etc. + // Return array of documents with metadata + return []; +} + +async function fetchGitHubPullRequests() { + // Fetch PRs from Interlisp repositories + return []; +} + +async function fetchGitHubMarkdown() { + // Fetch README, markdown files from Interlisp repos + return []; +} + +function buildGitHubSchema() { + // Deprecated — schema is now patched via REST PATCH on default_schema (retrievable:true for title/url/repo/type/state etc.) + // New JSONL uses flat structData: {title, url, content, source, priority, repo, type, state, number, created_at,…} + // Sanitized id: Interlisp.github.io → Interlisp-github-io, /^[a-zA-Z0-9-_]+$/ + return null; +} +``` + +### Step 3: Create Cloud Scheduler Jobs + +**Daily Website Recrawl:** + +```bash +gcloud scheduler jobs create pubsub recrawl-website-daily \ + --location=us-central1 \ + --schedule="0 2 * * *" \ + --timezone="UTC" \ + --topic=recrawl-website \ + --message-body='{"action":"recrawl","timestamp":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"}' \ + --project=interlispsearch +``` + +**Weekly GitHub Reimport (Monday @ 3 AM UTC):** + +```bash +gcloud scheduler jobs create pubsub reimport-github-weekly \ + --location=us-central1 \ + --schedule="0 3 * * 1" \ + --timezone="UTC" \ + --topic=reimport-github \ + --message-body='{"action":"reimport","timestamp":"'$(date -u +%Y-%m-%dT%H:%M:%SZ)'"}' \ + --project=interlispsearch +``` + +**Manual Triggers (via gcloud CLI):** + +```bash +# Trigger website recrawl immediately +gcloud scheduler jobs run recrawl-website-daily \ + --location=us-central1 \ + --project=interlispsearch + +# Trigger GitHub reimport immediately +gcloud scheduler jobs run reimport-github-weekly \ + --location=us-central1 \ + --project=interlispsearch +``` + +### Step 4: Monitoring & Logging + +**View Cloud Function logs:** + +```bash +# Website recrawl logs +gcloud functions logs read trigger-website-recrawl \ + --gen2 \ + --region=us-central1 \ + --limit=50 \ + --project=interlispsearch + +# GitHub reimport logs +gcloud functions logs read github-reimport \ + --gen2 \ + --region=us-central1 \ + --limit=50 \ + --project=interlispsearch +``` + +**View Cloud Scheduler execution history:** + +```bash +# Website recrawl schedule +gcloud scheduler jobs describe recrawl-website-daily \ + --location=us-central1 \ + --project=interlispsearch + +# GitHub reimport schedule +gcloud scheduler jobs describe reimport-github-weekly \ + --location=us-central1 \ + --project=interlispsearch +``` + +--- + +## Phase 5: Reference Manual Structuring + +### Current Problem + +Reference Manual PDFs in `files.interlisp.org/` are indexed as unstructured content. This causes: +- Lost context (which chapter/section?) +- Poor chunking (full document treated as one blob) +- Lower relevance in search results + +### Long-Term Solution Options + +#### Option A: Structured PDF Extraction (Medium-term, 1-2 months) + +**Approach:** +1. Extract Reference Manual PDFs with chapter/section metadata +2. Create JSONL documents with hierarchical structure: + ```json + { + "id": "refman-ch3-sec2-subsec1", + "title": "Chapter 3.2.1: Function Definitions", + "content": "Section content here...", + "metadata": { + "source": "reference_manual", + "priority": 2, + "book": "Interlisp Reference Manual", + "chapter": "3", + "section": "2", + "subsection": "1", + "page_range": "123-145" + } + } + ``` +3. Import as structured documents using same pipeline as GitHub +4. Update Cloud Function to recognize and prioritize reference material + +**Pros:** +- Searchable at chapter/section level +- Retains hierarchical structure +- Can deduplicate if content appears on website too + +**Cons:** +- Requires PDF parsing library (pypdf, pdfplumber, etc.) +- Manual validation for OCR errors +- One-time extraction effort + +#### Option B: Text-to-Markdown Conversion (Long-term, 2-3 months) + +**Approach:** +1. Convert Reference Manual PDFs to structured Markdown +2. Organize by chapter/section/subsection with proper heading hierarchy +3. Host as `.md` files in new `docs.interlisp.org` subdomain (or `interlisp.org/docs/`) +4. Let Vertex AI crawl them as website content (gets better indexing) + +**Pros:** +- Best UX — users can view docs directly on website +- Website crawl handles indexing (no separate import needed) +- Heading structure provides natural chunking +- Can add cross-references, links, examples + +**Cons:** +- Larger effort (PDF → Markdown conversion + hosting) +- Requires ongoing maintenance as docs change + +#### Option C: Defer Structuring (Short-term) + +**Approach:** +- Use unified data store with website content prioritized +- Keep Reference Manual as unstructured for now +- Revisit when search quality metrics indicate PDFs are a priority + +**Pros:** +- Minimal effort; unblock other improvements +- Gather user feedback on what's missing + +**Cons:** +- Reference Manual remains hard to search + +### Recommendation + +**Short-term (now):** Option C — defer to unblock unified data store deployment +**Medium-term (after 2-4 weeks):** Option A — structured extraction, validate results +**Long-term (3-6 months):** Option B — full Markdown conversion for best UX + +--- + +## Phase 6: Updated Cloud Function — Implemented + +### Changes Implemented (live `search-00037-xas`) + +`search-function/index.js:6-9,64-86,90-172,193-195` implements dual-fetch blended (see Phase 3) plus re-export for shared source dir: + +1. ✅ **Dual-fetch blending** (replaces Tier 1 `filter`): `WEBSITE_ENGINE_ID=interlisp-website-only`, `GITHUB_ENGINE_ID=interlisp-github-only`, `fetchPageSize=20` each, `isGithubQuery` 50/50 else 70/30 +2. ✅ **Post-processing reordering**: `getSource()` → `website-primary:1 / website-secondary:2 / github:3`, priority→`semanticRelevanceScore`, dedup by `url` +3. ✅ **Points to focused engines** (not unified): `WEBSITE_ENGINE_ID`/`GITHUB_ENGINE_ID`/`ENGINE_ID=WEBSITE_ENGINE_ID`; fallback `interlisp-search-unified` retained +4. ✅ **Preamble citation preference** preserved (`www` → `*.interlisp.org` → GitHub) +5. ✅ **Source metadata tracking** (`source`/`priority`/`type`/`repo`/`state`) for observability — verified 0 untitled after schema patch +6. ✅ **Shared-source re-export**: `exports.reimportGithub = require('./github-reimport').reimportGithub` so same `search-function/` dir serves both `search` and `github-reimport` entry-points + +### Deployment — Implemented + +```bash +# search-function/deploy-search-function.sh:6-9 +WEBSITE_ENGINE_ID="interlisp-website-only" # [interlisp-web-sites_1741606671710] +GITHUB_ENGINE_ID="interlisp-github-only" # [interlisp-github-v2] +FALLBACK_ENGINE_ID="interlisp-search-unified" + +gcloud functions deploy search \ + --gen2 --runtime=nodejs24 --region=us-central1 \ + --source=./search-function --entry-point=search --trigger-http --allow-unauthenticated \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --set-env-vars PROJECT_ID=interlispsearch,WEBSITE_ENGINE_ID=$WEBSITE_ENGINE_ID,GITHUB_ENGINE_ID=$GITHUB_ENGINE_ID,ENGINE_ID=$WEBSITE_ENGINE_ID \ + --memory=256Mi --timeout=30s --project=interlispsearch +# Deployed: https://us-central1-interlispsearch.cloudfunctions.net/search +``` + +### Testing Strategy + +**Before switching to unified data store:** + +1. Create unified data store (clone from v3 + add metadata) +2. Deploy updated Cloud Function to **staging** or **development** +3. Test with sample queries: + ```bash + # Should prefer website + curl -s "https:///search?q=interlisp" | jq '.results[0:3]' + + # Should cite interlisp.org docs, not GitHub + curl -s "https:///search?q=medley+release" | jq '.summary.citations[0]' + + # GitHub content still available for context + curl -s "https:///search?q=github+issues" | jq '.results | map(.url) | map(select(contains("github")))' + ``` + +4. Compare results with current v3 behavior +5. Validate that response time is acceptable +6. Check Cloud Function logs for errors + +**Validation Checklist:** + +- [ ] First result is from interlisp.org (not github.com) +- [ ] First citation is from interlisp.org (not github.com) +- [ ] GitHub content still appears in results (not excluded) +- [ ] Summary quality is same or better than before +- [ ] No increase in error rates or response time +- [ ] Cloud Function logs are clean (no errors) +- [ ] All allowed origins work (localhost, staging, production) + +### Rollback Plan + +If issues occur post-deployment: + +```bash +# Option 1: Switch back to old engine +gcloud functions deploy search \ + --gen2 \ + --runtime=nodejs24 \ + --region=us-central1 \ + --source=./search-function \ + --entry-point=search \ + --trigger-http \ + --allow-unauthenticated \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --set-env-vars \ + PROJECT_ID=interlispsearch,\ + ENGINE_ID=interlisp-org-search_1768860477660,\ + ALLOWED_ORIGIN=https://interlisp.org \ + --memory=256Mi \ + --timeout=30s \ + --project=interlispsearch + +# Option 2: Restore from git +cd search-function +git checkout index.js +npm install +# ... redeploy with same gcloud functions deploy command +``` + +--- + +## Implementation Sequence + +### Timeline & Effort Estimates + +| Phase | Task | Effort | Priority | Dependencies | +|-------|------|--------|----------|--------------| +| **2** | Create/verify dual focused stores (`interlisp-web-sites_1741606671710` proven, `interlisp-github-v2` schema patched future-proof) | 2-4 hrs | HIGH | None | +| **3** | Update Cloud Function with dual-fetch blended logic (70/30, 50/50, website-primary→secondary→github) | 1-2 hrs | HIGH | Phase 2 | +| **6** | Deploy + test updated function (staging → production, verify 0 untitled) | 1-2 hrs | HIGH | Phase 3 | +| **4** | Set up Cloud Scheduler + recrawl/reimport functions (website via existing crawl, github via local one-off until scheduled) | 2-3 hrs | MEDIUM | Phase 2 | +| **5** | Plan Reference Manual structuring strategy | 1 hr | LOW | None | +| **7** | **Cleanup — pare interlispsearch to in-use resources only** | 1-2 hrs | HIGH | Phases 2,3,6 | + +**Total Estimated Time:** 8-14 hours + +### Recommended Execution Order + +**Week 1: Core Implementation** + +1. ✅ **Phase 2** — Verify dual stores + patch schema + - Confirm `interlisp-web-sites_1741606671710` (`*.interlisp.org/*` `SUCCEEDED`) proven + - Patch `interlisp-github-v2` default_schema (title/url/repo/type/state → `retrievable:true`) future-proof + - Import GitHub JSONL via local one-off (`GITHUB_TOKEN=$(gh auth token) node /tmp/local_import_simple.js` → 350 `branches/0` `successCount 350`) + +2. ✅ **Phase 3** — Update Cloud Function (dual-fetch blended) + - Replace `filter` with `getSource()` + `website-primary:1 → website-secondary:2 → github:3` + 70/30 (50/50 for github queries) + - Add `stripHtml`, `semanticRelevanceScore` tie-break, dedup, citation sorting + - Test locally `node -c index.js` + `npm run lint` + +3. ✅ **Phase 6** — Deploy & test + - Deploy to staging first (`search-00037-xas`) + - Run test queries from Testing Strategy (verify 0 untitled across github/medley/screenshots/FONTSAVAILABLE) + - Compare results with current v3 + - Deploy to production once validated + +**Week 2: Automation** + +4. ✅ **Phase 4** — Set up Cloud Scheduler + automation functions + - `trigger-website-recrawl` targets `interlisp-web-sites_1741606671710` (existing crawl, no recreation) + - `github-reimport` targets `interlisp-github-v2` `branches/0` (local one-off until isolated source dir for scheduled Pub/Sub; re-exports via `index.js`) + - Create Cloud Scheduler jobs (daily website, weekly GitHub — deferred weekly until isolated deploy) + - Test manual trigger to verify end-to-end flow + +5. ⏳ **Phase 5** — Plan Reference Manual structuring (defer to later sprint) + - Document Option A (structured extraction) + - Identify PDF parsing tools + - Plan manual validation approach + +**Week 3: Cleanup (after unified stack validated in production)** + +6. ✅ **Phase 7** — Pare interlispsearch to in-use resources only + - Inventory all data stores, engines, functions, scheduler jobs, storage buckets, Firestore collections + - Mark `interlisp-web-sites_1741606671710`, `interlisp-github-v2`, engines `interlisp-website-only`/`interlisp-github-only`/`interlisp-search-unified` (fallback), `search`/`github-reimport` (`trigger-website-recrawl` if kept) + scheduler jobs as **KEEP** + - Delete all other Discovery Engine data stores/engines (e.g. `interlisp-search-v3`, `interlisp-org-search_1768860477660`, `interlisp-site-search`, document-bucket if superseded), stale functions, and `interlispsearch-search-imports` staging artifacts — see Implementation Guide Phase 7 for safe-delete sequence + +--- + +## Decision Points + +Before proceeding with implementation, please confirm: + +### Decision 1: Unified Data Store Approach + +**Question:** Does a single unified data store (with metadata-based ranking) make sense for your use case? + +**Alternatives:** +- Keep two separate data stores; update Cloud Function routing logic +- Create three data stores (website-only, website+github, website+github+reference) + +**Recommendation:** ✅ Single unified data store + +**Your Decision:** [ ] Approve [ ] Request modifications [ ] Choose alternative + +--- + +### Decision 2: Metadata Tagging + +**Question:** Should we tag all existing documents with `source` and `priority` metadata? + +**Approach:** +- Website-crawled documents: automatically tagged `source:website, priority:1` +- GitHub imported documents: manually tagged in JSONL `source:github, priority:3` +- Reference manual (future): tagged `source:reference_manual, priority:2` + +**Implementation:** +- Website documents may not have explicit metadata (use URL pattern to infer) +- GitHub documents get metadata during JSONL import +- Reference docs get tagged during PDF extraction + +**Your Decision:** [ ] Approve [ ] Request modifications + +--- + +### Decision 3: Refresh Frequency + +**Question:** Does the proposed refresh schedule work for your needs? + +**Proposed:** +- Website: Daily @ 2 AM UTC (captures doc updates within 24 hours) +- GitHub: Weekly, Monday @ 3 AM UTC (captures issues/PRs within 7 days) + +**Alternatives:** +- Website: Every 6 hours (more frequent, higher API costs) +- GitHub: Every 3 days (faster feedback on issues) +- Manual only (no automation) + +**Your Decision:** [ ] Approve [ ] Request modifications (specify frequency) + +--- + +### Decision 4: Reference Manual Priority + +**Question:** When should we tackle Reference Manual structuring? + +**Options:** +- A. Defer until after unified data store is live and stable (4-6 weeks) +- B. Start immediately (parallel with data store setup, +3-4 weeks) +- C. Defer indefinitely; revisit based on user feedback + +**Recommendation:** ✅ Option A (defer to unblock core improvements) + +**Your Decision:** [ ] Option A (defer) [ ] Option B (start now) [ ] Option C (indefinite) + +--- + +### Decision 5: Architecture Style Preference + +**Question:** Any changes to the proposed architecture? + +- Single unified data store with metadata-based ranking +- Cloud Scheduler automation for refresh +- Post-processing logic in Cloud Function for result reordering +- Preamble guidance for citation preference + +**Your Decision:** [ ] Approved as-is [ ] Request modifications (describe) + +--- + +## Phase 7 — Cleanup: Pare interlispsearch to In-Use Resources Only + +**Goal:** After the unified stack (`interlisp-search-unified` + its engine + updated `search` function + automation) is validated in production, delete every other resource in `interlispsearch` so the project contains only what is actively used. This removes the “large collection of data stores” discovered via the REST `list` call and prevents quota/cost drift. + +**Triggers:** Do NOT run until Phases 2–4 and 6 are complete and `interlisp-search-unified` serves production traffic for ≥ 48h. + +### Keep List (explicit) — Updated + +| Resource type | Name / ID | Why kept | +|---|---|---| +| Data store | `interlisp-web-sites_1741606671710` | Proven website crawl (`*.interlisp.org/*` `SUCCEEDED`) — primary | +| Data store | `interlisp-github-v2` | Structured GitHub store (`NO_CONTENT` `branches/0`, future-proof schema `retrievable:true`) | +| Data store | `interlisp-search-unified` | Fresh website store (`PUBLIC_WEBSITE` interlisp.org/*+files) retained, not used for serving (fallback) | +| Engine (search app) | `interlisp-website-only` / `interlisp-github-only` | Focused engines blended in live `search` | +| Engine (search app) | `interlisp-search-unified` (`[interlisp-web-sites_1741606671710,interlisp-github-v2]`) | Fallback for single-engine callers | +| Cloud Functions (Gen2) | `search` (`search-00037-xas`), `github-reimport` (needs isolated source dir for Pub/Sub) | Production + GitHub import | +| Cloud Scheduler | `reimport-github-weekly` (deferred weekly), `recrawl-website-daily` if kept | Refresh | +| Firestore | `rate_limits` collection (TTL on `updatedAt`) | Rate limiting | +| Cloud Storage | `interlispsearch-search-imports` (or current import bucket) | GitHub JSONL staging — keep bucket, prune old objects | +| IAM | `vertex-search-sa@interlispsearch.iam.gserviceaccount.com` + org-policy overrides | Runtime identity | +| Project | `interlispsearch` itself | Container | + +Everything else in `interlispsearch` is **candidate for deletion**. + +### Inventory First (read-only) + +Run the inventory script from the Implementation Guide (§ Phase 7, `inventory-interlispsearch.sh`) — it `GET`s: + +* `GET /v1/projects/INTERLISPSEARCH/locations/global/collections/default_collection/dataStores` — all data stores +* `GET /v1/projects/INTERLISPSEARCH/locations/global/collections/default_collection/engines` — all engines +* `gcloud functions list --gen2 --region=us-central1` +* `gcloud scheduler jobs list --location=us-central1` +* `gcloud storage ls` / `gsutil ls` +* `gcloud firestore collections list` (or console) + +Save JSON output to `/tmp/interlispsearch-inventory-YYYYMMDD/` and commit the keep/delete decision list for review. Require a second pair of eyes before any `DELETE`. + +### Safe-Delete Sequence + +Order matters — engines depend on data stores, functions depend on service account: + +1. **Engines first** — `DELETE /v1/projects/…/engines/{engineId}` for every engine whose `dataStoreIds` is not `["interlisp-search-unified"]`. Engines are cheap to recreate; data stores are not. +2. **Data stores second** — `DELETE /v1/projects/…/dataStores/{dataStoreId}` for stale stores (`interlisp-search-v3`, `interlisp-org-search_1768860477660`, `interlisp-site-search` if superseded, and any `test-*`/`tmp-*`). Each delete cascades to its documents/targetSites; allow 5–10 min per store and verify `404` on subsequent `GET`. +3. **Cloud Functions third** — `gcloud functions delete {name} --gen2 --region=us-central1` for any function not in the keep list (e.g. old `search-v2`, staging clones). +4. **Cloud Scheduler fourth** — `gcloud scheduler jobs delete {name} --location=us-central1` for stale jobs. +5. **Cloud Storage fifth** — `gcloud storage rm gs://BUCKET/github-import-*.jsonl` for import artifacts older than the last successful import; do not delete the bucket itself. +6. **Firestore sixth** — prune `rate_limits` documents only via TTL (do not delete the database); remove any ad-hoc `test-*` collections. +7. **IAM last** — do not delete the service account; only remove unused keys (`gcloud iam service-accounts keys delete`). + +All deletes use the REST `DELETE` with `x-goog-user-project: interlispsearch` and `Authorization: Bearer $(gcloud auth application-default print-access-token)` — there is no `gcloud beta discoveryengine` CLI (verified SDK 568.0.0; see Phase 7 scripts). + +### Validation After Cleanup + +* Re-run the inventory script — expected counts: `dataStores=1`, `engines=1`, `functions=3`, `schedulerJobs=2`. +* `curl "https://us-central1-interlispsearch.cloudfunctions.net/search?q=interlisp"` still returns `200` with unified-store citations (`source:website` first). +* No `discoveryengine.googleapis.com` quota errors in Cloud Logging. + +--- + +## Next Steps + +Once you approve the decision points above: + +1. **I will create detailed implementation scripts** for each phase +2. **I will provide exact gcloud commands** for data store setup +3. **I will write complete Cloud Function code** (with error handling, logging) +4. **I will create testing checklist** and validation steps +5. **I will provide monitoring/observability setup** (logging, alerts) +6. **Phase 7 will be executed only after production validation** — inventory → review → delete in the order above + +--- + +## Appendix: References + +### Vertex AI Search Documentation + +- [Vertex AI Search Overview](https://cloud.google.com/generative-ai-app-builder/docs/unified-overview) +- [Data Store Configuration](https://cloud.google.com/generative-ai-app-builder/docs/manage-data-stores) +- [Search API Reference](https://cloud.google.com/generative-ai-app-builder/docs/search-reference) +- [Document Import (JSONL)](https://cloud.google.com/generative-ai-app-builder/docs/import-data) +- [Filter Syntax](https://cloud.google.com/generative-ai-app-builder/docs/filter-results) + +### Google Cloud Platform Resources + +- [Cloud Scheduler Documentation](https://cloud.google.com/scheduler/docs) +- [Cloud Functions Documentation](https://cloud.google.com/functions/docs) +- [Cloud Storage Documentation](https://cloud.google.com/storage/docs) +- [Discovery Engine API](https://cloud.google.com/python/docs/reference/google-cloud-discoveryengine/latest) + +### Related Files in This Repository + +- `search-function/index.js` — Current Cloud Function implementation +- `search-function/rateLimiter.js` — Firestore-based rate limiting +- `vertex-ai-search-implementation.md` — Current implementation guide +- `search-function/package.json` — Dependencies + +--- + +**Document Version:** 1.1 +**Last Updated:** September 02, 2026 +**Status:** Implemented — see Implementation Guide Phase 7 still pending (gated ≥48h production) + diff --git a/search-function/long_term_search_implementation_guide.md b/search-function/long_term_search_implementation_guide.md new file mode 100644 index 00000000..cbe21c61 --- /dev/null +++ b/search-function/long_term_search_implementation_guide.md @@ -0,0 +1,1818 @@ +# Long-Term Search Implementation Guide +## Complete Step-by-Step Instructions with Scripts + +**Document Version:** 1.1 +**Date:** September 02, 2026 +**Status:** Implemented — Dual-Engine Blended (verified live `search-00037-xas` 2026-09-02; `0 Untitled` across github/medley/screenshots matrix) + +--- + +## Table of Contents + +1. [Overview](#overview) +2. [Prerequisites](#prerequisites) +3. [Phase 1: Create Unified Search (Data Store + Engine)](#phase-1-create-unified-search-data-store--engine) +4. [Phase 2: Update Cloud Function](#phase-2-update-cloud-function) +5. [Phase 3: Deploy & Test](#phase-3-deploy--test) +6. [Phase 4: Set Up Automation](#phase-4-set-up-automation) +7. [Testing Checklist](#testing-checklist) +8. [Rollback Procedures](#rollback-procedures) +9. [Monitoring & Troubleshooting](#monitoring--troubleshooting) +10. [Phase 7: Cleanup — Pare to In-Use Resources](#phase-7-cleanup--pare-interlispsearch-to-in-use-resources-only) +11. [Appendix: Script Reference](#appendix-script-reference) + +--- + +## Overview + +This guide walks you through implementing the unified data store architecture with automated refresh pipeline. + +### What You're Building — Implemented + +``` +interlisp-web-sites_1741606671710 (proven) interlisp-github-v2 (NO_CONTENT, branches/0) + PUBLIC_WEBSITE *.interlisp.org/* SUCCEEDED structured GitHub issues/PRs/markdown + \ / + \ interlisp-website-only interlisp-github-only + \ [interlisp-web-sites_…] [interlisp-github-v2] (fallback engine + \ \ / interlisp-search-unified + \ \--→ Cloud Function search (dual-fetch blended) ←--/ + ↓ WEBSITE_ENGINE_ID + GITHUB_ENGINE_ID + website-primary (1) → website-secondary (2) → github (3) + parallel fetch 20 each, blended 70/30 (50/50 for github|issue|pr|discussion) + dedup by url, priority→semanticRelevanceScore, citations sorted + title: derived.title || structData.title (schema retrievable:true) + ↓ + Cloud Scheduler (automation, local one-off until isolated deploy) + · Weekly GitHub JSONL → interlisp-github-v2 branches/0 (source/priority/last_indexed) + · Website crawl managed by existing *.interlisp.org targetSites (no recreation) +``` + +Fallback `interlisp-search-unified` (`[interlisp-web-sites_1741606671710, interlisp-github-v2]`) retained; fresh `interlisp-search-unified` website store (`interlisp.org/*` + `files.interlisp.org/*`) `SUCCEEDED` but not used for serving (returned only `files` for `history`/`Medley`). + +### Timeline + +- **Phase 1:** Data store setup (2-4 hours) +- **Phase 2:** Cloud Function updates (1-2 hours) +- **Phase 3:** Testing & deployment (1-2 hours) +- **Phase 4:** Automation setup (2-3 hours) + +**Total:** 6-11 hours + +--- + +## Prerequisites + +Before starting, verify you have: + +- ✅ gcloud CLI installed and authenticated +- ✅ Access to `interlispsearch` GCP project +- ✅ Service account `vertex-search-sa@interlispsearch.iam.gserviceaccount.com` exists +- ✅ Cloud Function `search` deployed and working +- ✅ `interlisp-search-v3` data store exists (will clone from this) +- ✅ GitHub token (for reimport function) +- ✅ Node.js 24+ (for Cloud Function testing) + +**Verify access:** + +```bash +gcloud config set project interlispsearch +gcloud auth application-default login + +# Test project access +gcloud projects describe interlispsearch + +createTime: '2025-03-09T12:06:51.391531Z' +lifecycleState: ACTIVE +name: InterlispSearch +parent: + id: '513691321000' + type: organization +projectId: interlispsearch +projectNumber: '191169864763' + +# Check existing data stores — Discovery Engine has NO gcloud CLI (REST API only) +# Use curl with Discovery Engine API instead: +TOKEN=$(gcloud auth application-default print-access-token) +curl -s -H "Authorization: Bearer $TOKEN" \ + -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores" \ + | python3 -m json.tool + +# We need to prune this list -- which data stores are no longer in use? And, which ones become obsolete +# when this work is done? + "dataStores": [ + { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/interlisp-document-bucket_1741737147622", + "displayName": "interlisp document bucket", + "industryVertical": "GENERIC", + "createTime": "2025-03-11T23:52:48.241959Z", + "solutionTypes": [ + "SOLUTION_TYPE_SEARCH" + ], + "contentConfig": "CONTENT_REQUIRED", + "defaultSchemaId": "default_schema", + "billingEstimation": { + "unstructuredDataSize": "4512314", + "unstructuredDataUpdateTime": "2026-08-28T05:04:14.419426546Z" + }, + "documentProcessingConfig": { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/interlisp-document-bucket_1741737147622/documentProcessingConfig", + "defaultParsingConfig": { + "digitalParsingConfig": {} + } + }, + "servingConfigDataStore": {} + }, + { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/interlisp-github", + "displayName": "Interlisp GitHub Content", + "industryVertical": "GENERIC", + "createTime": "2026-05-05T03:59:14.530998Z", + "solutionTypes": [ + "SOLUTION_TYPE_SEARCH" + ], + "contentConfig": "CONTENT_REQUIRED", + "defaultSchemaId": "default_schema", + "documentProcessingConfig": { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/interlisp-github/documentProcessingConfig", + "defaultParsingConfig": { + "digitalParsingConfig": {} + } + }, + "servingConfigDataStore": {} + }, + { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/interlisp-github-v2", + "displayName": "Interlisp GitHub Content", + "industryVertical": "GENERIC", + "createTime": "2026-05-13T03:30:54.827835Z", + "solutionTypes": [ + "SOLUTION_TYPE_SEARCH" + ], + "contentConfig": "NO_CONTENT", + "defaultSchemaId": "default_schema", + "billingEstimation": { + "structuredDataSize": "5107743", + "structuredDataUpdateTime": "2026-08-28T08:40:17.553964358Z" + }, + "servingConfigDataStore": {}, + "naturalLanguageQueryUnderstandingConfig": { + "mode": "ENABLED" + } + }, + { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/interlisp-site-search", + "displayName": "Interlisp Site Search", + "industryVertical": "GENERIC", + "createTime": "2026-04-29T11:34:57.642327Z", + "solutionTypes": [ + "SOLUTION_TYPE_SEARCH" + ], + "contentConfig": "PUBLIC_WEBSITE", + "defaultSchemaId": "default_schema", + "advancedSiteSearchConfig": { + "disableAutomaticRefresh": false + }, + "billingEstimation": { + "websiteDataSize": "1224192000", + "websiteDataUpdateTime": "2026-08-28T10:10:53.793689149Z" + }, + "servingConfigDataStore": {} + }, + { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/interlisp-web-sites_1741606671710", + "displayName": "Interlisp Web Sites", + "industryVertical": "GENERIC", + "createTime": "2025-03-10T11:40:25.475329Z", + "solutionTypes": [ + "SOLUTION_TYPE_SEARCH" + ], + "contentConfig": "PUBLIC_WEBSITE", + "defaultSchemaId": "default_schema", + "advancedSiteSearchConfig": { + "disableInitialIndex": true, + "disableAutomaticRefresh": true + }, + "billingEstimation": { + "websiteDataSize": "407040000", + "websiteDataUpdateTime": "2026-08-28T10:10:53.793689149Z" + }, + "documentProcessingConfig": { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/interlisp-web-sites_1741606671710/documentProcessingConfig", + "chunkingConfig": { + "layoutBasedChunkingConfig": { + "chunkSize": 500 + } + }, + "defaultParsingConfig": { + "layoutParsingConfig": {} + } + }, + "servingConfigDataStore": {}, + "naturalLanguageQueryUnderstandingConfig": {} + }, + { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/lispcore_1742267831416", + "displayName": "LispCore", + "industryVertical": "GENERIC", + "createTime": "2025-03-18T03:17:18.742179Z", + "solutionTypes": [ + "SOLUTION_TYPE_SEARCH" + ], + "contentConfig": "GOOGLE_WORKSPACE", + "aclEnabled": true, + "workspaceConfig": { + "type": "GOOGLE_GROUPS" + }, + "servingConfigDataStore": {} + }, + { + "name": "projects/191169864763/locations/global/collections/default_collection/dataStores/medley-interlisp-core_1741735842821", + "displayName": "Medley Interlisp Core", + "industryVertical": "GENERIC", + "createTime": "2025-03-11T23:31:01.436125Z", + "solutionTypes": [ + "SOLUTION_TYPE_SEARCH" + ], + "contentConfig": "GOOGLE_WORKSPACE", + "aclEnabled": true, + "workspaceConfig": { + "type": "GOOGLE_GROUPS" + }, + "servingConfigDataStore": {} + } + ] +} + +``` + +--- + +## Phase 1: Create Unified Search (Data Store + Engine) + +Production unified search uses two focused engines blended in the Cloud Function: + +* `interlisp-website-only` (`[interlisp-web-sites_1741606671710]` `PUBLIC_WEBSITE`, `GENERIC`, `*.interlisp.org/*` `SUCCEEDED` — proven to return `https://interlisp.org/...` for `history`/`software`/`Medley` where `interlisp-search-unified` returned only `files`/`online`/`primer`) +* `interlisp-github-v2` (`NO_CONTENT`, `GENERIC`) — structured GitHub issues / PRs / markdown via `interlisp-github-only` (`[interlisp-github-v2]`) +* Fallback `interlisp-search-unified` (`[interlisp-web-sites_1741606671710, interlisp-github-v2]`) retained for single-engine callers; the freshly created `interlisp-search-unified` website store (`PUBLIC_WEBSITE` `interlisp.org/*` + `files.interlisp.org/*`) is retained but not used for serving (its crawl returned only `files`). + +All Discovery Engine operations use the REST API (`https://discoveryengine.googleapis.com/v1/...` with `x-goog-user-project`); there is no `gcloud discoveryengine` group (SDK 568.0.0). + +### Step 1.1: Inventory Existing Resources + +Document current state before creating new resources: + +```bash +TOKEN=$(gcloud auth application-default print-access-token) + +# All data stores and engines (for reference) +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores?pageSize=100" \ + | python3 -m json.tool > /tmp/datastores_backup.json + +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines?pageSize=100" \ + | python3 -m json.tool > /tmp/engines_backup.json + +# Reference engine — shows expected multi-store pattern and industryVertical +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines/interlisp-search-v3" \ + | python3 -m json.tool > /tmp/interlisp-search-v3.json + +echo "Backups saved to /tmp/datastores_backup.json, /tmp/engines_backup.json, /tmp/interlisp-search-v3.json" +``` + +### Step 1.2: Create the Unified Website Data Store — Retained, Not Used for Serving (its crawl returned only `files` for `history`/`Medley`) + +> **Implemented note (2026-09-02):** This fresh `interlisp-search-unified` `PUBLIC_WEBSITE` store (`interlisp.org/*` + `files.interlisp.org/*`) is `SUCCEEDED` but not used for serving; proven `interlisp-web-sites_1741606671710` (`*.interlisp.org/*` `SUCCEEDED`) is primary. Steps 1.2–1.5 retained for inventory/reproducibility; Phase 1.7 fallback engine `interlisp-search-unified` (`[interlisp-web-sites_1741606671710, interlisp-github-v2]`) is the kept unified engine. + +### Step 1.2 (historical): Create the Unified Website Data Store + +```bash +#!/bin/bash +# File: create-unified-datastore.sh +PROJECT_ID="interlispsearch" +DATA_STORE_ID="interlisp-search-unified" +DISPLAY_NAME="Interlisp Unified Search" +LOCATION="global" +TOKEN=$(gcloud auth application-default print-access-token) + +curl -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores?dataStoreId=$DATA_STORE_ID" \ + -d '{ + "displayName": "'"$DISPLAY_NAME"'", + "industryVertical": "GENERIC", + "solutionTypes": ["SOLUTION_TYPE_SEARCH"], + "contentConfig": "PUBLIC_WEBSITE" + }' +sleep 5 +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores/$DATA_STORE_ID" \ + | python3 -m json.tool +``` + +```bash +chmod +x create-unified-datastore.sh && ./create-unified-datastore.sh +``` + +### Step 1.3: Add Target Sites + +```bash +#!/bin/bash +# File: add-target-sites.sh +PROJECT_ID="interlispsearch" +DATA_STORE_ID="interlisp-search-unified" +LOCATION="global" +TOKEN=$(gcloud auth application-default print-access-token) + +for pattern in "interlisp.org/*" "files.interlisp.org/*"; do + echo "Adding target site: $pattern" + curl -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores/$DATA_STORE_ID/siteSearchEngine/targetSites" \ + -d "{\"providedUriPattern\": \"$pattern\", \"type\": \"INCLUDE\", \"exactMatch\": false}" + sleep 2 +done + +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores/$DATA_STORE_ID/siteSearchEngine/targetSites" \ + | python3 -m json.tool +``` + +```bash +chmod +x add-target-sites.sh && ./add-target-sites.sh +``` + +### Step 1.4: Upgrade to Advanced Indexing (Console) + +1. Cloud Console → **AI Applications → Data Stores → `interlisp-search-unified` → Data tab** +2. For each target site, click **Upgrade to Advanced** +3. Wait 4–8 hours for indexing to reach `SUCCEEDED` + +There is no REST/`gcloud` API for this step. + +### Step 1.5: Verify Indexing Status + +```bash +#!/bin/bash +# File: check-indexing-status.sh +PROJECT_ID="interlispsearch" +DATA_STORE_ID="interlisp-search-unified" +LOCATION="global" +TOKEN=$(gcloud auth application-default print-access-token) +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores/$DATA_STORE_ID/siteSearchEngine/targetSites" \ + | python3 -c " +import sys, json +data = json.load(sys.stdin) +for site in data.get('targetSites', []): + print(f\"{site.get('providedUriPattern')}: {site.get('indexingStatus','UNKNOWN')}\") +" +``` + +```bash +chmod +x check-indexing-status.sh && ./check-indexing-status.sh +# watch -n 1800 ./check-indexing-status.sh # during initial crawl +``` + +### Step 1.6: Initial GitHub Import into `interlisp-github-v2` + +The website crawl (Steps 1.2–1.5) does not include GitHub. Import structured GitHub content into the `NO_CONTENT` store `interlisp-github-v2` before creating the unified engine so ranking can be validated in Phase 2. + +Documents are stored as `Document.structData` (not top-level `title`/`url`); `document.id` must match `^[a-zA-Z0-9-_]+$` (sanitize `.` → `-`) and use `branches/0` (not `branches/default`). Metadata `source`/`priority`/`last_indexed` is additive to existing `repo`/`type`/`created_date`. + +**Option A — via Cloud Function (recommended for repeatability):** + +```bash +export GITHUB_TOKEN=ghp_... # from https://github.com/settings/tokens (repo read) + +# Pub/Sub topic must exist before deploy +gcloud pubsub topics create reimport-github --project=interlispsearch 2>/dev/null || true + +# Deploy from repo root (or --source=. if inside search-function/) +gcloud functions deploy github-reimport \ + --gen2 --runtime=nodejs24 --region=us-central1 \ + --source=./search-function --entry-point=reimportGithub --trigger-topic=reimport-github \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --set-env-vars GITHUB_TOKEN=$GITHUB_TOKEN --memory=512Mi --timeout=300s \ + --project=interlispsearch + +# Trigger one initial import +gcloud pubsub topics publish reimport-github --message='{"action":"reimport","reason":"initial-seed-unified"}' --project=interlispsearch +gcloud functions logs read github-reimport --gen2 --region=us-central1 --limit=50 --project=interlispsearch + +# Verify import operation +TOKEN=$(gcloud auth application-default print-access-token) +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores/interlisp-github-v2/branches/0/documents?pageSize=1" \ + | python3 -m json.tool +``` + +**Option B — local one-off (no function deploy):** + +```bash +cd search-function && npm install +# Writes /tmp/github-initial-*.jsonl, uploads to gs://interlispsearch-search-imports/, POST .../branches/0/documents:import +node github-initial-import.js # anonymous GitHub API (60 req/h) or with GITHUB_TOKEN +GITHUB_TOKEN=$GITHUB_TOKEN node github-initial-import.js # authenticated (higher quota) +# Dry run only: +node github-initial-import.js --dry-run +``` + +Both options produce JSONL lines `{"id":"github-issue-medley-123","structData":{"id":"...","title":"...","url":"...","content":"...","source":"github","priority":3,...}}`, sanitize IDs (`Interlisp.github.io` → `Interlisp-github-io`), upload to `gs://interlispsearch-search-imports/github-initial-*.jsonl`, and `POST .../branches/0/documents:import`. + +Verify `successCount == totalCount` in the returned `operations/import-documents-*` (e.g. `successCount: 350, done: true`). + +### Step 1.7: Create Engines (two focused + one unified fallback) + +`interlisp-web-sites_1741606671710` (`PUBLIC_WEBSITE` `*.interlisp.org/*` `SUCCEEDED`) is the proven website crawl — `interlisp-search-unified` website store created in 1.2 returned only `files`/`online`/`primer` for `history`/`interlisp` and is retained but not used for serving. `industryVertical: GENERIC` is required for multi-store engines. + +```bash +TOKEN=$(gcloud auth application-default print-access-token) + +# Website-only (primary, used for dual fetch) +curl -X POST -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines?engineId=interlisp-website-only" \ + -d '{ + "displayName": "Interlisp Website Only", + "dataStoreIds": ["interlisp-web-sites_1741606671710"], + "solutionType": "SOLUTION_TYPE_SEARCH", + "industryVertical": "GENERIC", + "searchEngineConfig": {"searchTier":"SEARCH_TIER_ENTERPRISE","searchAddOns":["SEARCH_ADD_ON_LLM"]} + }' | python3 -m json.tool + +# GitHub-only (secondary) +curl -X POST -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines?engineId=interlisp-github-only" \ + -d '{ + "displayName": "Interlisp GitHub Only", + "dataStoreIds": ["interlisp-github-v2"], + "solutionType": "SOLUTION_TYPE_SEARCH", + "industryVertical": "GENERIC", + "searchEngineConfig": {"searchTier":"SEARCH_TIER_ENTERPRISE","searchAddOns":["SEARCH_ADD_ON_LLM"]} + }' | python3 -m json.tool + +# Unified fallback (website + GitHub) +curl -X POST -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines?engineId=interlisp-search-unified" \ + -d '{ + "displayName": "Interlisp Unified Search", + "dataStoreIds": ["interlisp-web-sites_1741606671710","interlisp-github-v2"], + "solutionType": "SOLUTION_TYPE_SEARCH", + "industryVertical": "GENERIC", + "searchEngineConfig": {"searchTier":"SEARCH_TIER_ENTERPRISE","searchAddOns":["SEARCH_ADD_ON_LLM"]} + }' | python3 -m json.tool + +# Verify +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines/interlisp-website-only" | python3 -m json.tool +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines/interlisp-search-unified" | python3 -m json.tool +# Phase 2 uses WEBSITE_ENGINE_ID=interlisp-website-only, GITHUB_ENGINE_ID=interlisp-github-only (fallback ENGINE_ID=interlisp-search-unified) +``` + +**Phase 1 complete when:** `GET .../engines/interlisp-website-only` and `interlisp-github-only` exist, `GET .../engines/interlisp-search-unified` returns `["interlisp-web-sites_1741606671710","interlisp-github-v2"]`, and GitHub import `successCount == totalCount` (350). + +--- + +## Phase 2: Update Cloud Function + +### Step 2.1: Create Updated `index.js` + +This implements dual-engine blended ranking: `website-primary` (`interlisp.org`/`www.interlisp.org` →1) over `website-secondary` (`*.interlisp.org` →2) over `github` →3, with guaranteed GitHub representation (70/30, 50/50 for GitHub queries). + +**File: `search-function/index.js`** (deployed as `search-00037-xas` 2026-09-02 with `WEBSITE_ENGINE_ID=interlisp-website-only`, `GITHUB_ENGINE_ID=interlisp-github-only`; verified `0 Untitled`) + +```javascript +'use strict'; +const { GoogleAuth } = require('google-auth-library'); +const { isRateLimited } = require('./rateLimiter'); +const PROJECT_ID = process.env.PROJECT_ID; +const WEBSITE_ENGINE_ID = process.env.WEBSITE_ENGINE_ID || process.env.ENGINE_ID; +const GITHUB_ENGINE_ID = process.env.GITHUB_ENGINE_ID || null; +const LOCATION = 'global'; +const auth = new GoogleAuth({ scopes: ['https://www.googleapis.com/auth/cloud-platform'] }); +exports.search = async (req, res) => { + // ... CORS + rate limiting (see repo) ... + const query = req.query.q || req.body?.q || ''; + const pageSize = parseInt(req.query.pageSize) || 10; + const stripHtml = str => str ? str.replace(/<[^>]*>/g, '') : null; + const getSource = (url) => { + if (!url) return 'unknown'; + try { + const u = new URL(url); const host = u.hostname.toLowerCase(); + if (host === 'www.interlisp.org' || host === 'interlisp.org') return 'website-primary'; + if (host.endsWith('.interlisp.org')) return 'website-secondary'; + if (host.includes('github.com') || url.includes('github.com')) return 'github'; + return 'other'; + } catch (_) { return url.includes('github.com') ? 'github' : url.includes('interlisp.org') ? 'website-secondary' : 'other'; } + }; + const sourcePriorityMap = { 'website-primary':1,'website-secondary':2,'github':3,'other':4,'unknown':999 }; + const fetchPageSize = 20; + const searchEngine = async (engineId, withSummary) => { + const endpoint = `https://discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/collections/default_collection/engines/${engineId}/servingConfigs/default_config:search`; + const body = { query, pageSize: fetchPageSize, contentSearchSpec: { snippetSpec:{returnSnippet:true}, extractiveContentSpec:{maxExtractiveAnswerCount:3} } }; + if (withSummary) body.contentSearchSpec.summarySpec = { summaryResultCount:5, includeCitations:true, useSemanticChunks:true, languageCode:'en-US', modelPromptSpec:{preamble:buildPreamble(context)}, modelSpec:{version:'stable'} }; + const resp = await fetch(endpoint, { method:'POST', headers:{'Authorization':`Bearer ${(await (await auth.getClient()).getAccessToken()).token}`,'Content-Type':'application/json','x-goog-user-project':PROJECT_ID}, body:JSON.stringify(body) }); + if (!resp.ok) throw new Error(`Vertex API error ${resp.status} on ${engineId}: ${await resp.text()}`); + return resp.json(); + }; + let [websiteData, githubData] = GITHUB_ENGINE_ID ? await Promise.all([searchEngine(WEBSITE_ENGINE_ID,true), searchEngine(GITHUB_ENGINE_ID,false)]) : [await searchEngine(WEBSITE_ENGINE_ID,true), null]; + // Blended merge: guarantee GitHub despite website priority + const isGithubQuery = /github|issue|pull\s*request|\bpr\b|discussion/i.test(query); + const websiteTake = isGithubQuery ? Math.ceil(pageSize*0.5) : Math.ceil(pageSize*0.7); + const githubTake = pageSize - websiteTake; + const toItems = (data, hint) => (data?.results||[]).map(r=>{ const d=r.document?.derivedStructData, s=r.document?.structData, url=d?.link||d?.url||s?.url||s?.link||null, src=hint||getSource(url); return {raw:r, url, source:src, priority:sourcePriorityMap[src]||999, derived:d, structData:s, score:r.retrievalSignals?.semanticRelevanceScore||0}; }).filter(i=>i.url); + let websiteItems = toItems(websiteData,null).sort((a,b)=>a.priority-b.priority||b.score-a.score); + let githubItems = toItems(githubData,'github').sort((a,b)=>b.score-a.score); + const seen=new Set(); websiteItems=websiteItems.filter(i=>!seen.has(i.url)&&seen.add(i.url)); githubItems=githubItems.filter(i=>!seen.has(i.url)&&seen.add(i.url)); + let allResults = [...websiteItems.slice(0,websiteTake), ...githubItems.slice(0,githubTake)]; + if (allResults.length < pageSize) { const rem=[...websiteItems.slice(websiteTake),...githubItems.slice(githubTake)].sort((a,b)=>a.priority-b.priority||b.score-a.score); allResults=[...allResults,...rem].slice(0,pageSize); } + allResults.sort((a,b)=>a.priority-b.priority||b.score-a.score); + const docIdToUrl={}; [...websiteItems,...githubItems].forEach(i=>{ if(i.raw.document?.id) docIdToUrl[i.raw.document.id]=i.url; }); + let references = (websiteData?.summary?.summaryWithMetadata?.references||[]).map(ref=>{ const docId=ref.document?.split('/').pop(), uri=docIdToUrl[docId]||null, src=getSource(uri); return {title:ref.title,uri,docId,source:src,priority:sourcePriorityMap[src]||999}; }).sort((a,b)=>a.priority-b.priority); + const results = allResults.map(i=>({ id:i.raw.document?.id, title:i.derived?.title||i.structData?.title||null, url:i.url, snippet:stripHtml(i.derived?.snippets?.[0]?.snippet||i.structData?.content?.slice(0,300)||null), source:i.source, priority:i.priority, section:i.url.replace(/^https:\/\/[^\/]+\//,'').split('/')?.[0]||'', type:i.structData?.type||null, repo:i.structData?.repo||null, state:i.structData?.state||null })).slice(0,pageSize); + res.json({ summary: (websiteData?.summary?.summaryText ? {summaryText:websiteData.summary.summaryText, citations:references} : null), results }); +}; +function buildPreamble(context) { + const base = `You are a search assistant for Interlisp.org. ... Citation Priorities: 1. www.interlisp.org/interlisp.org →2. *.interlisp.org (files/primer/online) →3. GitHub (issues/PRs/discussions) →4. source files. When both cover topic, cite www. ... Answer in Markdown only.`; + return context ? `${base}\n\n## Current Context\nThe user is browsing the "${context}" section of www.interlisp.org — prioritize results from that section.` : base; +} +``` + +### Step 2.2: Test Cloud Function Locally (Optional) + +```bash +# Copy updated code to search-function directory +cp index.js search-function/index.js + +# Run npm install to ensure dependencies are up to date +cd search-function +npm install + +# Test with local function emulator (if you have Google Cloud Functions emulator) +# Or manually test by reviewing the code for syntax errors +node -c index.js # Check syntax +``` + +--- + +## Phase 3: Deploy & Test + +### Step 3.1: Deploy Updated Cloud Function (dual-engine blended) + +Deployed as `search-00037-xas` (2026-09-02) with two engines for guaranteed website primary + GitHub secondary (70/30, 50/50 for GitHub queries); `0 Untitled` verified. + +```bash +#!/bin/bash +# File: deploy-search-function.sh +PROJECT_ID="interlispsearch" +REGION="us-central1" +WEBSITE_ENGINE_ID="interlisp-website-only" # [interlisp-web-sites_1741606671710] — proven www history/medley +GITHUB_ENGINE_ID="interlisp-github-only" # [interlisp-github-v2] — issues/PRs/markdown +# Verify engines exist (REST, no gcloud discoveryengine): +# TOKEN=$(gcloud auth application-default print-access-token); curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines" | python3 -m json.tool | grep -E '"name"|interlisp-website-only|interlisp-github-only' + +echo "Deploying search function to $REGION..." +echo "WEBSITE_ENGINE_ID=$WEBSITE_ENGINE_ID GITHUB_ENGINE_ID=$GITHUB_ENGINE_ID" + +gcloud functions deploy search \ + --gen2 \ + --runtime=nodejs24 \ + --region=$REGION \ + --source=. \ + --entry-point=search \ + --trigger-http \ + --allow-unauthenticated \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --set-env-vars PROJECT_ID=$PROJECT_ID,WEBSITE_ENGINE_ID=$WEBSITE_ENGINE_ID,GITHUB_ENGINE_ID=$GITHUB_ENGINE_ID,ENGINE_ID=$WEBSITE_ENGINE_ID \ + --memory=256Mi \ + --timeout=30s \ + --project=$PROJECT_ID + +echo "Deployed: https://$REGION-$PROJECT_ID.cloudfunctions.net/search" +echo "Env: WEBSITE_ENGINE_ID=$WEBSITE_ENGINE_ID GITHUB_ENGINE_ID=$GITHUB_ENGINE_ID" +``` + +**Then run:** + +```bash +chmod +x deploy-search-function.sh +./deploy-search-function.sh +``` + +### Step 3.2: Verify Deployment + +```bash +#!/bin/bash +# File: verify-deployment.sh + +PROJECT_ID="interlispsearch" +REGION="us-central1" + +echo "Verifying Cloud Function deployment..." +echo "" + +gcloud functions describe search \ + --gen2 \ + --region=$REGION \ + --project=$PROJECT_ID \ + --format=json | python3 -m json.tool + +echo "" +echo "Function URL: https://$REGION-$PROJECT_ID.cloudfunctions.net/search" +echo "" +echo "Testing function with sample query..." +curl -s "https://$REGION-$PROJECT_ID.cloudfunctions.net/search?q=interlisp&pageSize=5" \ + | python3 -m json.tool | head -50 +``` + +**Run it:** + +```bash +chmod +x verify-deployment.sh +./verify-deployment.sh +``` + +### Step 3.3: Test Citation Preference + +```bash +#!/bin/bash +# File: test-citations.sh + +FUNCTION_URL="https://us-central1-interlispsearch.cloudfunctions.net/search" + +echo "Testing citation preference..." +echo "" + +# Test 1: General query (should cite interlisp.org first) +echo "Test 1: Query 'interlisp' (should cite website first)" +echo "==========================================" +curl -s "$FUNCTION_URL?q=interlisp&pageSize=10" | python3 -c " +import sys, json +data = json.load(sys.stdin) +if data.get('summary') and data['summary'].get('citations'): + for i, citation in enumerate(data['summary']['citations'][:3], 1): + print(f\"[{i}] {citation.get('title', 'Untitled')}\") + print(f\" URI: {citation.get('uri', 'N/A')}\") + print() +else: + print('No citations found') +" +echo "" + +# Test 2: Check first result URL +echo "Test 2: First result should be from interlisp.org" +echo "==========================================" +curl -s "$FUNCTION_URL?q=medley&pageSize=5" | python3 -c " +import sys, json +data = json.load(sys.stdin) +if data.get('results'): + result = data['results'][0] + print(f\"Title: {result.get('title')}\") + print(f\"URL: {result.get('url')}\") + print(f\"Source: {result.get('source')}\") + print(f\"Priority: {result.get('priority')}\") +else: + print('No results found') +" +echo "" + +# Test 3: Verify GitHub content still appears +echo "Test 3: GitHub content should still appear in results" +echo "==========================================" +curl -s "$FUNCTION_URL?q=github&pageSize=20" | python3 -c " +import sys, json +data = json.load(sys.stdin) +if data.get('results'): + github_results = [r for r in data['results'] if 'github' in r.get('url', '')] + print(f\"Found {len(github_results)} GitHub results\") + for r in github_results[:2]: + print(f\" - {r.get('title')}\") + print(f\" {r.get('url')}\") +else: + print('No results found') +" +``` + +**Run it:** + +```bash +chmod +x test-citations.sh +./test-citations.sh +``` + +--- + +## Phase 4: Set Up Automation + +### Step 4.1: Create Website Recrawl Function + +**File: `search-function/trigger-website-recrawl.js`** (targets proven website store — not the fresh `interlisp-search-unified` crawl) + +```javascript +'use strict'; + +const { GoogleAuth } = require('google-auth-library'); + +const PROJECT_ID = 'interlispsearch'; +const LOCATION = 'global'; +const DATA_STORE_ID = 'interlisp-web-sites_1741606671710'; // proven *.interlisp.org/* SUCCEEDED; NOT interlisp-search-unified + +const auth = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'] +}); + +exports.recrawlWebsite = async (message, context) => { + console.log(`[${new Date().toISOString()}] Starting website recrawl...`); + + try { + const client = await auth.getClient(); + const token = await client.getAccessToken(); + + // Get target sites + const targetSitesUrl = `https://discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/collections/default_collection/dataStores/${DATA_STORE_ID}/siteSearchEngine/targetSites`; + + console.log(`Fetching target sites from: ${targetSitesUrl}`); + + const targetSitesResponse = await fetch(targetSitesUrl, { + method: 'GET', + headers: { + 'Authorization': `Bearer ${token.token}`, + 'x-goog-user-project': PROJECT_ID + } + }); + + if (!targetSitesResponse.ok) { + throw new Error(`Failed to fetch target sites: ${targetSitesResponse.statusText}`); + } + + const targetSitesData = await targetSitesResponse.json(); + const targetSites = targetSitesData.targetSites || []; + + console.log(`Found ${targetSites.length} target sites`); + + if (targetSites.length === 0) { + console.warn('No target sites found. Data store may not be initialized.'); + return; + } + + // Trigger recrawl for each target site + const recrawlResults = []; + + for (const site of targetSites) { + try { + console.log(`Triggering recrawl for: ${site.providedUriPattern}`); + + const recrawlUrl = `https://discoveryengine.googleapis.com/v1/${site.name}:recrawl`; + + const recrawlResponse = await fetch(recrawlUrl, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token.token}`, + 'Content-Type': 'application/json', + 'x-goog-user-project': PROJECT_ID + }, + body: JSON.stringify({}) + }); + + if (!recrawlResponse.ok) { + const errorText = await recrawlResponse.text(); + console.error(`Recrawl failed for ${site.providedUriPattern}: ${recrawlResponse.statusText}`); + console.error(`Error details: ${errorText}`); + recrawlResults.push({ + pattern: site.providedUriPattern, + status: 'FAILED', + error: recrawlResponse.statusText + }); + } else { + const recrawlData = await recrawlResponse.json(); + console.log(`Recrawl triggered for ${site.providedUriPattern}`); + console.log(`Operation: ${recrawlData.name}`); + recrawlResults.push({ + pattern: site.providedUriPattern, + status: 'SUCCESS', + operation: recrawlData.name + }); + } + } catch (siteError) { + console.error(`Error recrawling ${site.providedUriPattern}:`, siteError.message); + recrawlResults.push({ + pattern: site.providedUriPattern, + status: 'ERROR', + error: siteError.message + }); + } + } + + console.log(`[${new Date().toISOString()}] Website recrawl completed`); + console.log(JSON.stringify(recrawlResults, null, 2)); + + return recrawlResults; + + } catch (err) { + console.error(`[${new Date().toISOString()}] Recrawl error:`, err); + throw err; + } +}; +``` + +**Deploy recrawl function:** + +```bash +gcloud functions deploy trigger-website-recrawl \ + --gen2 \ + --runtime=nodejs24 \ + --region=us-central1 \ + --source=./search-function \ + --entry-point=recrawlWebsite \ + --trigger-topic=recrawl-website \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --memory=256Mi \ + --timeout=60s \ + --project=interlispsearch +``` + +### Step 4.2: Create GitHub Reimport Function — Implemented (future-proof schema, `branches/0`, sanitized ids) + +**File: `search-function/github-reimport.js`** (structured GitHub store `interlisp-github-v2`, NOT website store) + +```javascript +'use strict'; + +const { GoogleAuth } = require('google-auth-library'); +const { Storage } = require('@google-cloud/storage'); + +const PROJECT_ID = 'interlispsearch'; +const LOCATION = 'global'; +// Structured GitHub store — NOT the website store. Keeps website crawl separate +// and is consumed via interlisp-github-only engine (blended 70/30 in search function). +// Fallback engine interlisp-search-unified ([interlisp-web-sites_1741606671710, interlisp-github-v2]) retained. +const DATA_STORE_ID = 'interlisp-github-v2'; +const GITHUB_TOKEN = process.env.GITHUB_TOKEN; +const GITHUB_ORG = 'Interlisp'; +const GITHUB_REPOS = ['medley', 'maiko', 'Interlisp.github.io']; + +// Schema (future-proof, patched 2026-09-02): default_schema title/url/repo/type/state/author/number etc. all retrievable:true +// so search-function/index.js:193-195 title: derived.title || structData.title resolves and frontend never shows "Untitled". +// JSONL shape: {"id": sanitized, "structData":{id,title,url,content,source,priority,repo,type,state,number,created_at,last_indexed,…}} +// sanitized id: Interlisp.github.io → Interlisp-github-io, branches/0 not default + +// Shared-source re-export: search-function/index.js re-exports reimportGithub so same dir can serve both entry-points: +// try { exports.reimportGithub = require('./github-reimport').reimportGithub; } catch(_) {} + +const auth = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'] +}); + +const storage = new Storage({ projectId: PROJECT_ID }); + +exports.reimportGithub = async (message, context) => { + console.log(`[${new Date().toISOString()}] Starting GitHub reimport...`); + + try { + // 1. Fetch GitHub content + const gitHubDocuments = []; + + console.log('Fetching GitHub Issues...'); + const issues = await fetchGitHubIssues(); + gitHubDocuments.push(...issues); + console.log(`Fetched ${issues.length} GitHub issues`); + + console.log('Fetching GitHub Pull Requests...'); + const prs = await fetchGitHubPullRequests(); + gitHubDocuments.push(...prs); + console.log(`Fetched ${prs.length} GitHub pull requests`); + + console.log('Fetching GitHub Markdown files...'); + const markdownFiles = await fetchGitHubMarkdown(); + gitHubDocuments.push(...markdownFiles); + console.log(`Fetched ${markdownFiles.length} GitHub markdown files`); + + if (gitHubDocuments.length === 0) { + console.warn('No GitHub documents fetched. Check GitHub token and repo access.'); + return { status: 'NO_DOCUMENTS', count: 0 }; + } + + // 2. Convert to JSONL format — interlisp-github-v2 is NO_CONTENT: each line is + // {id, structData:{...}} with sanitized id ^[a-zA-Z0-9-_]+$ on branches/0 (see guide 1.6). Future-proof: all display fields retrievable. + const sanitizeId = (id) => id.replace(/\./g, '-').replace(/[^a-zA-Z0-9-_]/g, '-'); + console.log(`Converting ${gitHubDocuments.length} documents to JSONL (structData, branches/0, sanitized ids, future-proof schema)...`); + const jsonlLines = gitHubDocuments.map(doc => { + const rawId = doc.id; const sid = sanitizeId(rawId); + const m = doc.metadata || {}; + const structData = { id: sid, title: doc.title||'', url: doc.url||'', content: doc.content||'', source: m.source||'github', priority: m.priority||3, type: m.content_type||null, content_type: m.content_type||null, repo: m.repo||null, state: m.state||null, number: m.issue_number||m.pr_number||m.number||null, issue_number: m.issue_number||null, pr_number: m.pr_number||null, file: m.file||null, created_at: m.created_date||null, updated_at: m.updated_date||null, created_date: m.created_date||null, updated_date: m.updated_date||null, last_indexed: m.last_indexed||new Date().toISOString() }; + Object.keys(structData).forEach(k => structData[k]==null && delete structData[k]); + return JSON.stringify({ id: sid, structData }); + }); + const jsonlContent = jsonlLines.join('\n'); + + // 3. Upload JSONL to Cloud Storage + const bucketName = `${PROJECT_ID}-search-imports`; + const fileName = `github-import-${Date.now()}.jsonl`; + + console.log(`Creating Cloud Storage bucket if needed: ${bucketName}`); + const bucket = storage.bucket(bucketName); + + try { + await bucket.exists(); + } catch (e) { + console.log(`Creating bucket: ${bucketName}`); + await storage.createBucket(bucketName, { location: 'US' }); + } + + console.log(`Uploading JSONL to gs://${bucketName}/${fileName}`); + const file = bucket.file(fileName); + await file.save(jsonlContent); + + // 4. Call Vertex AI importDocuments API + const client = await auth.getClient(); + const token = await client.getAccessToken(); + + const importUrl = `https://discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/collections/default_collection/dataStores/${DATA_STORE_ID}/branches/0/documents:import`; // branches/0, not default + + console.log(`Calling Vertex AI import API: ${importUrl}`); + + const importResponse = await fetch(importUrl, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token.token}`, + 'Content-Type': 'application/json', + 'x-goog-user-project': PROJECT_ID + }, + body: JSON.stringify({ + gcsSource: { + inputUris: [`gs://${bucketName}/${fileName}`] + } + }) + }); + + if (!importResponse.ok) { + const errorText = await importResponse.text(); + throw new Error(`Import failed: ${importResponse.statusText} - ${errorText}`); + } + + const importData = await importResponse.json(); + console.log(`Import operation started: ${importData.name}`); + console.log(`[${new Date().toISOString()}] GitHub reimport completed successfully`); + + return { + status: 'SUCCESS', + documentsImported: gitHubDocuments.length, + operation: importData.name, + gcsLocation: `gs://${bucketName}/${fileName}` + }; + + } catch (err) { + console.error(`[${new Date().toISOString()}] GitHub reimport error:`, err); + throw err; + } +}; + +async function fetchGitHubIssues() { + const issues = []; + + for (const repo of GITHUB_REPOS) { + try { + const url = `https://api.github.com/repos/${GITHUB_ORG}/${repo}/issues?state=all&per_page=100`; + const response = await fetch(url, { + headers: { + 'Authorization': `token ${GITHUB_TOKEN}`, + 'Accept': 'application/vnd.github.v3+json' + } + }); + + if (!response.ok) { + console.warn(`Failed to fetch issues from ${GITHUB_ORG}/${repo}: ${response.statusText}`); + continue; + } + + const repoIssues = await response.json(); + + for (const issue of repoIssues) { + issues.push({ + id: `github-issue-${repo}-${issue.number}`, + title: issue.title, + url: issue.html_url, + content: issue.body || '', + metadata: { + source: 'github', + priority: 3, + content_type: 'issue', + repo: `${GITHUB_ORG}/${repo}`, + issue_number: issue.number, + state: issue.state, + created_date: issue.created_at, + updated_date: issue.updated_at, + last_indexed: new Date().toISOString() + } + }); + } + } catch (err) { + console.error(`Error fetching issues from ${GITHUB_ORG}/${repo}:`, err.message); + } + } + + return issues; +} + +async function fetchGitHubPullRequests() { + const prs = []; + + for (const repo of GITHUB_REPOS) { + try { + const url = `https://api.github.com/repos/${GITHUB_ORG}/${repo}/pulls?state=all&per_page=100`; + const response = await fetch(url, { + headers: { + 'Authorization': `token ${GITHUB_TOKEN}`, + 'Accept': 'application/vnd.github.v3+json' + } + }); + + if (!response.ok) { + console.warn(`Failed to fetch PRs from ${GITHUB_ORG}/${repo}: ${response.statusText}`); + continue; + } + + const repoPrs = await response.json(); + + for (const pr of repoPrs) { + prs.push({ + id: `github-pr-${repo}-${pr.number}`, + title: pr.title, + url: pr.html_url, + content: (pr.body || '') + '\n' + (pr.title || ''), + metadata: { + source: 'github', + priority: 3, + content_type: 'pull_request', + repo: `${GITHUB_ORG}/${repo}`, + pr_number: pr.number, + state: pr.state, + created_date: pr.created_at, + updated_date: pr.updated_at, + last_indexed: new Date().toISOString() + } + }); + } + } catch (err) { + console.error(`Error fetching PRs from ${GITHUB_ORG}/${repo}:`, err.message); + } + } + + return prs; +} + +async function fetchGitHubMarkdown() { + const markdownFiles = []; + + for (const repo of GITHUB_REPOS) { + try { + // Fetch README + const readmeUrl = `https://api.github.com/repos/${GITHUB_ORG}/${repo}/readme`; + const readmeResponse = await fetch(readmeUrl, { + headers: { + 'Authorization': `token ${GITHUB_TOKEN}`, + 'Accept': 'application/vnd.github.v3.raw' + } + }); + + if (readmeResponse.ok) { + const content = await readmeResponse.text(); + markdownFiles.push({ + id: `github-markdown-${repo}-README`, + title: `${repo} README`, + url: `https://github.com/${GITHUB_ORG}/${repo}/blob/main/README.md`, + content: content, + metadata: { + source: 'github', + priority: 3, + content_type: 'markdown', + repo: `${GITHUB_ORG}/${repo}`, + file: 'README.md', + last_indexed: new Date().toISOString() + } + }); + } + } catch (err) { + console.warn(`No README found in ${GITHUB_ORG}/${repo}`); + } + } + + return markdownFiles; +} +``` + +**Deploy GitHub reimport function:** + +First, create or get a GitHub token: + +```bash +# You can generate one at: https://github.com/settings/tokens +# Needs: repo (full), read:org permissions + +export GITHUB_TOKEN="your_github_token_here" + +gcloud functions deploy github-reimport \ + --gen2 \ + --runtime=nodejs24 \ + --region=us-central1 \ + --source=./search-function \ + --entry-point=reimportGithub \ + --trigger-topic=reimport-github \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --set-env-vars GITHUB_TOKEN=$GITHUB_TOKEN \ + --memory=512Mi \ + --timeout=300s \ + --project=interlispsearch +``` + +### Step 4.3: Create Cloud Scheduler Jobs + +**Daily Website Recrawl (2 AM UTC):** + +```bash +gcloud scheduler jobs create pubsub recrawl-website-daily \ + --location=us-central1 \ + --schedule="0 2 * * *" \ + --timezone="UTC" \ + --topic=recrawl-website \ + --message-body='{"action":"recrawl","timestamp":"$(date -u +%Y-%m-%dT%H:%M:%SZ)"}' \ + --project=interlispsearch +``` + +**Weekly GitHub Reimport (Monday 3 AM UTC):** + +```bash +gcloud scheduler jobs create pubsub reimport-github-weekly \ + --location=us-central1 \ + --schedule="0 3 * * 1" \ + --timezone="UTC" \ + --topic=reimport-github \ + --message-body='{"action":"reimport","timestamp":"$(date -u +%Y-%m-%dT%H:%M:%SZ)"}' \ + --project=interlispsearch +``` + +**Verify jobs created:** + +```bash +gcloud scheduler jobs list --location=us-central1 --project=interlispsearch +``` + +### Step 4.4: Test Automation (Optional) + +**Manually trigger recrawl:** + +```bash +gcloud scheduler jobs run recrawl-website-daily \ + --location=us-central1 \ + --project=interlispsearch +``` + +**Manually trigger GitHub reimport:** + +```bash +gcloud scheduler jobs run reimport-github-weekly \ + --location=us-central1 \ + --project=interlispsearch +``` + +**Check logs:** + +```bash +gcloud functions logs read trigger-website-recrawl \ + --gen2 \ + --region=us-central1 \ + --limit=20 \ + --project=interlispsearch + +gcloud functions logs read github-reimport \ + --gen2 \ + --region=us-central1 \ + --limit=20 \ + --project=interlispsearch +``` + +--- + +## Testing Checklist — Implemented (dual-engine blended, 0 Untitled verified 2026-09-02) + +Use this checklist to validate the implementation at each phase. + +### Before Deployment + +- [x] Proven website store `interlisp-web-sites_1741606671710` (`*.interlisp.org/*` `SUCCEEDED`) confirmed — fresh `interlisp-search-unified` website store (`interlisp.org/*`+`files`) also `SUCCEEDED` but not used for serving +- [x] GitHub store `interlisp-github-v2` schema patched future-proof (`title/url/repo/type/state/author/number` `retrievable:true`; `PATCH …/schemas/default_schema`) +- [x] Target sites `*.interlisp.org/*` verified `SUCCEEDED` (proven store) +- [x] GitHub import `branches/0` `structData` sanitized (`Interlisp.github.io`→`Interlisp-github-io`) `successCount 350/350` via local one-off `GITHUB_TOKEN=$(gh auth token) node /tmp/local_import_simple.js` → `gs://interlispsearch-search-imports/github-import-*.jsonl` +- [x] Engines `interlisp-website-only` `[interlisp-web-sites_…]` + `interlisp-github-only` `[interlisp-github-v2]` + fallback `interlisp-search-unified` `[both]` exist +- [x] Cloud Function `search-00037-xas` dual-fetch blended (70/30, 50/50 for `github|issue|pr|discussion`) with `website-primary:1 → website-secondary:2 → github:3`, dedup by `url`, `priority→semanticRelevanceScore` +- [x] No syntax errors (`node -c index.js`, `npm run lint` — 5 warnings allowed) + +### After Cloud Function Deployment — Verified 2026-09-02 + +**Test URL:** `https://us-central1-interlispsearch.cloudfunctions.net/search` (`search-00037-xas` `WEBSITE_ENGINE_ID=interlisp-website-only` `GITHUB_ENGINE_ID=interlisp-github-only`) + +- [x] **Health check:** `?q=test` returns results without errors +- [x] **Website prioritization:** First result is `website-primary` `interlisp.org` (not github.com) +- [x] **Citation preference:** First citation is `website-primary` `interlisp.org` (not github.com) +- [x] **GitHub still present (blended):** `?q=github` 5+5 (50/50), `?q=interlisp` 7+3 (70/30) — `github|issue|pr|discussion` triggers 50/50 +- [x] **No Untitled:** `github/medley/pull request/issue/screenshots/FONTSAVAILABLE` all `title` non-null (schema patch `retrievable:true` + `derived.title || structData.title`) +- [x] **Repo/type/state chips:** `Interlisp/medley` etc. visible (structData retrievable) +- [x] **Summary quality:** Summaries accurate with `www` preamble preference +- [x] **Response time:** Queries complete in <5 seconds (fetch 20 each, parallel) +- [x] **Error handling:** Errors return proper HTTP status codes + +**Sample test queries:** + +```bash +# Test 1: Basic query +curl -s "https://us-central1-interlispsearch.cloudfunctions.net/search?q=interlisp&pageSize=10" | jq '.results[0]' + +# Test 2: Citation preference +curl -s "https://us-central1-interlispsearch.cloudfunctions.net/search?q=medley&pageSize=10" | jq '.summary.citations[0]' + +# Test 3: GitHub content availability +curl -s "https://us-central1-interlispsearch.cloudfunctions.net/search?q=github&pageSize=20" | jq '.results | map(.url) | map(select(contains("github")))' + +# Test 4: Error handling (missing query) +curl -s "https://us-central1-interlispsearch.cloudfunctions.net/search?q=" | jq '.' + +# Test 5: Rate limiting (run 25 times quickly) +for i in {1..25}; do + STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://us-central1-interlispsearch.cloudfunctions.net/search?q=test") + echo "Request $i: $STATUS" +done +``` + +### After Automation Deployment + +- [ ] **Recrawl function deployed:** `trigger-website-recrawl` is active +- [ ] **GitHub reimport function deployed:** `github-reimport` is active +- [ ] **Cloud Scheduler jobs created:** Both jobs listed in scheduler +- [ ] **Manual trigger test:** Recrawl can be triggered manually and succeeds +- [ ] **Manual trigger test:** GitHub reimport can be triggered manually and succeeds +- [ ] **Function logs:** No errors in recrawl/reimport function logs +- [ ] **Scheduled execution:** Jobs execute at scheduled times + +**Check scheduled job status:** + +```bash +gcloud scheduler jobs describe recrawl-website-daily --location=us-central1 --project=interlispsearch +gcloud scheduler jobs describe reimport-github-weekly --location=us-central1 --project=interlispsearch +``` + +--- + +## Rollback Procedures + +If issues arise, use these procedures to roll back to a known good state. + +### Rollback Cloud Function to Previous Version + +**Option 1: Restore from Git** + +```bash +cd search-function + +# Restore previous index.js +git checkout HEAD~1 index.js + +# Redeploy +gcloud functions deploy search \ + --gen2 \ + --runtime=nodejs24 \ + --region=us-central1 \ + --source=. \ + --entry-point=search \ + --trigger-http \ + --allow-unauthenticated \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --set-env-vars PROJECT_ID=interlispsearch,ENGINE_ID=interlisp-org-search_1768860477660 \ + --memory=256Mi \ + --timeout=30s \ + --project=interlispsearch +``` + +**Option 2: Switch Engine Back to Previous Data Store** + +```bash +# Deploy current code but point to old engine +gcloud functions deploy search \ + --gen2 \ + --runtime=nodejs24 \ + --region=us-central1 \ + --source=./search-function \ + --entry-point=search \ + --trigger-http \ + --allow-unauthenticated \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --set-env-vars PROJECT_ID=interlispsearch,ENGINE_ID=interlisp-org-search_1768860477660 \ + --memory=256Mi \ + --timeout=30s \ + --project=interlispsearch +``` + +### Disable Automation (if causing issues) + +```bash +# Pause scheduler jobs +gcloud scheduler jobs pause recrawl-website-daily --location=us-central1 --project=interlispsearch +gcloud scheduler jobs pause reimport-github-weekly --location=us-central1 --project=interlispsearch + +# Resume when ready +gcloud scheduler jobs resume recrawl-website-daily --location=us-central1 --project=interlispsearch +gcloud scheduler jobs resume reimport-github-weekly --location=us-central1 --project=interlispsearch +``` + +### Delete and Recreate Automation Functions + +```bash +# Delete functions if broken +gcloud functions delete trigger-website-recrawl --gen2 --region=us-central1 --project=interlispsearch --quiet +gcloud functions delete github-reimport --gen2 --region=us-central1 --project=interlispsearch --quiet + +# Then redeploy with corrected code +# (See Phase 4 deployment steps) +``` + +--- + +## Monitoring & Troubleshooting + +### View Cloud Function Logs + +```bash +# Search function logs (real-time) +gcloud functions logs read search --gen2 --region=us-central1 --limit=50 --follow --project=interlispsearch + +# Recrawl function logs +gcloud functions logs read trigger-website-recrawl --gen2 --region=us-central1 --limit=50 --project=interlispsearch + +# GitHub reimport logs +gcloud functions logs read github-reimport --gen2 --region=us-central1 --limit=50 --project=interlispsearch +``` + +### Check Data Store Indexing Status + +```bash +#!/bin/bash +# File: check-datastore-status.sh + +PROJECT_ID="interlispsearch" +DATA_STORE_ID="interlisp-search-unified" +LOCATION="global" + +TOKEN=$(gcloud auth application-default print-access-token) + +curl -s -H "Authorization: Bearer $TOKEN" \ + -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores/$DATA_STORE_ID/siteSearchEngine/targetSites" \ + | python3 -c " +import sys, json +data = json.load(sys.stdin) +print('Data Store: interlisp-search-unified') +print('='*50) +if 'targetSites' in data: + for site in data['targetSites']: + print(f\"URI Pattern: {site.get('providedUriPattern')}\") + print(f\"Status: {site.get('indexingStatus', 'UNKNOWN')}\") + print(f\"Type: {site.get('type')}\") + if site.get('failureReasons'): + print(f\"Failures: {site['failureReasons']}\") + print() +else: + print('No target sites found') + print(json.dumps(data, indent=2)) +" +``` + +### Common Issues & Fixes + +#### Issue: "indexingStatus: FAILED" + +**Cause:** Website structure incompatible with crawler + +**Fix:** +1. Check if robots.txt is blocking crawler +2. Verify URL patterns are correct (no `https://` prefix needed) +3. Check GCP Console logs for specific errors +4. Retry upgrade to Advanced indexing + +#### Issue: "403 Forbidden" on API calls + +**Cause:** Service account lacks permissions + +**Fix:** +```bash +# Verify service account has required roles +gcloud projects get-iam-policy interlispsearch \ + --flatten="bindings[].members" \ + --format='table(bindings.role)' \ + --filter="bindings.members:vertex-search-sa*" +``` + +#### Issue: Cloud Function returns empty results + +**Cause:** ENGINE_ID points to data store that hasn't finished indexing + +**Fix:** +1. Wait for indexing to complete (check target sites status) +2. Verify ENGINE_ID matches actual engine ID +3. Test with sample query via GCP Console first + +#### Issue: GitHub documents not appearing + +**Cause:** GitHub reimport function failed or GitHub token expired + +**Fix:** +```bash +# Check recent logs +gcloud functions logs read github-reimport --gen2 --region=us-central1 --limit=30 --project=interlispsearch + +# Manually trigger to see detailed error +gcloud scheduler jobs run reimport-github-weekly --location=us-central1 --project=interlispsearch + +# Verify GitHub token is still valid +# Regenerate at: https://github.com/settings/tokens +``` + +### Monitoring Setup (Optional) + +Create Cloud Monitoring alerts for failures: + +```bash +# Create alert for Cloud Function errors +gcloud alpha monitoring policies create \ + --notification-channels=CHANNEL_ID \ + --display-name="Search Function Errors" \ + --condition-display-name="Search function error rate > 5%" \ + --condition-threshold-value=0.05 \ + --condition-threshold-duration=300s +``` + +--- + +## Appendix: Script Reference + +All scripts mentioned in this guide are collected here for easy reference. + +### Complete Deployment Checklist Script + +```bash +#!/bin/bash +# File: deployment-checklist.sh + +echo "===============================================" +echo "INTERLISP SEARCH - UNIFIED DATA STORE CHECKLIST" +echo "===============================================" +echo "" + +PROJECT_ID="interlispsearch" +DATA_STORE_ID="interlisp-search-unified" +LOCATION="global" +REGION="us-central1" + +# Check 1: Unified data store exists (REST API — no gcloud CLI) +echo "[1/8] Checking unified data store..." +TOKEN=$(gcloud auth application-default print-access-token 2>/dev/null) +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores/$DATA_STORE_ID") +if [ "$HTTP_CODE" = "200" ]; then + echo " ✓ Data store exists" +else + echo " ✗ Data store not found (HTTP $HTTP_CODE)" +fi +echo "" + +# Check 2: Target sites configured +echo "[2/8] Checking target sites..." +TOKEN=$(gcloud auth application-default print-access-token) +RESPONSE=$(curl -s -H "Authorization: Bearer $TOKEN" \ + -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores/$DATA_STORE_ID/siteSearchEngine/targetSites") +SITE_COUNT=$(echo "$RESPONSE" | python3 -c "import sys, json; data=json.load(sys.stdin); print(len(data.get('targetSites', [])))") +echo " Found $SITE_COUNT target sites" +echo "" + +# Check 3: Cloud Function deployed +echo "[3/8] Checking search Cloud Function..." +if gcloud functions describe search --gen2 --region=$REGION --project=$PROJECT_ID &>/dev/null; then + ENGINE_ID=$(gcloud functions describe search --gen2 --region=$REGION --project=$PROJECT_ID --format='value(environmentVariables.ENGINE_ID)') + echo " ✓ Function deployed" + echo " ENGINE_ID: $ENGINE_ID" +else + echo " ✗ Function not deployed" +fi +echo "" + +# Check 4: Recrawl function deployed +echo "[4/8] Checking website recrawl function..." +if gcloud functions describe trigger-website-recrawl --gen2 --region=$REGION --project=$PROJECT_ID &>/dev/null; then + echo " ✓ Function deployed" +else + echo " ✗ Function not deployed" +fi +echo "" + +# Check 5: GitHub reimport function deployed +echo "[5/8] Checking GitHub reimport function..." +if gcloud functions describe github-reimport --gen2 --region=$REGION --project=$PROJECT_ID &>/dev/null; then + echo " ✓ Function deployed" +else + echo " ✗ Function not deployed" +fi +echo "" + +# Check 6: Scheduler jobs configured +echo "[6/8] Checking Cloud Scheduler jobs..." +RECRAWL_EXISTS=$(gcloud scheduler jobs list --location=us-central1 --filter="name:recrawl-website-daily" --project=$PROJECT_ID --format='value(name)' 2>/dev/null) +REIMPORT_EXISTS=$(gcloud scheduler jobs list --location=us-central1 --filter="name:reimport-github-weekly" --project=$PROJECT_ID --format='value(name)' 2>/dev/null) + +[ -n "$RECRAWL_EXISTS" ] && echo " ✓ Recrawl job exists" || echo " ✗ Recrawl job missing" +[ -n "$REIMPORT_EXISTS" ] && echo " ✓ Reimport job exists" || echo " ✗ Reimport job missing" +echo "" + +# Check 7: Service account permissions +echo "[7/8] Checking service account permissions..." +SA_EMAIL="vertex-search-sa@interlispsearch.iam.gserviceaccount.com" +ROLES=$(gcloud projects get-iam-policy $PROJECT_ID \ + --flatten="bindings[].members" \ + --format='value(bindings.role)' \ + --filter="bindings.members:$SA_EMAIL" 2>/dev/null | wc -l) +echo " Service account has $ROLES roles assigned" +echo "" + +# Check 8: Test search function +echo "[8/8] Testing search function..." +FUNCTION_URL="https://$REGION-$PROJECT_ID.cloudfunctions.net/search" +HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$FUNCTION_URL?q=test&pageSize=5") +if [ "$HTTP_STATUS" = "200" ]; then + echo " ✓ Function responsive (HTTP $HTTP_STATUS)" +else + echo " ✗ Function returned HTTP $HTTP_STATUS" +fi +echo "" + +echo "===============================================" +echo "CHECKLIST COMPLETE" +echo "===============================================" +``` + +**Run it:** + +```bash +chmod +x deployment-checklist.sh +./deployment-checklist.sh +``` + +--- + +## Phase 7: Cleanup — Pare interlispsearch to In-Use Resources Only + +**Goal:** After dual-engine blended (`interlisp-website-only` + `interlisp-github-only` + fallback `interlisp-search-unified`) is validated in production (≥48h serving traffic, verified 2026-09-02 `search-00037-xas`), delete the “large collection of data stores” and any other stale resources so `interlispsearch` contains only the keep-list below. Run only after Phases 1–4 are complete and `0 Untitled` matrix passes. + +**Keep list (do NOT delete):** + +| Type | Name | +|---|---| +| Data store | `interlisp-web-sites_1741606671710` (proven website `*.interlisp.org/*` SUCCEEDED) | +| Data store | `interlisp-github-v2` (structured `NO_CONTENT` `branches/0` future-proof schema) | +| Data store | `interlisp-search-unified` (fresh website store `PUBLIC_WEBSITE` interlisp.org/*+files — retained, not used for serving) | +| Engine | `interlisp-website-only` (`[interlisp-web-sites_1741606671710]`) — primary | +| Engine | `interlisp-github-only` (`[interlisp-github-v2]`) — secondary | +| Engine | `interlisp-search-unified` (`[interlisp-web-sites_1741606671710, interlisp-github-v2]`) — fallback | +| Cloud Functions (Gen2, us-central1) | `search` (`search-00037-xas`), `github-reimport` (needs isolated source dir for Pub/Sub; re-export via `index.js`) | +| Cloud Scheduler (us-central1) | `reimport-github-weekly` (deferred weekly until isolated deploy; manual local one-off active), `recrawl-website-daily` if kept | +| Firestore | Database + `rate_limits` collection (TTL on `updatedAt`) | +| Cloud Storage | `interlispsearch-search-imports` bucket (prune old `github-import-*.jsonl` objects only) | +| IAM | `vertex-search-sa@` service account + org-policy overrides | + +Everything else — `interlisp-search-v3`, `interlisp-org-search_1768860477660`, `interlisp-site-search`, `interlisp-document-bucket_1741737147622`, `lispcore_…`, `medley-interlisp-core_…` if superseded, `interlisp-github` (old `CONTENT_REQUIRED`), test engines/stores, stale functions/jobs, old import objects — is a **delete candidate**. Keep `interlisp-github-v2` and `interlisp-web-sites_1741606671710` plus the three engines above. + +### Step 7.1: Inventory (read-only, save for review) + +```bash +#!/bin/bash +# File: inventory-interlispsearch.sh — run BEFORE any deletes +set -e +PROJECT_ID="interlispsearch" +LOCATION="global" +REGION="us-central1" +OUTDIR="/tmp/interlispsearch-inventory-$(date +%Y%m%d-%H%M%S)" +mkdir -p "$OUTDIR" +TOKEN=$(gcloud auth application-default print-access-token) + +echo "Inventory -> $OUTDIR" + +# 1. Data stores (REST — no gcloud CLI) +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores?pageSize=100" \ + | python3 -m json.tool > "$OUTDIR/datastores.json" +echo "dataStores -> $OUTDIR/datastores.json" +cat "$OUTDIR/datastores.json" | python3 -c "import json; d=json.load(open('$OUTDIR/datastores.json')); print(f\" {len(d.get('dataStores',[]))} data stores\")" + +# 2. Engines / search apps +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/engines?pageSize=100" \ + | python3 -m json.tool > "$OUTDIR/engines.json" +cat "$OUTDIR/engines.json" | python3 -c "import json; d=json.load(open('$OUTDIR/engines.json')); print(f\" {len(d.get('engines',[]))} engines\")" + +# 3. Target sites for each data store (helps decide what to keep) +python3 -c " +import json +d=json.load(open('$OUTDIR/datastores.json')) +for ds in d.get('dataStores',[]): + print(ds['name'].split('/')[-1], '—', ds.get('displayName','')) +" > "$OUTDIR/datastore-names.txt" +cat "$OUTDIR/datastore-names.txt" + +# 4. Cloud Functions +gcloud functions list --gen2 --region=$REGION --project=$PROJECT_ID --format=json > "$OUTDIR/functions.json" 2>&1 || true +python3 -c "import json; d=json.load(open('$OUTDIR/functions.json')); print(f\" {len(d)} functions\")" 2>/dev/null || echo " (functions list requires permissions)" + +# 5. Scheduler jobs +gcloud scheduler jobs list --location=$REGION --project=$PROJECT_ID --format=json > "$OUTDIR/scheduler.json" 2>&1 || true + +# 6. Storage buckets +gcloud storage ls --project=$PROJECT_ID > "$OUTDIR/storage-ls.txt" 2>&1 || gsutil ls -p $PROJECT_ID > "$OUTDIR/storage-ls.txt" 2>&1 || true +cat "$OUTDIR/storage-ls.txt" + +# 7. Firestore collections (console is authoritative) +echo "Firestore: check console -> Firestore -> Data, or: gcloud firestore databases describe --project=$PROJECT_ID" + +echo "" +echo "REVIEW $OUTDIR/datastores.json and $OUTDIR/engines.json" +echo "Mark KEEP vs DELETE before running Step 7.2. Commit $OUTDIR to git or share for review." +``` + +**Review gate:** Open `$OUTDIR/datastores.json` and `$OUTDIR/engines.json`, build a `KEEP = {interlisp-search-unified}` list, and get a second reviewer before any `DELETE`. + +### Step 7.2: Delete in Safe Order (engines → data stores → functions → scheduler → storage) + +> **Safety:** Each `DELETE` is irreversible. Re-run inventory after each sub-step and verify `curl "https://us-central1-interlispsearch.cloudfunctions.net/search?q=interlisp"` still returns `200`. + +```bash +#!/bin/bash +# File: cleanup-interlispsearch.sh — DESTRUCTIVE. Run only after inventory review. +set -e +PROJECT_ID="interlispsearch" +LOCATION="global" +REGION="us-central1" +TOKEN=$(gcloud auth application-default print-access-token) + +# KEEP — edit if your unified engine ID differs +KEEP_DATASTORE="interlisp-search-unified" + +# --- 1. Engines first (depend on data stores) --- +echo "=== 1. Deleting stale engines (keeping only *$KEEP_DATASTORE*) ===" +ENGINES_JSON=$(curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/engines?pageSize=100") +python3 -c " +import json, sys +d=json.loads('''$ENGINES_JSON'''.replace(\"'\", \"'\")) # placeholder +" 2>&1 | head +# Preferred: iterate with jq/python in a loop: +# For each engine where dataStoreIds != ["interlisp-search-unified"]: +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/engines" \ + | python3 -c " +import json, subprocess, shlex, sys +import json as j +data=json.load(sys.stdin) +for e in data.get('engines',[]): + name=e['name'] + ds_ids=e.get('dataStoreIds',[]) + eng_id=name.split('/')[-1] + if ds_ids != ['interlisp-search-unified']: + print(f'DELETE engine {eng_id} (dataStoreIds={ds_ids})') + # Uncomment to actually delete: + # subprocess.run(['curl','-X','DELETE','-H',f'Authorization: Bearer {TOKEN}','-H',f'x-goog-user-project: {PROJECT_ID}',f'https://discoveryengine.googleapis.com/v1/{name}'], check=False) + else: + print(f'KEEP engine {eng_id}') +" + +# --- 2. Data stores second (cascades to documents/targetSites) --- +echo "" +echo "=== 2. Deleting stale data stores (keeping $KEEP_DATASTORE) ===" +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ + "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores" \ + | python3 -c " +import json, sys +data=json.load(sys.stdin) +for ds in data.get('dataStores',[]): + ds_id=ds['name'].split('/')[-1] + if ds_id != 'interlisp-search-unified': + print(f'DELETE dataStore {ds_id} ({ds.get(\"displayName\",\"\")})') + # Uncomment to actually delete: + # import subprocess, os + # subprocess.run(['curl','-X','DELETE','-H',f'Authorization: Bearer {os.environ[\"TOKEN\"]}','-H','x-goog-user-project: interlispsearch',f'https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores/{ds_id}']) + else: + print(f'KEEP dataStore {ds_id}') +" +# Actual delete command for one data store (run after review): +# curl -X DELETE -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ +# "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores/interlisp-search-v3" +# curl -X DELETE -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: $PROJECT_ID" \ +# "https://discoveryengine.googleapis.com/v1/projects/$PROJECT_ID/locations/$LOCATION/collections/default_collection/dataStores/interlisp-org-search_1768860477660" + +# --- 3. Cloud Functions third (not in keep list) --- +echo "" +echo "=== 3. Stale Cloud Functions (review then delete) ===" +gcloud functions list --gen2 --region=$REGION --project=$PROJECT_ID --format="value(name)" 2>/dev/null | while read f; do + base=$(basename "$f") + case "$base" in search|trigger-website-recrawl|github-reimport) echo "KEEP function $base" ;; *) echo "DELETE function $base — run: gcloud functions delete $base --gen2 --region=$REGION --project=$PROJECT_ID" ;; esac +done + +# --- 4. Scheduler fourth --- +echo "" +echo "=== 4. Stale Scheduler jobs ===" +gcloud scheduler jobs list --location=$REGION --project=$PROJECT_ID --format="value(name)" 2>/dev/null | while read j; do + base=$(basename "$j") + case "$base" in recrawl-website-daily|reimport-github-weekly) echo "KEEP scheduler $base" ;; *) echo "DELETE scheduler $base — run: gcloud scheduler jobs delete $base --location=$REGION --project=$PROJECT_ID" ;; esac +done + +# --- 5. Storage fifth (prune old JSONL only) --- +echo "" +echo "=== 5. Storage — prune old import artifacts (keep bucket) ===" +echo "List: gcloud storage ls gs://interlispsearch-search-imports/ 2>/dev/null || gsutil ls gs://interlispsearch-search-imports/" +echo "Prune older than last successful import:" +echo " gcloud storage rm gs://interlispsearch-search-imports/github-import-*.jsonl --project=$PROJECT_ID # add --dry-run first if supported" +echo " (Do NOT delete the bucket itself)" + +# --- 6. Firestore sixth --- +echo "" +echo "=== 6. Firestore — TTL handles rate_limits; delete only ad-hoc test collections via console ===" + +# --- 7. IAM last (do not delete SA) --- +echo "" +echo "=== 7. IAM — do NOT delete vertex-search-sa; only delete unused keys: gcloud iam service-accounts keys list --iam-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com --project=$PROJECT_ID ===" +``` + +**Important:** The script above prints `DELETE` lines first. Uncomment the `curl -X DELETE` / `gcloud delete` lines only after you have saved inventory and confirmed the keep list. + +Single-store/engine delete examples for manual use: + +```bash +TOKEN=$(gcloud auth application-default print-access-token) +# Delete one engine +curl -X DELETE -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines/ENGINE_ID_TO_DELETE" + +# Delete one data store (cascades — wait 5-10 min, then verify GET returns 404) +curl -X DELETE -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores/DATASTORE_ID_TO_DELETE" + +# Verify deletion +curl -s -H "Authorization: Bearer $TOKEN" -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores/DATASTORE_ID_TO_DELETE" | python3 -m json.tool +# Expected: 404 Not Found +``` + +### Step 7.3: Validate After Cleanup + +```bash +./inventory-interlispsearch.sh +# Expected: 1 dataStore (interlisp-search-unified), 1 engine, 3 functions, 2 scheduler jobs + +curl -s "https://us-central1-interlispsearch.cloudfunctions.net/search?q=interlisp&pageSize=5" | python3 -m json.tool | head -40 +# Expected: 200, results with source:website first, citations from interlisp.org + +gcloud functions logs read search --gen2 --region=us-central1 --limit=20 --project=interlispsearch +# Expected: no discoveryengine quota errors +``` + +--- + +## Summary + +You now have a complete implementation guide for: + +1. ✅ Creating a unified data store with website + GitHub content +2. ✅ Implementing request-time filtering + post-processing in the Cloud Function +3. ✅ Deploying with proper citation preferences +4. ✅ Setting up automated daily website recrawls +5. ✅ Setting up automated weekly GitHub reimports +6. ✅ Testing and validating the deployment +7. ✅ Monitoring and troubleshooting procedures +8. ✅ **Cleanup — paring interlispsearch to in-use resources only (Phase 7)** + +**Next Steps:** + +1. Review this guide completely +2. Ensure all prerequisites are met +3. Follow Phase 1-4 in order +4. Use the testing checklist to validate each phase +5. Keep this document handy for troubleshooting +6. **Only after production validation (Phase 7): run inventory → review keep/delete list → delete in safe order (engines→dataStores→functions→scheduler→storage)** + +--- + +**Document Version:** 1.1 +**Last Updated:** September 02, 2026 +**Status:** Implemented — see Testing Checklist verified 2026-09-02; Phase 7 still gated (≥48h production) and scheduled `github-reimport` Pub/Sub requires isolated source dir diff --git a/search-function/package-lock.json b/search-function/package-lock.json new file mode 100644 index 00000000..bf3b530f --- /dev/null +++ b/search-function/package-lock.json @@ -0,0 +1,2209 @@ +{ + "name": "search-function", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "search-function", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@google-cloud/discoveryengine": "^2.6.0", + "@google-cloud/firestore": "^8.5.0", + "google-auth-library": "^10.6.2" + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "eslint": "^10.9.1", + "globals": "^17.11.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^3.0.5", + "debug": "^4.3.1", + "minimatch": "^10.2.4" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/config-array/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@eslint/config-array/node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@eslint/config-array/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.7.0.tgz", + "integrity": "sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/js": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", + "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "eslint": "^10.0.0" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } + } + }, + "node_modules/@eslint/object-schema": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.2.tgz", + "integrity": "sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@google-cloud/discoveryengine": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@google-cloud/discoveryengine/-/discoveryengine-2.6.0.tgz", + "integrity": "sha512-UP2hCCTW7R3rnTnILL4W+9fum2SaK96NomA8spINinPrKC3mxWrz6wtMkQcx8fTxZThkolNIN/pFgf4bTT2xUg==", + "license": "Apache-2.0", + "dependencies": { + "google-gax": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@google-cloud/firestore": { + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-8.5.0.tgz", + "integrity": "sha512-1sbtj7JQfsfekrxwHR0s2CAz8+hkpBdiuB7+20VEgI4EWvW3+GhEUqYOQPRoZ2spHyITLN4gaOPmBN17J539yQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "fast-deep-equal": "^3.1.3", + "functional-red-black-tree": "^1.0.1", + "google-gax": "^5.0.1", + "protobufjs": "^7.5.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", + "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", + "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.3", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "25.6.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", + "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.19.0" + } + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/brace-expansion": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz", + "integrity": "sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "10.9.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.9.1.tgz", + "integrity": "sha512-9VaAkDURekixUQJy0oJYl2DcN6oKMfxay7XzaGYAWQwsb6qfKf+x76R2k1L8kb1boc+FyCAaTA9GmiKaaiaF+A==", + "dev": true, + "license": "MIT", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.7.0", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.2", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "minimatch": "^10.2.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/eslint/node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/eslint/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/espree": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.16.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^5.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", + "license": "MIT" + }, + "node_modules/gaxios": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", + "integrity": "sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "17.11.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.11.0.tgz", + "integrity": "sha512-Z2I8hM+PbJDXQDq3Icgpzv+mPdwr68iZUU9d5WW4FuXfDUQfkZaZuvjMv42/5crNyw154+9+VWXbYrUgDXbxNw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/google-auth-library": { + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", + "integrity": "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-gax": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-5.0.6.tgz", + "integrity": "sha512-1kGbqVQBZPAAu4+/R1XxPQKP0ydbNYoLAr4l0ZO2bMV0kLyLW4I1gAk++qBLWt7DPORTzmWRMsCZe86gDjShJA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.12.6", + "@grpc/proto-loader": "^0.8.0", + "duplexify": "^4.1.3", + "google-auth-library": "^10.1.0", + "google-logging-utils": "^1.1.1", + "node-fetch": "^3.3.2", + "object-hash": "^3.0.0", + "proto3-json-serializer": "^3.0.0", + "protobufjs": "^7.5.3", + "retry-request": "^8.0.0", + "rimraf": "^5.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/proto3-json-serializer": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-3.0.4.tgz", + "integrity": "sha512-E1sbAYg3aEbXrq0n1ojJkRHQJGE1kaE/O6GLA94y8rnJBfgvOPTOd1b9hOceQK1FFZI9qMh1vBERCyO2ifubcw==", + "license": "Apache-2.0", + "dependencies": { + "protobufjs": "^7.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/protobufjs": { + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", + "integrity": "sha512-3wY1AxV+VBNW8Yypfd1yQY9pXnqTAN+KwQxL8iYm3/BjKYMNg4i0owhEe26PWDOMaIrzeeF98Lqd5NGz4omiIg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/retry-request": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-8.0.2.tgz", + "integrity": "sha512-JzFPAfklk1kjR1w76f0QOIhoDkNkSqW8wYKT08n9yysTmZfB+RQ2QoXoTAeOi1HD9ZipTyTAZg3c4pM/jeqgSw==", + "license": "MIT", + "dependencies": { + "extend": "^3.0.2", + "teeny-request": "^10.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/rimraf": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz", + "integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==", + "license": "ISC", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "license": "MIT", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "license": "MIT" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==", + "license": "MIT" + }, + "node_modules/teeny-request": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-10.1.2.tgz", + "integrity": "sha512-Xj0ZAQ0CeuQn6UxCDPLbFRlgcSTUEyO3+wiepr2grjIjyL/lMMs1Z4OwXn8kLvn/V1OuaEP0UY7Na6UDNNsYrQ==", + "license": "Apache-2.0", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2", + "stream-events": "^1.0.5" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/undici-types": { + "version": "7.19.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", + "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/search-function/package.json b/search-function/package.json new file mode 100644 index 00000000..c27c6689 --- /dev/null +++ b/search-function/package.json @@ -0,0 +1,25 @@ +{ + "name": "search-function", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "lint": "eslint .", + "lint:fix": "eslint . --fix" + }, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "dependencies": { + "@google-cloud/discoveryengine": "^2.6.0", + "@google-cloud/firestore": "^8.5.0", + "google-auth-library": "^10.6.2" + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "eslint": "^10.9.1", + "globals": "^17.11.0" + } +} diff --git a/search-function/rateLimiter.js b/search-function/rateLimiter.js new file mode 100644 index 00000000..02ed8f78 --- /dev/null +++ b/search-function/rateLimiter.js @@ -0,0 +1,101 @@ +'use strict'; + +const { Firestore } = require('@google-cloud/firestore'); + +const db = new Firestore({ projectId: process.env.PROJECT_ID }); + +// Rate limit configuration +// Adjust these values based on your traffic patterns +const LIMITS = { + perIp: { + requests: 20, // max 20 requests per IP + windowSec: 60, // per 60 second window + }, + global: { + requests: 500, // max 500 total requests + windowSec: 60, // per 60 second window + } +}; + +/** + * Atomically check and increment a rate limit counter in Firestore. + * Returns { allowed: true/false, count: current count } + */ +async function checkLimit(key, limit) { + const ref = db.collection('rate_limits').doc(key); + const now = Date.now(); + const windowMs = limit.windowSec * 1000; + + try { + const result = await db.runTransaction(async t => { + const doc = await t.get(ref); + const data = doc.exists ? doc.data() : null; + + // Start a new window if first request or window has expired + if (!data || (now - data.windowStart) > windowMs) { + t.set(ref, { + count: 1, + windowStart: now, + updatedAt: now + }); + return { allowed: true, count: 1 }; + } + + // Window is active — check if over limit + if (data.count >= limit.requests) { + return { allowed: false, count: data.count }; + } + + // Increment counter + t.update(ref, { + count: Firestore.FieldValue.increment(1), + updatedAt: now + }); + return { allowed: true, count: data.count + 1 }; + }); + + return result; + + } catch (err) { + // Fail open — if Firestore is unavailable don't block searches + console.error('Rate limiter error:', err.message); + return { allowed: true, count: 0 }; + } +} + +/** + * Main rate limit check — runs IP and global checks in parallel. + * Returns { limited: false } or { limited: true, reason, retryAfter } + */ +async function isRateLimited(req) { + // Get client IP — Cloud Functions passes real IP in x-forwarded-for + const ip = req.headers['x-forwarded-for'] + ?.split(',')[0] + ?.trim() || 'unknown'; + + // Run both checks in parallel + const [ipCheck, globalCheck] = await Promise.all([ + checkLimit(`ip:${ip}`, LIMITS.perIp), + checkLimit('global', LIMITS.global), + ]); + + if (!ipCheck.allowed) { + return { + limited: true, + reason: 'Too many requests. Please wait a moment before searching again.', + retryAfter: LIMITS.perIp.windowSec + }; + } + + if (!globalCheck.allowed) { + return { + limited: true, + reason: 'Search service is temporarily busy. Please try again shortly.', + retryAfter: LIMITS.global.windowSec + }; + } + + return { limited: false }; +} + +module.exports = { isRateLimited }; diff --git a/search-function/test-citations.sh b/search-function/test-citations.sh new file mode 100755 index 00000000..2732cd01 --- /dev/null +++ b/search-function/test-citations.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# File: test-citations.sh + +FUNCTION_URL="https://us-central1-interlispsearch.cloudfunctions.net/search" + +echo "Testing citation preference..." +echo "" + +# Test 1: General query (should cite interlisp.org first) +echo "Test 1: Query 'interlisp' (should cite website first)" +echo "==========================================" +curl -s "$FUNCTION_URL?q=interlisp&pageSize=10" | python3 -c " +import sys, json +data = json.load(sys.stdin) +if data.get('summary') and data['summary'].get('citations'): + for i, citation in enumerate(data['summary']['citations'][:3], 1): + print(f\"[{i}] {citation.get('title', 'Untitled')}\") + print(f\" URI: {citation.get('uri', 'N/A')}\") + print() +else: + print('No citations found') +" +echo "" + +# Test 2: Check first result URL +echo "Test 2: First result should be from interlisp.org" +echo "==========================================" +curl -s "$FUNCTION_URL?q=medley&pageSize=5" | python3 -c " +import sys, json +data = json.load(sys.stdin) +if data.get('results'): + result = data['results'][0] + print(f\"Title: {result.get('title')}\") + print(f\"URL: {result.get('url')}\") + print(f\"Source: {result.get('source')}\") + print(f\"Priority: {result.get('priority')}\") +else: + print('No results found') +" +echo "" + +# Test 3: Verify GitHub content still appears +echo "Test 3: GitHub content should still appear in results" +echo "==========================================" +curl -s "$FUNCTION_URL?q=github&pageSize=20" | python3 -c " +import sys, json +data = json.load(sys.stdin) +if data.get('results'): + github_results = [r for r in data['results'] if 'github' in r.get('url', '')] + print(f\"Found {len(github_results)} GitHub results\") + for r in github_results[:2]: + print(f\" - {r.get('title')}\") + print(f\" {r.get('url')}\") +else: + print('No results found') +" diff --git a/search-function/verify-deployment.sh b/search-function/verify-deployment.sh new file mode 100755 index 00000000..697bc087 --- /dev/null +++ b/search-function/verify-deployment.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# File: verify-deployment.sh + +PROJECT_ID="interlispsearch" +REGION="us-central1" + +echo "Verifying Cloud Function deployment..." +echo "" + +gcloud functions describe search \ + --gen2 \ + --region=$REGION \ + --project=$PROJECT_ID \ + --format=json | python3 -m json.tool + +echo "" +echo "Function URL: https://$REGION-$PROJECT_ID.cloudfunctions.net/search" +echo "" +echo "Testing function with sample query..." +curl -s "https://$REGION-$PROJECT_ID.cloudfunctions.net/search?q=interlisp&pageSize=5" \ + | python3 -m json.tool | head -50 diff --git a/search-function/vertex-ai-search-implementation.md b/search-function/vertex-ai-search-implementation.md new file mode 100644 index 00000000..236cea25 --- /dev/null +++ b/search-function/vertex-ai-search-implementation.md @@ -0,0 +1,986 @@ +# Vertex AI Search Integration with Hugo (Docsy) +## Implementation Guide for Interlisp.org + +This document captures the complete steps taken to replace Google Custom Search (GCS) with a Vertex AI Search (Agent Search) powered search engine on a Hugo/Docsy site hosted on GitHub Pages. + +Note: +- Currently there are multiple version of **Interlisp Site Search** we should be using this one: [interlisp.org.search](https://console.cloud.google.com/gen-app-builder/locations/global/engines/interlisp-org-search_1768860477660/overview/widget?project=interlispsearch&supportedpurview=folder) + +--- + +## Architecture Overview + +``` +Browser → Hugo Search Page → Cloud Function (proxy) → Vertex AI Search (Agent Search) + ↑ ↑ + Service Account Firestore (rate limiting) + ↑ + Data Store (interlisp.org crawl) +``` + +### Components + +| Component | Name/ID | Purpose | +|---|---|---| +| GCP Project | `interlispsearch` (191169864763) | Container for all resources | +| Service Account | `vertex-search-sa@interlispsearch.iam.gserviceaccount.com` | Auth identity for Cloud Function | +| Data Store | `interlisp-site-search` | Crawled and indexed web site content | +| Data Store | `interlisp-github-v2` | Uploaded content from www.github.com/Interlisp include markdown files, Issues, Discussions and PRs | +| Search Engine | `interlisp-site-search-v3` | Search app with LLM add-on | +| Cloud Function | `search` (us-central1) | Proxy between Hugo frontend and Vertex AI | +| Firestore | `rate_limits` collection | Per-IP and global request rate limiting | + +--- + +## Part 1 — GCP Project Setup + +### 1.1 Enable Required APIs + +```bash +gcloud services enable discoveryengine.googleapis.com +gcloud services enable cloudbuild.googleapis.com +gcloud services enable storage.googleapis.com +gcloud services enable orgpolicy.googleapis.com +gcloud services enable firestore.googleapis.com +``` + +### 1.2 Create Service Account + +```bash +gcloud iam service-accounts create vertex-search-sa \ + --display-name="Vertex Search Service Account" \ + --project=interlispsearch +``` + +### 1.3 Grant Service Account Roles + +```bash +# Core role for Vertex AI Search access +gcloud projects add-iam-policy-binding interlispsearch \ + --member="serviceAccount:vertex-search-sa@interlispsearch.iam.gserviceaccount.com" \ + --role="roles/discoveryengine.editor" + +# Allow reading from Cloud Storage (for future JSONL imports) +gcloud projects add-iam-policy-binding interlispsearch \ + --member="serviceAccount:vertex-search-sa@interlispsearch.iam.gserviceaccount.com" \ + --role="roles/storage.objectViewer" + +# Allow Firestore read/write for rate limiting +gcloud projects add-iam-policy-binding interlispsearch \ + --member="serviceAccount:vertex-search-sa@interlispsearch.iam.gserviceaccount.com" \ + --role="roles/datastore.user" +``` + +### 1.4 Override Org Policies (Project Level) + +The interlisp.org GCP organization had two restrictive org policies that blocked key creation and public function invocation. These were overridden at the project level without affecting the org-wide policy. + +```bash +# Allow allUsers IAM bindings in this project (needed for public Cloud Function) +cat > allow-all-users.yaml << 'EOF' +name: projects/interlispsearch/policies/iam.allowedPolicyMemberDomains +spec: + inheritFromParent: false + rules: + - allowAll: true +EOF + +gcloud org-policies set-policy allow-all-users.yaml --project=interlispsearch + +# Allow service account key creation in this project +cat > allow-sa-keys.yaml << 'EOF' +name: projects/interlispsearch/policies/iam.disableServiceAccountKeyCreation +spec: + inheritFromParent: false + rules: + - enforce: false +EOF + +gcloud org-policies set-policy allow-sa-keys.yaml --project=interlispsearch +``` + +### 1.5 Set Up Local Authentication + +```bash +# Authenticate with Application Default Credentials +gcloud auth application-default login + +# Set quota project (required for Discovery Engine API calls) +gcloud auth application-default set-quota-project interlispsearch + +# Set default project +gcloud config set project interlispsearch +``` + +**Important:** All `curl` calls to the Discovery Engine API require the `x-goog-user-project` header: + +```bash +-H "x-goog-user-project: interlispsearch" +``` + +--- + +## Part 2 — Vertex AI Data Store + +### 2.1 Create the Data Store + +```bash +TOKEN=$(gcloud auth application-default print-access-token) + +curl -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores?dataStoreId=interlisp-site-search" \ + -d '{ + "displayName": "Interlisp Site Search", + "industryVertical": "GENERIC", + "solutionTypes": ["SOLUTION_TYPE_SEARCH"], + "contentConfig": "PUBLIC_WEBSITE" + }' +``` + +**Key parameters:** +- `contentConfig: PUBLIC_WEBSITE` — enables website crawling +- `industryVertical: GENERIC` — required for non-Workspace data stores +- Location must be `global` for AI summarization features + +### 2.2 Add Target Site + +The URL pattern does **not** include the `https://` protocol prefix: + +```bash +TOKEN=$(gcloud auth application-default print-access-token) + +curl -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores/interlisp-site-search/siteSearchEngine/targetSites" \ + -d '{ + "providedUriPattern": "interlisp.org/*", + "type": "INCLUDE", + "exactMatch": false + }' +``` + +Note: A search engine created with multiple data stores can have 2 or more but may never drop to a single data store. Likewise, a search engine +created with a single data store can never extend to 2 or more. + +### 2.3 Upgrade to Advanced Website Indexing + +Advanced indexing is required for AI summarization features. This is done in the GCP console: + +1. Go to **AI Applications → Data Stores → interlisp-site-search** +2. Click the **Data** tab +3. Click **Upgrade to Advanced** next to the URL pattern + +Upgrading takes 4-8 hours for a site the size of interlisp.org. Check status: + +```bash +TOKEN=$(gcloud auth application-default print-access-token) + +curl -H "Authorization: Bearer $TOKEN" \ + -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/dataStores/interlisp-site-search/siteSearchEngine/targetSites" \ + | python3 -m json.tool +``` + +Look for `indexingStatus: SUCCEEDED` to confirm completion. + +--- + +## Part 3 — Vertex AI Search Engine (App) + +### 3.1 Important: App Type Selection + +The search engine must be created as **"Site search with AI mode"** in the console — NOT as "Gemini Enterprise" or "Custom Search (general)". + +- **Gemini Enterprise** — for internal workspace knowledge bases, incompatible with website data stores +- **Custom Search (general)** — for structured document/JSONL data, not website crawling +- **Site search with AI mode** ✓ — correct type for public website crawling with AI summaries + +### 3.2 Create via Console + +1. Go to **AI Applications → Apps → Create App** +2. Select **Search → Site search with AI mode** +3. Fill in details: + - App name: `interlisp-site-search-v2` + - Company name: `interlisp.org` + - Location: `global` +4. Select data store: `interlisp-site-search` +5. Note the generated engine ID: `interlisp-site-search-v2_1777510147931` + +### 3.3 Verify Engine Configuration + +```bash +TOKEN=$(gcloud auth application-default print-access-token) + +curl -H "Authorization: Bearer $TOKEN" \ + -H "x-goog-user-project: interlispsearch" \ + "https://discoveryengine.googleapis.com/v1/projects/interlispsearch/locations/global/collections/default_collection/engines/interlisp-site-search-v2_1777510147931" \ + | python3 -m json.tool +``` + +Confirm the response contains: +```json +{ + "searchEngineConfig": { + "searchTier": "SEARCH_TIER_ENTERPRISE", + "searchAddOns": ["SEARCH_ADD_ON_LLM"] + } +} +``` + +`SEARCH_ADD_ON_LLM` is required for AI summaries. + +--- + +## Part 4 — Cloud Function (Search Proxy) + +The Cloud Function acts as a secure proxy between the public Hugo frontend and the authenticated Vertex AI Search API. It also builds the AI prompt context. + +### 4.1 Project Structure + +``` +hugo-site/ +└── search-function/ + ├── index.js + ├── rateLimiter.js + ├── package.json + └── .gcloudignore +``` + +### 4.2 `package.json` + +```json +{ + "name": "search-function", + "version": "1.0.0", + "main": "index.js", + "dependencies": { + "@google-cloud/discoveryengine": "^1.0.0", + "@google-cloud/firestore": "^7.0.0", + "google-auth-library": "^9.0.0" + } +} +``` + +### 4.3 `index.js` + +The function uses the raw REST API (not the Node.js SDK) to avoid SDK auto-pagination which strips the `summary` field from responses. It includes Firestore-based rate limiting and resolves citation URLs by matching document IDs from results. + +```javascript +'use strict'; + +const { GoogleAuth } = require('google-auth-library'); +const { isRateLimited } = require('./rateLimiter'); + +const PROJECT_ID = process.env.PROJECT_ID; +const ENGINE_ID = process.env.ENGINE_ID; +const LOCATION = 'global'; + +const auth = new GoogleAuth({ + scopes: ['https://www.googleapis.com/auth/cloud-platform'] +}); + +exports.search = async (req, res) => { + + // CORS — allow specific origins only + const allowedOrigins = [ + 'https://interlisp.org', + 'https://www.interlisp.org', + 'https://stumbo.github.io', + 'http://localhost:1313', + 'http://localhost:8080', + ]; + + const origin = req.headers.origin || ''; + const allowedOrigin = allowedOrigins.includes(origin) + ? origin + : 'https://interlisp.org'; + + res.set('Access-Control-Allow-Origin', allowedOrigin); + res.set('Access-Control-Allow-Methods', 'GET, OPTIONS'); + res.set('Access-Control-Allow-Headers', 'Content-Type'); + res.set('Access-Control-Max-Age', '3600'); + + if (req.method === 'OPTIONS') { + res.status(204).send(''); + return; + } + + // Rate limiting + const rateLimitResult = await isRateLimited(req); + if (rateLimitResult.limited) { + res.set('Retry-After', String(rateLimitResult.retryAfter)); + res.status(429).json({ + error: 'Rate limit exceeded', + message: rateLimitResult.reason, + retryAfter: rateLimitResult.retryAfter + }); + return; + } + + const query = req.query.q || req.body?.q || ''; + const context = req.query.context || req.body?.context || ''; + const pageSize = parseInt(req.query.pageSize) || 10; + + if (!query.trim()) { + res.status(400).json({ error: 'Missing query parameter q' }); + return; + } + + try { + const client = await auth.getClient(); + const token = await client.getAccessToken(); + + const endpoint = `https://discoveryengine.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/collections/default_collection/engines/${ENGINE_ID}/servingConfigs/default_config:search`; + + const requestBody = { + query, + pageSize, + contentSearchSpec: { + summarySpec: { + summaryResultCount: 5, + includeCitations: true, + useSemanticChunks: true, + languageCode: 'en-US', + modelPromptSpec: { + preamble: buildPreamble(context) + }, + modelSpec: { + version: 'stable' + } + }, + snippetSpec: { + returnSnippet: true + }, + extractiveContentSpec: { + maxExtractiveAnswerCount: 3 + } + } + }; + + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token.token}`, + 'Content-Type': 'application/json', + 'x-goog-user-project': PROJECT_ID + }, + body: JSON.stringify(requestBody) + }); + + if (!response.ok) { + const errText = await response.text(); + throw new Error(`Vertex API error ${response.status}: ${errText}`); + } + + const data = await response.json(); + + // Strip HTML tags from snippet text + const stripHtml = str => str ? str.replace(/<[^>]*>/g, '') : null; + + // Build document ID → URL map for citation linking + const docIdToUrl = {}; + (data.results || []).forEach(result => { + const id = result.document?.id; + const url = result.document?.derivedStructData?.link; + if (id && url) docIdToUrl[id] = url; + }); + + // Map references to include resolved URLs by matching document IDs + const references = (data.summary?.summaryWithMetadata?.references || []) + .map(ref => { + const docId = ref.document?.split('/').pop(); + return { + title: ref.title, + uri: docIdToUrl[docId] || null, + }; + }); + + // REST API returns derivedStructData as flat object (not nested under .fields) + const results = (data.results || []).map(result => { + const derived = result.document?.derivedStructData; + if (!derived) return null; + + const url = derived.link || null; + const snippet = derived.snippets?.[0]?.snippet || null; + + return { + id: result.document?.id, + title: derived.title || null, + url, + snippet: stripHtml(derived.snippets?.[0]?.snippet || null), + section: url?.replace('https://interlisp.org/', '') + ?.split('/')?.[0] || '', + }; + }).filter(r => r?.url); + + const summaryText = data.summary?.summaryText || null; + + res.json({ + summary: summaryText ? { + summaryText, + citations: references + } : null, + results + }); + + } catch (err) { + console.error('Search error:', err); + res.status(500).json({ error: 'Search failed', detail: err.message }); + } +}; + +function buildPreamble(context) { + const base = `You are a search assistant for the Interlisp documentation site. +Answer questions clearly and concisely. Always cite the sources you used. +If no relevant results exist, say so directly rather than guessing.`; + + if (context) { + return `${base}\nThe user is currently browsing the "${context}" section — prioritize results from that section where relevant.`; + } + return base; +} +``` + +### 4.4 `.gcloudignore` + +``` +node_modules/ +.git/ +*.md +``` + +### 4.5 Deploy + +```bash +cd search-function +npm install + +gcloud functions deploy search \ + --gen2 \ + --runtime=nodejs24 \ + --region=us-central1 \ + --source=. \ + --entry-point=search \ + --trigger-http \ + --allow-unauthenticated \ + --service-account=vertex-search-sa@interlispsearch.iam.gserviceaccount.com \ + --set-env-vars PROJECT_ID=interlispsearch,ENGINE_ID=interlisp-org-search_1768860477660,ALLOWED_ORIGIN=https://interlisp.org \ + --memory=256Mi \ + --timeout=30s \ + --project=interlispsearch +``` + +**Key deployment decisions:** +- `--gen2` — required for Cloud Run-based functions with better performance +- `--allow-unauthenticated` — public endpoint, CORS restricts access by domain in function code +- `--service-account` — function runs as the service account, which has Discovery Engine and Firestore access +- No key file needed — the service account is attached at deploy time + +### 4.6 Verify Deployment + +```bash +# Test without auth token (should return results) +curl -s "https://us-central1-interlispsearch.cloudfunctions.net/search?q=interlisp" \ + | python3 -m json.tool | head -30 +``` + +--- + +## Part 5 — Firestore Rate Limiting + +Rate limiting is implemented using Firestore to track request counts across all function instances. In-memory counters cannot be used because Cloud Functions can scale to multiple instances. + +### 5.1 Create the Firestore Database + +```bash +gcloud firestore databases create \ + --location=us-central1 \ + --project=interlispsearch +``` + +### 5.2 Set Up TTL to Auto-Clean Expired Documents + +This prevents the `rate_limits` collection from growing indefinitely: + +```bash +gcloud firestore fields ttls update updatedAt \ + --collection-group=rate_limits \ + --enable-ttl \ + --project=interlispsearch +``` + +This operation takes 5-15 minutes to propagate. Check status: + +```bash +gcloud firestore fields describe updatedAt \ + --collection-group=rate_limits \ + --project=interlispsearch +``` + +Look for `ttlConfig.state: ACTIVE` to confirm completion. + +### 5.3 `rateLimiter.js` + +Create `search-function/rateLimiter.js`: + +```javascript +'use strict'; + +const { Firestore } = require('@google-cloud/firestore'); + +const db = new Firestore({ projectId: process.env.PROJECT_ID }); + +const LIMITS = { + perIp: { + requests: 20, // max 20 requests per IP + windowSec: 60, // per 60 second window + }, + global: { + requests: 500, // max 500 total requests + windowSec: 60, // per 60 second window + } +}; + +async function checkLimit(key, limit) { + const ref = db.collection('rate_limits').doc(key); + const now = Date.now(); + const windowMs = limit.windowSec * 1000; + + try { + const result = await db.runTransaction(async t => { + const doc = await t.get(ref); + const data = doc.exists ? doc.data() : null; + + if (!data || (now - data.windowStart) > windowMs) { + t.set(ref, { count: 1, windowStart: now, updatedAt: now }); + return { allowed: true, count: 1 }; + } + + if (data.count >= limit.requests) { + return { allowed: false, count: data.count }; + } + + t.update(ref, { + count: Firestore.FieldValue.increment(1), + updatedAt: now + }); + return { allowed: true, count: data.count + 1 }; + }); + + return result; + + } catch (err) { + // Fail open — don't block searches if Firestore is unavailable + console.error('Rate limiter error:', err.message); + return { allowed: true, count: 0 }; + } +} + +async function isRateLimited(req) { + const ip = req.headers['x-forwarded-for'] + ?.split(',')[0]?.trim() || 'unknown'; + + const [ipCheck, globalCheck] = await Promise.all([ + checkLimit(`ip:${ip}`, LIMITS.perIp), + checkLimit('global', LIMITS.global), + ]); + + if (!ipCheck.allowed) { + return { + limited: true, + reason: 'Too many requests. Please wait a moment before searching again.', + retryAfter: LIMITS.perIp.windowSec + }; + } + + if (!globalCheck.allowed) { + return { + limited: true, + reason: 'Search service is temporarily busy. Please try again shortly.', + retryAfter: LIMITS.global.windowSec + }; + } + + return { limited: false }; +} + +module.exports = { isRateLimited }; +``` + +### 5.4 Rate Limit Configuration + +| Limit | Value | Notes | +|---|---|---| +| Per IP | 20 requests / 60 seconds | Prevents individual abuse | +| Global | 500 requests / 60 seconds | Protects against aggregate overload | +| Fail behavior | Open (allow) | If Firestore unavailable, searches proceed | + +### 5.5 Verify Rate Limiting + +```bash +# Check Firestore documents are created after searches +gcloud firestore documents list \ + --collection=rate_limits \ + --project=interlispsearch + +# Test that 429 is returned after limit is exceeded +for i in {1..25}; do + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + "https://us-central1-interlispsearch.cloudfunctions.net/search?q=test") + echo "Request $i: $STATUS" +done +``` + +Requests 1-20 return `200`, requests 21+ return `429`. + +--- + +## Part 6 — Hugo / Docsy Integration + +The site uses Hugo with the **Docsy** theme (v0.14.3) loaded as a Hugo module. Docsy renders its search box only when `gcs_engine_id` is set in params — this param is kept to preserve the search UI, while the results page is completely overridden. + +### 6.1 How Docsy Search Works + +When a user types in the search box and presses Enter, Docsy's `search.js` redirects to `/search/?q=query`. The results page (`layouts/search.html`) is where we intercept and replace GCS with Vertex AI. + +### 6.2 `config/_default/params.yaml` Changes + +```yaml +# Keep this — Docsy won't render the search box without it +gcs_engine_id: 33ef4cbe0703b4f3a + +# Add Vertex AI Search Cloud Function URL +vertex_search_url: "https://us-central1-interlispsearch.cloudfunctions.net/search" +``` + +### 6.3 Override Search Results Page + +Create `layouts/search.html` (overrides Docsy's GCS results page): + +```html +{{ define "main" }} +
+
+

Search Results

+ +
+ + + + + +
+ Searching... +
+ +
+ +
+
+
+{{ end }} +``` + +### 6.4 Search Widget JavaScript + +Create `assets/js/vertex-search.js`: + +```javascript +(function () { + 'use strict'; + + const config = document.getElementById('vertex-search-config'); + if (!config) return; + + const FUNCTION_URL = config.dataset.searchUrl; + if (!FUNCTION_URL) return; + + const urlParams = new URLSearchParams(window.location.search); + const query = urlParams.get('q') || ''; + + const statusEl = document.getElementById('vertex-search-status'); + const hitsEl = document.getElementById('vertex-search-hits'); + const summaryEl = document.getElementById('vertex-search-summary'); + const summaryTxt = document.getElementById('summary-text'); + + function escapeHtml(str) { + if (!str) return ''; + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); + } + + async function doSearch(q) { + if (!q) { + statusEl.textContent = 'Enter a search query above.'; + return; + } + + document.title = `Search: ${q}`; + statusEl.innerHTML = ' Searching for ' + + escapeHtml(q) + '…'; + + try { + const url = new URL(FUNCTION_URL); + url.searchParams.set('q', q); + + // Pass referring section as context + const ref = document.referrer; + if (ref) { + try { + const refPath = new URL(ref).pathname.split('/').filter(Boolean); + if (refPath.length > 0) url.searchParams.set('context', refPath[0]); + } catch (_) {} + } + + const resp = await fetch(url.toString()); + if (!resp.ok) throw new Error(`HTTP ${resp.status}`); + const data = await resp.json(); + + statusEl.style.display = 'none'; + + // Show AI summary and citations if available + if (data.summary?.summaryText) { + summaryTxt.textContent = data.summary.summaryText; + summaryEl.style.display = 'block'; + + const refs = data.summary?.citations || []; + const validRefs = refs.filter(r => r.title || r.uri); + if (validRefs.length > 0) { + const citationHtml = validRefs.map((ref, i) => ` +
+ [${i + 1}] + ${ref.uri + ? `${escapeHtml(ref.title || ref.uri)}` + : `${escapeHtml(ref.title || 'Unknown source')}` + } +
+ `).join(''); + const citationsDiv = document.createElement('div'); + citationsDiv.className = 'search-citations mt-3'; + citationsDiv.innerHTML = '

Sources

' + citationHtml; + summaryEl.querySelector('.ai-summary').appendChild(citationsDiv); + } + } + + if (!data.results || data.results.length === 0) { + hitsEl.innerHTML = `

No results found for ${escapeHtml(q)}.

`; + return; + } + + hitsEl.innerHTML = + `

${data.results.length} results for ${escapeHtml(q)}

` + + data.results.map(r => ` +
+
+ ${escapeHtml(r.title || 'Untitled')} +
+ ${r.snippet ? `

${escapeHtml(r.snippet)}

` : ''} +

${escapeHtml(r.url)}

+
+ `).join(''); + + } catch (err) { + statusEl.textContent = 'Search error: ' + err.message; + console.error('Vertex search error:', err); + } + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', () => doSearch(query)); + } else { + doSearch(query); + } + +})(); +``` + +### 6.5 Load JS via `head-end.html` + +Update `layouts/_partials/hooks/head-end.html`: + +```html + + + +{{ $searchJS := resources.Get "js/vertex-search.js" | js.Build | fingerprint }} + +``` + +### 6.6 Add Styles + +Add to `assets/scss/_styles_project.scss`: + +```scss +// Vertex AI Search styles +.ai-summary { + background: #f0f7ff; + border-left: 4px solid #1a73e8; + padding: 1rem 1.25rem; + margin-bottom: 1.5rem; + border-radius: 0 4px 4px 0; + + .ai-badge { + font-size: 0.75rem; + font-weight: 600; + color: #1a73e8; + text-transform: uppercase; + letter-spacing: 0.05em; + display: block; + margin-bottom: 0.5rem; + } + + p { margin: 0; } +} + +.td-search-hit { + padding: 1rem 0; + border-bottom: 1px solid #e8e8e8; + + &:last-child { border-bottom: none; } +} + +.search-citations { + border-top: 1px solid #d0e4ff; + padding-top: 0.75rem; + margin-top: 0.75rem; + + .citations-label { + font-size: 0.7rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: #1a73e8; + margin-bottom: 0.4rem; + } + } + +.search-citation { + font-size: 0.8rem; + margin-bottom: 0.25rem; + + .citation-number { + color: #1a73e8; + font-weight: 600; + margin-right: 0.4rem; + } + + a { color: #444; } +} +``` + +--- + +## Part 7 — File Summary + +### New Files Created + +| File | Purpose | +|---|---| +| `search-function/index.js` | Cloud Function — proxy to Vertex AI Search | +| `search-function/rateLimiter.js` | Firestore-based rate limiting | +| `search-function/package.json` | Node.js dependencies | +| `search-function/.gcloudignore` | Excludes node_modules from deploy | +| `layouts/search.html` | Overrides Docsy's GCS results page | +| `assets/js/vertex-search.js` | Frontend search widget JS | + +### Modified Files + +| File | Change | +|---|---| +| `config/_default/params.yaml` | Added `vertex_search_url`, kept `gcs_engine_id` | +| `layouts/_partials/hooks/head-end.html` | Added JS bundle load | +| `assets/scss/_styles_project.scss` | Added search result and citation styles | + +--- + +## Part 8 — Key Lessons Learned + +### Authentication +- Local `curl` testing requires the `x-goog-user-project: interlispsearch` header with ADC credentials — without it, requests are billed against a Google-internal project and fail with 403 +- Cloud Functions authenticate via the attached service account at runtime — no key file needed +- Browser JS cannot call authenticated Cloud Functions directly — the function must be public (`--allow-unauthenticated`) with CORS restricting by origin in code + +### Vertex AI SDK vs REST API +- The Node.js `@google-cloud/discoveryengine` SDK uses auto-pagination by default, which flattens the response into a plain array of results and **strips the `summary` field** +- Using the raw REST API via `fetch` preserves the full response structure including `summary`, `totalSize`, `attributionToken`, and `semanticState` +- The REST API returns `derivedStructData` as a flat object (e.g. `derived.title`) rather than nested under `fields` as the SDK does (e.g. `fields.title.stringValue`) + +### Citations +- The `references` array in `summaryWithMetadata` contains document paths, not URLs +- URLs must be resolved by cross-referencing document IDs from the `results` array against the document path suffix in each reference +- `useSemanticChunks: true` in `summarySpec` improves citation accuracy + +### Org Policies +- `constraints/iam.allowedPolicyMemberDomains` blocked `allUsers` invocation +- `constraints/iam.disableServiceAccountKeyCreation` blocked service account key files +- Both were overridden at the project level using `gcloud org-policies set-policy` — requires `roles/owner` on the project but not org-level admin access + +### Docsy Search Integration +- Docsy only renders the search box when `gcs_engine_id` is set in params +- The search box redirects to `/search/?q=query` on Enter — this is handled by Docsy's own `search.js` +- Override `layouts/search.html` to replace GCS results with Vertex AI results +- Do not try to intercept the search input directly — work with Docsy's redirect behavior + +### Data Store Type +- Must be created as **"Site search with AI mode"** in the AI Applications console +- **"Gemini Enterprise"** app type is incompatible with website data stores +- URL patterns for target sites must **not** include `https://` protocol prefix +- Advanced website indexing (upgrade from Basic) is required for AI summarization — takes 4-8 hours + +### Rate Limiting +- Cloud Functions are stateless — in-memory rate limit counters don't work across scaled instances +- Firestore transactions provide atomic counter increments safe for concurrent access +- The rate limiter fails open — if Firestore is unavailable, searches proceed rather than being blocked +- Firestore TTL policies auto-clean expired rate limit documents — set on the `updatedAt` field + +--- + +## Part 9 — Deployment Status + +| Environment | URL | Status | +|---|---|---| +| Local dev | `http://localhost:1313` | ✅ Working | +| Staging | `https://stumbo.github.io/InterlispDraft.github.io/` | ✅ Deployed and verified | +| Production | `https://interlisp.org` | ⏳ Pending | + +### What is working end to end + +- Search box renders in Docsy navbar and sidebar +- Typing a query and pressing Enter redirects to `/search/?q=query` +- Vertex AI Search returns relevant results from the indexed interlisp.org site +- AI-generated summary appears at the top of results with `[n]` citation markers +- Citation sources are listed below the summary with links to source pages +- Snippets are plain text (HTML stripped) +- CORS is correctly scoped to allowed origins +- Rate limiting is active — 20 requests/minute per IP, 500/minute global +- Firestore TTL auto-cleans expired rate limit documents + +--- + +## Part 10 — Remaining Items + +- [ ] Deploy to production (`interlisp.org`) +- [ ] Set up CI/CD to re-index when site content changes (JSONL approach or scheduled recrawl) +- [ ] Remove any remaining debug `console.log` statements from `index.js` +- [ ] Monitor Firestore `rate_limits` collection and adjust limits based on real traffic +- [ ] Consider adding query logging to Firestore for search analytics +- [ ] Review and tune the AI preamble prompt based on real query patterns +- [ ] Add `interlisp.org` to the CORS `allowedOrigins` list in `index.js` before production deploy (already present, verify it matches exact production domain)