verint.com/kme_content_adapter
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: sitemap pagination, HTML wrapper, and title from vkm:name

Browse Source 
- Paginate sitemap using hydra:view['hydra:last'] (0-based item index model)
- Select latest vkm:datePublished fragment per SearchResultItem
- Cap sitemap at 50,000 URLs per sitemaps.org protocol
- Wrap content fetch response in full HTML document (DOCTYPE, head, body)
- Add <head><title> populated from vkm:name field
- Remove oidcAuthFlow route (404 for unmatched paths)
- All 51 tests passing

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Peter Morton
2026-04-23 19:07:06 -05:00
parent f840587e5e
commit d1563e8190
4 changed files with 236 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/globalVariables/kmeContentSourceAdapterHelpers.js
View File

@@ -21,12 +21,25 @@ function validateSettings(settings, requiredFields) {
* structure returned by the KME Knowledge Search Service: * structure returned by the KME Knowledge Search Service:
* data["hydra:member"][n] → SearchResultItem * data["hydra:member"][n] → SearchResultItem
* data["hydra:member"][n]["hydra:member"] → SearchResultItemFragment[] (has vkm:url) * data["hydra:member"][n]["hydra:member"] → SearchResultItemFragment[] (has vkm:url)
*
* For each SearchResultItem, only the fragment with the latest vkm:datePublished
* is returned. If no vkm:datePublished is present the fragment is treated as
* epoch 0, so dated fragments always win over undated ones.
*
* @param {object} data – response.data from the search API * @param {object} data – response.data from the search API
* @returns {object[]} * @returns {object[]}
*/ */
function extractHydraItems(data) { function extractHydraItems(data) {
const topMembers = data['hydra:member'] ?? []; const topMembers = data['hydra:member'] ?? [];
return topMembers.flatMap(resultItem => resultItem['hydra:member'] ?? []); return topMembers.map(resultItem => {
const fragments = resultItem['hydra:member'] ?? [];
if (fragments.length === 0) return null;
return fragments.reduce((latest, current) => {
const latestDate = new Date(latest['vkm:datePublished'] ?? 0).getTime();
const currentDate = new Date(current['vkm:datePublished'] ?? 0).getTime();
return currentDate > latestDate ? current : latest;
});
}).filter(Boolean);
} }
/** /**

58
src/proxyScripts/kmeContentSourceAdapter.js
View File

@@ -101,9 +101,17 @@
return; return;
} }
// Step 9: Return HTML // Step 9: Return HTML wrapped in a full document
const title = data['vkm:name'] ?? '';
const html = `<!DOCTYPE html>
<html>
<head><title>${title}</title></head>
<body>
${articleBody}
</body>
</html>`;
res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
res.end(articleBody); res.end(html);
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -143,19 +151,57 @@
const token = await kmeContentSourceAdapterHelpers.getValidToken(req.url, req.method); const token = await kmeContentSourceAdapterHelpers.getValidToken(req.url, req.method);
const reqParams = new URL(req.url, 'http://localhost').searchParams; const reqParams = new URL(req.url, 'http://localhost').searchParams;
const pageSize = reqParams.get('size') ?? '100';
const searchParams = new URLSearchParams({ const searchParams = new URLSearchParams({
query: reqParams.get('query') ?? '*', query: reqParams.get('query') ?? '*',
size: reqParams.get('size') ?? '100', size: pageSize,
category: reqParams.get('category') ?? 'vkm:ArticleCategory', category: reqParams.get('category') ?? 'vkm:ArticleCategory',
}); });
const searchUrl = `${searchApiBaseUrl}/${tenant}/search?${searchParams}`; const searchUrl = `${searchApiBaseUrl}/${tenant}/search?${searchParams}`;
console.info({ message: 'Sitemap flow: calling search API', url: searchUrl }); console.info({ message: 'Sitemap flow: calling search API (page 1)', url: searchUrl });
const searchResponse = await axios.get(searchUrl, { const firstResponse = await axios.get(searchUrl, {
headers: { Authorization: `OIDC_id_token ${token}`, 'Accept': 'application/ld+json' }, headers: { Authorization: `OIDC_id_token ${token}`, 'Accept': 'application/ld+json' },
timeout: 10000, timeout: 10000,
}); });
const items = kmeContentSourceAdapterHelpers.extractHydraItems(searchResponse.data); const firstData = firstResponse.data;
let allData = [firstData];
// Paginate: hydra:last is nested inside hydra:view.
// hydra:view is absent when all results fit on one page — no pagination needed.
// start= is a 0-based item index; subsequent page start values increment by size.
// e.g. 22 results, size=5 → hydra:view.hydra:last start=20, pages at start=5,10,15,20
const hydraLast = firstData['hydra:view']?.['hydra:last'];
if (hydraLast) {
const lastUrl = new URL(hydraLast);
const lastStart = parseInt(lastUrl.searchParams.get('start') ?? '0', 10);
const size = parseInt(lastUrl.searchParams.get('size') ?? pageSize, 10);
if (lastStart > 0 && size > 0) {
const pageUrls = [];
for (let start = size; start <= lastStart; start += size) {
const pageUrl = new URL(searchUrl);
pageUrl.searchParams.set('start', String(start));
pageUrls.push(pageUrl.toString());
}
console.info({ message: 'Sitemap flow: fetching additional pages', count: pageUrls.length });
const pageResponses = await Promise.all(
pageUrls.map(url => axios.get(url, {
headers: { Authorization: `OIDC_id_token ${token}`, 'Accept': 'application/ld+json' },
timeout: 10000,
}))
);
allData = [firstData, ...pageResponses.map(r => r.data)];
}
}
const SITEMAP_MAX_URLS = 50_000;
const allItems = allData.flatMap(
data => kmeContentSourceAdapterHelpers.extractHydraItems(data)
);
const items = allItems.length > SITEMAP_MAX_URLS ? allItems.slice(0, SITEMAP_MAX_URLS) : allItems;
if (allItems.length > SITEMAP_MAX_URLS) {
console.warn({ message: 'Sitemap flow: result set truncated to 50,000 (sitemaps.org limit)', total: allItems.length });
}
console.debug({ message: 'Sitemap flow: items received', count: items.length }); console.debug({ message: 'Sitemap flow: items received', count: items.length });
const xml = kmeContentSourceAdapterHelpers.buildSitemapXml(items, proxyBaseUrl); const xml = kmeContentSourceAdapterHelpers.buildSitemapXml(items, proxyBaseUrl);

6
tests/contract/proxy-http.test.js
View File

@@ -200,7 +200,7 @@ describe('sitemap endpoint', () => {
describe('content fetch: happy path', () => { describe('content fetch: happy path', () => {
test('GET /?kmeURL=<mock-server> → 200 text/html with article body (SC-001 < 11s)', async () => { test('GET /?kmeURL=<mock-server> → 200 text/html with article body (SC-001 < 11s)', async () => {
// Mock content server returning a valid article JSON-LD response // Mock content server returning a valid article JSON-LD response
const contentMock = await startMockServer(200, { 'vkm:articleBody': '<p>Contract test article</p>' }); const contentMock = await startMockServer(200, { 'vkm:name': 'Contract Article', 'vkm:articleBody': '<p>Contract test article</p>' });
try { try {
const redis = makeRedisFake(); const redis = makeRedisFake();
@@ -233,7 +233,9 @@ describe('content fetch: happy path', () => {
res.headers['Content-Type'].startsWith('text/html'), res.headers['Content-Type'].startsWith('text/html'),
`Content-Type was: ${res.headers['Content-Type']}`, `Content-Type was: ${res.headers['Content-Type']}`,
); );
assert.strictEqual(res.body, '<p>Contract test article</p>'); assert.ok(res.body.includes('<!DOCTYPE html>'), 'body should contain DOCTYPE');
assert.ok(res.body.includes('<title>Contract Article</title>'), 'body should contain title');
assert.ok(res.body.includes('<p>Contract test article</p>'), 'body should contain article content verbatim');
assert.ok(elapsed < 11000, `Round-trip should be under 11 s, took ${elapsed}ms`); assert.ok(elapsed < 11000, `Round-trip should be under 11 s, took ${elapsed}ms`);
} finally { } finally {
await contentMock.close(); await contentMock.close();

181
tests/unit/proxy.test.js
View File

@@ -240,8 +240,8 @@ describe('sitemap flow', () => {
const ctx = makeSitemapContext(t, async () => ({ const ctx = makeSitemapContext(t, async () => ({
data: { data: {
'hydra:member': [ 'hydra:member': [
{ 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/doc-1' }] }, { 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/doc-1', 'vkm:datePublished': '2024-01-01T00:00:00Z' }] },
{ 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/doc-2' }] }, { 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/doc-2', 'vkm:datePublished': '2024-06-01T00:00:00Z' }] },
], ],
}, },
})); }));
@@ -291,6 +291,139 @@ describe('sitemap flow', () => {
assert.ok(ctx._res.body.includes('valid'), 'the valid URL should appear in the loc'); assert.ok(ctx._res.body.includes('valid'), 'the valid URL should appear in the loc');
}); });
test('multiple fragments per SearchResultItem → only latest vkm:datePublished wins', async (t) => {
const ctx = makeSitemapContext(t, async () => ({
data: {
'hydra:member': [
{
'hydra:member': [
{ 'vkm:url': 'https://kme.example.com/doc/v1', 'vkm:datePublished': '2023-01-01T00:00:00Z' },
{ 'vkm:url': 'https://kme.example.com/doc/v3', 'vkm:datePublished': '2024-06-01T00:00:00Z' },
{ 'vkm:url': 'https://kme.example.com/doc/v2', 'vkm:datePublished': '2023-12-01T00:00:00Z' },
],
},
],
},
}));
await runScript(ctx);
assert.strictEqual(ctx._res.statusCode, 200);
const locMatches = ctx._res.body.match(/<loc>/g);
assert.strictEqual(locMatches?.length ?? 0, 1, 'exactly one <loc> element (latest version only)');
assert.ok(ctx._res.body.includes('doc%2Fv3'), 'the latest fragment (v3) should be the loc');
assert.ok(!ctx._res.body.includes('doc%2Fv1'), 'older fragment v1 should not appear');
assert.ok(!ctx._res.body.includes('doc%2Fv2'), 'older fragment v2 should not appear');
});
// Pagination: hydra:last nested inside hydra:view drives multi-page fetching.
// hydra:view is absent when all results fit on one page — no pagination needed.
// e.g. 22 results, size=5 → hydra:view['hydra:last'] start=20, fetch start=5,10,15,20
test('hydra:last (22 results, size=5, start=20) → fetches 4 extra pages, all 5 pages combined', async (t) => {
// Simulate the example from the spec: 22 results, page size 5
// First call has no start param; subsequent pages: start=5,10,15,20
const base = 'https://search.example.com/api/test-tenant/search?query=*&size=5&category=vkm%3AArticleCategory';
const pageData = {
[`${base}`]: {
'hydra:view': { 'hydra:last': `${base}&start=20` },
'hydra:member': [
{ 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/doc-p1', 'vkm:datePublished': '2024-01-01T00:00:00Z' }] },
],
},
[`${base}&start=5`]: {
'hydra:member': [
{ 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/doc-p2', 'vkm:datePublished': '2024-02-01T00:00:00Z' }] },
],
},
[`${base}&start=10`]: {
'hydra:member': [
{ 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/doc-p3', 'vkm:datePublished': '2024-03-01T00:00:00Z' }] },
],
},
[`${base}&start=15`]: {
'hydra:member': [
{ 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/doc-p4', 'vkm:datePublished': '2024-04-01T00:00:00Z' }] },
],
},
[`${base}&start=20`]: {
'hydra:member': [
{ 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/doc-p5', 'vkm:datePublished': '2024-05-01T00:00:00Z' }] },
],
},
};
// Build context with size=5 in the request URL
const ctx = makeContext(t, {
req: { url: '/sitemap.xml?size=5', method: 'GET', headers: { host: 'proxy.example.com', 'x-forwarded-proto': 'https' } },
});
ctx.kme_CSA_settings.searchApiBaseUrl = 'https://search.example.com/api';
ctx.kme_CSA_settings.tenant = 'test-tenant';
ctx._store['authorization:token'] = 'sitemap-token';
ctx._store['authorization:expiry'] = '9999999999';
ctx._axios.get = t.mock.fn(async (url) => ({ data: pageData[url] ?? { 'hydra:member': [] } }));
await runScript(ctx);
assert.strictEqual(ctx._res.statusCode, 200);
assert.strictEqual(ctx._axios.get.mock.calls.length, 5, 'should make 5 GET calls (start 0,5,10,15,20)');
const locMatches = ctx._res.body.match(/<loc>/g);
assert.strictEqual(locMatches?.length ?? 0, 5, 'all 5 items from all pages should appear');
assert.ok(ctx._res.body.includes('doc-p1'));
assert.ok(ctx._res.body.includes('doc-p5'));
});
test('hydra:view absent (all results on one page) → no additional pages fetched', async (t) => {
const ctx = makeSitemapContext(t, async () => ({
data: {
// No hydra:view — all 22 results fit in size=50
'hydra:member': [
{ 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/only-doc', 'vkm:datePublished': '2024-01-01T00:00:00Z' }] },
],
},
}));
await runScript(ctx);
assert.strictEqual(ctx._res.statusCode, 200);
assert.strictEqual(ctx._axios.get.mock.calls.length, 1, 'only one GET call when hydra:view absent');
const locMatches = ctx._res.body.match(/<loc>/g);
assert.strictEqual(locMatches?.length ?? 0, 1);
});
test('hydra:view present but hydra:last start=0 → no additional pages fetched', async (t) => {
const ctx = makeSitemapContext(t, async () => ({
data: {
'hydra:view': { 'hydra:last': 'https://search.example.com/api/test-tenant/search?query=*&size=100&category=vkm%3AArticleCategory&start=0' },
'hydra:member': [
{ 'hydra:member': [{ 'vkm:url': 'https://kme.example.com/only-doc', 'vkm:datePublished': '2024-01-01T00:00:00Z' }] },
],
},
}));
await runScript(ctx);
assert.strictEqual(ctx._res.statusCode, 200);
assert.strictEqual(ctx._axios.get.mock.calls.length, 1, 'only one GET call when hydra:last start=0');
const locMatches = ctx._res.body.match(/<loc>/g);
assert.strictEqual(locMatches?.length ?? 0, 1);
});
test('more than 50,000 items → sitemap truncated to exactly 50,000 <loc> elements', async (t) => {
const LIMIT = 50_000;
// Build a response with LIMIT + 5 items
const members = Array.from({ length: LIMIT + 5 }, (_, i) => ({
'hydra:member': [{ 'vkm:url': `https://kme.example.com/doc-${i}`, 'vkm:datePublished': '2024-01-01T00:00:00Z' }],
}));
const ctx = makeSitemapContext(t, async () => ({ data: { 'hydra:member': members } }));
await runScript(ctx);
assert.strictEqual(ctx._res.statusCode, 200);
const locMatches = ctx._res.body.match(/<loc>/g);
assert.strictEqual(locMatches?.length ?? 0, LIMIT, `should be capped at ${LIMIT}`);
});
// US3 error scenarios (T011b) // US3 error scenarios (T011b)
test('upstream 503 → 502 with Search service error message', async (t) => { test('upstream 503 → 502 with Search service error message', async (t) => {
@@ -394,10 +527,10 @@ describe('extractArticleBody helper', () => {
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
describe('US-content-fetch: happy path', () => { describe('US-content-fetch: happy path', () => {
test('cached token + valid article response → 200 text/html with body', async (t) => { test('cached token + valid article response → 200 text/html with body and title', async (t) => {
const contentAxios = { const contentAxios = {
post: t.mock.fn(async () => ({ data: { id_token: 'mock-token', expires_in: 9_999_999_999 } })), post: t.mock.fn(async () => ({ data: { id_token: 'mock-token', expires_in: 9_999_999_999 } })),
get: t.mock.fn(async () => ({ data: { 'vkm:articleBody': '<p>Hello</p>' } })), get: t.mock.fn(async () => ({ data: { 'vkm:name': 'My Article', 'vkm:articleBody': '<p>Hello</p>' } })),
}; };
const ctx = makeContext(t, { const ctx = makeContext(t, {
req: { url: '/?kmeURL=https://kme.example.com/content/article/123', method: 'GET', headers: {} }, req: { url: '/?kmeURL=https://kme.example.com/content/article/123', method: 'GET', headers: {} },
@@ -410,18 +543,18 @@ describe('US-content-fetch: happy path', () => {
await runScript(ctx); await runScript(ctx);
assert.strictEqual(ctx._res.statusCode, 200); assert.strictEqual(ctx._res.statusCode, 200);
assert.ok( assert.ok(ctx._res.headers['Content-Type'].startsWith('text/html'), `Content-Type was: ${ctx._res.headers['Content-Type']}`);
ctx._res.headers['Content-Type'].startsWith('text/html'), assert.ok(ctx._res.body.includes('<!DOCTYPE html>'), 'body should contain DOCTYPE');
`Content-Type was: ${ctx._res.headers['Content-Type']}`, assert.ok(ctx._res.body.includes('<title>My Article</title>'), 'body should contain title');
); assert.ok(ctx._res.body.includes('<p>Hello</p>'), 'body should contain article content verbatim');
assert.strictEqual(ctx._res.body, '<p>Hello</p>'); assert.ok(!ctx._res.body.includes('<p><p>'), 'article content should not be double-wrapped in <p>');
assert.strictEqual(contentAxios.post.mock.calls.length, 0, 'should not re-fetch token on cache hit'); assert.strictEqual(contentAxios.post.mock.calls.length, 0, 'should not re-fetch token on cache hit');
}); });
test('cache miss (fresh token acquired) → 200 text/html with body', async (t) => { test('cache miss (fresh token acquired) → 200 text/html with body', async (t) => {
const contentAxios = { const contentAxios = {
post: t.mock.fn(async () => ({ data: { id_token: 'fresh-token', expires_in: 9_999_999_999 } })), post: t.mock.fn(async () => ({ data: { id_token: 'fresh-token', expires_in: 9_999_999_999 } })),
get: t.mock.fn(async () => ({ data: { 'vkm:articleBody': '<p>Hello</p>' } })), get: t.mock.fn(async () => ({ data: { 'vkm:name': 'Fresh Article', 'vkm:articleBody': '<p>Hello</p>' } })),
}; };
const ctx = makeContext(t, { const ctx = makeContext(t, {
req: { url: '/?kmeURL=https://kme.example.com/content/article/123', method: 'GET', headers: {} }, req: { url: '/?kmeURL=https://kme.example.com/content/article/123', method: 'GET', headers: {} },
@@ -432,13 +565,31 @@ describe('US-content-fetch: happy path', () => {
await runScript(ctx); await runScript(ctx);
assert.strictEqual(ctx._res.statusCode, 200); assert.strictEqual(ctx._res.statusCode, 200);
assert.ok( assert.ok(ctx._res.headers['Content-Type'].startsWith('text/html'), `Content-Type was: ${ctx._res.headers['Content-Type']}`);
ctx._res.headers['Content-Type'].startsWith('text/html'), assert.ok(ctx._res.body.includes('<!DOCTYPE html>'), 'body should contain DOCTYPE');
`Content-Type was: ${ctx._res.headers['Content-Type']}`, assert.ok(ctx._res.body.includes('<title>Fresh Article</title>'), 'body should contain title');
); assert.ok(ctx._res.body.includes('<p>Hello</p>'), 'body should contain article content');
assert.strictEqual(ctx._res.body, '<p>Hello</p>');
assert.strictEqual(contentAxios.post.mock.calls.length, 1, 'should have fetched fresh token'); assert.strictEqual(contentAxios.post.mock.calls.length, 1, 'should have fetched fresh token');
}); });
test('vkm:name absent → title element is empty', async (t) => {
const contentAxios = {
post: t.mock.fn(),
get: t.mock.fn(async () => ({ data: { 'vkm:articleBody': '<p>No title</p>' } })),
};
const ctx = makeContext(t, {
req: { url: '/?kmeURL=https://kme.example.com/content/article/123', method: 'GET', headers: {} },
axios: contentAxios,
});
ctx._store['authorization:token'] = 'cached-token';
ctx._store['authorization:expiry'] = '9999999999';
await runScript(ctx);
assert.strictEqual(ctx._res.statusCode, 200);
assert.ok(ctx._res.body.includes('<title></title>'), 'title should be empty when vkm:name absent');
assert.ok(ctx._res.body.includes('<p>No title</p>'));
});
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------