import http from 'node:http'; import {randomUUID} from 'node:crypto'; import {appendFile, mkdir, readFile, readdir, writeFile} from 'node:fs/promises'; import path from 'node:path'; import {fileURLToPath} from 'node:url'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT_DIR = path.resolve(__dirname, '..'); const DEFAULT_HOST = process.env.HOST || '127.0.0.1'; const DEFAULT_PORT = Number(process.env.PORT || 7010); const DEFAULT_DATA_DIR = path.resolve(process.env.DATA_DIR || path.join(ROOT_DIR, '../../var/index-data')); const MAX_JSON_BODY = 96 * 1024; const MAX_STRING = 4096; const MAX_SHORT_STRING = 256; const MAX_TAGS = 32; const MAX_LINKS = 16; const MAX_LIMIT = 100; function jsonLine(value) { return `${JSON.stringify(value)}\n`; } function sendJson(res, status, body, headers = {}) { res.writeHead(status, { 'content-type': 'application/json; charset=utf-8', ...headers }); res.end(`${JSON.stringify(body, null, 2)}\n`); } function methodAllowed(req, res, allowed) { if (allowed.includes(req.method)) { return true; } if (req.method === 'OPTIONS') { res.writeHead(204, {allow: allowed.join(', ')}); res.end(); return false; } sendJson(res, 405, {error: 'method not allowed'}, {allow: allowed.join(', ')}); return false; } async function readJsonBody(req, limit = MAX_JSON_BODY) { const chunks = []; let size = 0; for await (const chunk of req) { size += chunk.length; if (size > limit) { const error = new Error('request body too large'); error.statusCode = 413; throw error; } chunks.push(chunk); } const text = Buffer.concat(chunks).toString('utf8').trim(); if (!text) { return {}; } try { return JSON.parse(text); } catch { const error = new Error('invalid json body'); error.statusCode = 400; throw error; } } function safeJsonParse(text, fallback = null) { try { return JSON.parse(text); } catch { return fallback; } } async function readJson(filePath, fallback = null) { try { return safeJsonParse(await readFile(filePath, 'utf8'), fallback); } catch (err) { if (err && typeof err === 'object' && err.code === 'ENOENT') { return fallback; } throw err; } } function encodedFilePart(value) { return Buffer.from(String(value)).toString('base64url'); } function cleanString(value, {max = MAX_STRING} = {}) { if (typeof value !== 'string') { return null; } const text = value.trim().replace(/[\u0000-\u001f\u007f]/g, ''); return text ? text.slice(0, max) : null; } function cleanKind(value) { const kind = cleanString(value, {max: 64}); if (!kind || !/^[a-z][a-z0-9._-]{0,63}$/.test(kind)) { return null; } return kind; } function cleanUrl(value) { const text = cleanString(value, {max: MAX_STRING}); if (!text) { return null; } try { const url = new URL(text); if (url.protocol !== 'http:' && url.protocol !== 'https:') { return null; } return url.toString(); } catch { return null; } } function cleanTags(value) { if (!Array.isArray(value)) { if (typeof value === 'string') { value = value.split(','); } else { return []; } } const seen = new Set(); const tags = []; for (const item of value) { const tag = cleanString(String(item), {max: 64}); if (!tag) { continue; } const normalized = tag.toLowerCase(); if (seen.has(normalized)) { continue; } seen.add(normalized); tags.push(normalized); if (tags.length >= MAX_TAGS) { break; } } return tags; } function cleanLinks(value) { if (!Array.isArray(value)) { return []; } const links = []; for (const item of value) { const url = cleanUrl(item); if (url) { links.push(url); } if (links.length >= MAX_LINKS) { break; } } return links; } function cleanMetadata(value) { if (!value || typeof value !== 'object' || Array.isArray(value)) { return null; } const text = JSON.stringify(value); if (Buffer.byteLength(text) > 16 * 1024) { return null; } return value; } function booleanOr(value, fallback) { if (typeof value === 'boolean') { return value; } return fallback; } function headerString(headers, name, options = {}) { return cleanString(headers[name.toLowerCase()], options); } function writerContext(headers) { const identityId = headerString(headers, 'x-jmpkit-identity-id'); const context = { identityId, requesterIdentityId: headerString(headers, 'x-jmpkit-requester-identity-id'), billingRef: headerString(headers, 'x-jmpkit-billing-ref'), billingPath: headerString(headers, 'x-jmpkit-billing-path'), appId: headerString(headers, 'x-jmpkit-app-id'), appHost: headerString(headers, 'x-jmpkit-app-host'), resourceVersion: headerString(headers, 'x-jmpkit-resource-version') || 'index10' }; return Object.values(context).some(Boolean) ? context : null; } function publicEntityProfile(profile, {viewerEntityId = null} = {}) { if (!profile) { return null; } const owner = viewerEntityId && viewerEntityId === profile.entityId; if (!profile.indexed && !owner) { return null; } return { entityId: profile.entityId, indexed: Boolean(profile.indexed), followable: Boolean(profile.followable), displayName: profile.displayName, nickname: profile.nickname, description: profile.description, links: profile.links || [], createdAt: profile.createdAt, updatedAt: profile.updatedAt }; } function publicItem(record, profile = null) { const entityProfile = publicEntityProfile(profile) || { entityId: record.ownerEntityId, indexed: false }; return { itemId: record.itemId, kind: record.kind, resourceId: record.resourceId, url: record.url, title: record.title, description: record.description, tags: record.tags || [], indexed: Boolean(record.indexed), appId: record.appId, appHost: record.appHost, ownerEntityId: record.ownerEntityId, entity: entityProfile, createdAt: record.createdAt, updatedAt: record.updatedAt }; } function ownerViewItem(record, profile = null) { return { ...publicItem(record, profile), actorEntityId: record.actorEntityId, deletedAt: record.deletedAt || null, metadata: record.metadata || null, billing: record.billing || null }; } function itemMatchesQuery(record, profile, q) { if (!q) { return true; } const text = [ record.kind, record.resourceId, record.url, record.title, record.description, ...(record.tags || []), profile?.displayName, profile?.nickname, profile?.description ].filter(Boolean).join('\n').toLowerCase(); return q.split(/\s+/).every((part) => text.includes(part)); } function itemIsVisible(record, viewerEntityId = null) { if (record.deletedAt) { return false; } return Boolean(record.indexed || (viewerEntityId && viewerEntityId === record.ownerEntityId)); } export class IndexStore { constructor({dataDir = DEFAULT_DATA_DIR} = {}) { this.dataDir = dataDir; this.itemsDir = path.join(dataDir, 'items'); this.entitiesDir = path.join(dataDir, 'entities'); this.eventsDir = path.join(dataDir, 'events'); } async health() { return {ok: true, resourceVersion: 'index10'}; } async createItem(body, context, {now = new Date()} = {}) { if (!context?.identityId) { const error = new Error('index item write requires identity context'); error.statusCode = 401; throw error; } const kind = cleanKind(body.kind); const url = cleanUrl(body.url); if (!kind) { const error = new Error('kind is required'); error.statusCode = 400; throw error; } if (!url) { const error = new Error('valid http(s) url is required'); error.statusCode = 400; throw error; } const timestamp = now.toISOString(); const item = { itemId: `idx_${randomUUID()}`, kind, resourceId: cleanString(body.resourceId, {max: MAX_SHORT_STRING}), url, title: cleanString(body.title, {max: MAX_SHORT_STRING}), description: cleanString(body.description, {max: 2048}), tags: cleanTags(body.tags), indexed: booleanOr(body.indexed, true), appId: context.appId, appHost: context.appHost, ownerEntityId: context.identityId, actorEntityId: context.requesterIdentityId || context.identityId, metadata: cleanMetadata(body.metadata), billing: { billingRef: context.billingRef, billingPath: context.billingPath }, createdAt: timestamp, updatedAt: timestamp }; await mkdir(this.itemsDir, {recursive: true}); await writeFile(this.#itemFile(item.itemId), `${JSON.stringify(item, null, 2)}\n`, {flag: 'wx'}); await this.#recordEvent('item.created', item, {now}); return item; } async getItem(itemId) { return readJson(this.#itemFile(itemId)); } async patchItem(itemId, body, context, {now = new Date()} = {}) { const item = await this.getItem(itemId); if (!item) { const error = new Error('unknown index item'); error.statusCode = 404; throw error; } if (!context?.identityId || context.identityId !== item.ownerEntityId) { const error = new Error('index item owner authorization required'); error.statusCode = context?.identityId ? 403 : 401; throw error; } const next = { ...item, updatedAt: now.toISOString() }; if (Object.hasOwn(body, 'kind')) { const kind = cleanKind(body.kind); if (!kind) { const error = new Error('kind is invalid'); error.statusCode = 400; throw error; } next.kind = kind; } if (Object.hasOwn(body, 'url')) { const url = cleanUrl(body.url); if (!url) { const error = new Error('url is invalid'); error.statusCode = 400; throw error; } next.url = url; } for (const key of ['resourceId', 'title']) { if (Object.hasOwn(body, key)) { next[key] = cleanString(body[key], {max: MAX_SHORT_STRING}); } } if (Object.hasOwn(body, 'description')) { next.description = cleanString(body.description, {max: 2048}); } if (Object.hasOwn(body, 'tags')) { next.tags = cleanTags(body.tags); } if (Object.hasOwn(body, 'indexed')) { next.indexed = Boolean(body.indexed); } if (Object.hasOwn(body, 'metadata')) { next.metadata = cleanMetadata(body.metadata); } await writeFile(this.#itemFile(itemId), `${JSON.stringify(next, null, 2)}\n`); await this.#recordEvent('item.updated', next, {now}); return next; } async deleteItem(itemId, context, {now = new Date()} = {}) { const item = await this.getItem(itemId); if (!item) { const error = new Error('unknown index item'); error.statusCode = 404; throw error; } if (!context?.identityId || context.identityId !== item.ownerEntityId) { const error = new Error('index item owner authorization required'); error.statusCode = context?.identityId ? 403 : 401; throw error; } const next = { ...item, indexed: false, deletedAt: now.toISOString(), updatedAt: now.toISOString() }; await writeFile(this.#itemFile(itemId), `${JSON.stringify(next, null, 2)}\n`); await this.#recordEvent('item.deleted', next, {now}); return next; } async listItems(filters = {}, context = null) { const records = await this.#readAllItems(); const viewerEntityId = context?.identityId || null; const q = cleanString(filters.q, {max: 256})?.toLowerCase() || null; const kind = cleanKind(filters.kind); const appId = cleanString(filters.appId, {max: MAX_SHORT_STRING}); const tag = cleanString(filters.tag, {max: 64})?.toLowerCase() || null; let entityId = cleanString(filters.entityId, {max: MAX_SHORT_STRING}); if (entityId === 'me') { entityId = viewerEntityId; } const limit = Math.max(1, Math.min( MAX_LIMIT, Number.parseInt(filters.limit || '50', 10) || 50 )); const out = []; const profileCache = new Map(); for (const record of records) { if (!itemIsVisible(record, viewerEntityId)) { continue; } if (kind && record.kind !== kind) { continue; } if (appId && record.appId !== appId) { continue; } if (entityId && record.ownerEntityId !== entityId) { continue; } if (tag && !(record.tags || []).includes(tag)) { continue; } const profile = await this.#profileFor(record.ownerEntityId, profileCache); if (!itemMatchesQuery(record, profile, q)) { continue; } out.push(viewerEntityId === record.ownerEntityId ? ownerViewItem(record, profile) : publicItem(record, profile)); } out.sort((left, right) => { const leftTime = Date.parse(left.updatedAt || left.createdAt || '') || 0; const rightTime = Date.parse(right.updatedAt || right.createdAt || '') || 0; return rightTime - leftTime || left.itemId.localeCompare(right.itemId); }); return out.slice(0, limit); } async updateEntityProfile(body, context, {now = new Date()} = {}) { if (!context?.identityId) { const error = new Error('entity profile write requires identity context'); error.statusCode = 401; throw error; } const existing = await this.getEntityProfile(context.identityId, {includePrivate: true}); const timestamp = now.toISOString(); const profile = { entityId: context.identityId, indexed: existing?.indexed || false, followable: existing?.followable || false, displayName: existing?.displayName || null, nickname: existing?.nickname || null, description: existing?.description || null, links: existing?.links || [], createdAt: existing?.createdAt || timestamp, updatedAt: timestamp }; if (Object.hasOwn(body, 'indexed')) { profile.indexed = Boolean(body.indexed); } if (Object.hasOwn(body, 'followable')) { profile.followable = Boolean(body.followable); } if (Object.hasOwn(body, 'displayName')) { profile.displayName = cleanString(body.displayName, {max: MAX_SHORT_STRING}); } if (Object.hasOwn(body, 'nickname')) { profile.nickname = cleanString(body.nickname, {max: 64}); } if (Object.hasOwn(body, 'description')) { profile.description = cleanString(body.description, {max: 2048}); } if (Object.hasOwn(body, 'links')) { profile.links = cleanLinks(body.links); } await mkdir(this.entitiesDir, {recursive: true}); await writeFile(this.#entityFile(context.identityId), `${JSON.stringify(profile, null, 2)}\n`); await this.#recordEvent('entity.updated', profile, {now}); return profile; } async getEntityProfile(entityId, {includePrivate = false, viewerEntityId = null} = {}) { const cleanEntityId = cleanString(entityId, {max: MAX_SHORT_STRING}); if (!cleanEntityId) { return null; } const profile = await readJson(this.#entityFile(cleanEntityId)); if (!profile) { return null; } if (includePrivate) { return profile; } return publicEntityProfile(profile, {viewerEntityId}); } async #readAllItems() { try { const entries = await readdir(this.itemsDir, {withFileTypes: true}); const records = []; for (const entry of entries) { if (!entry.isFile() || !entry.name.endsWith('.json')) { continue; } const record = await readJson(path.join(this.itemsDir, entry.name)); if (record?.itemId && record?.kind && record?.url && record?.ownerEntityId) { records.push(record); } } return records; } catch (err) { if (err && typeof err === 'object' && err.code === 'ENOENT') { return []; } throw err; } } async #profileFor(entityId, cache) { if (cache.has(entityId)) { return cache.get(entityId); } const profile = await this.getEntityProfile(entityId, {includePrivate: true}); cache.set(entityId, profile); return profile; } async #recordEvent(type, record, {now = new Date()} = {}) { const day = now.toISOString().slice(0, 10); await mkdir(this.eventsDir, {recursive: true}); await appendFile(path.join(this.eventsDir, `${day}.jsonl`), jsonLine({ type, at: now.toISOString(), id: record.itemId || record.entityId, ownerEntityId: record.ownerEntityId || record.entityId, appId: record.appId || null })); } #itemFile(itemId) { return path.join(this.itemsDir, `${encodedFilePart(itemId)}.json`); } #entityFile(entityId) { return path.join(this.entitiesDir, `${encodedFilePart(entityId)}.json`); } } function routesList() { return [ 'GET /healthz', 'POST /items', 'GET /items', 'GET /items/:itemId', 'PATCH /items/:itemId', 'DELETE /items/:itemId', 'GET /search', 'PUT /entities/me', 'PATCH /entities/me', 'GET /entities/:entityId/profile', 'GET /entities/:entityId/items' ]; } export function createIndexServer({store = new IndexStore()} = {}) { return http.createServer(async (req, res) => { try { const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`); const context = writerContext(req.headers); if (url.pathname === '/' || url.pathname === '') { if (!methodAllowed(req, res, ['GET', 'HEAD', 'OPTIONS'])) { return; } if (req.method === 'HEAD') { res.writeHead(200, {'content-type': 'application/json; charset=utf-8'}); res.end(); return; } sendJson(res, 200, { ok: true, resourceVersion: 'index10', routes: routesList() }); return; } if (url.pathname === '/healthz') { if (!methodAllowed(req, res, ['GET', 'HEAD', 'OPTIONS'])) { return; } if (req.method === 'HEAD') { res.writeHead(200, {'content-type': 'application/json; charset=utf-8'}); res.end(); return; } sendJson(res, 200, await store.health()); return; } if (url.pathname === '/items') { if (!methodAllowed(req, res, ['GET', 'HEAD', 'POST', 'OPTIONS'])) { return; } if (req.method === 'GET' || req.method === 'HEAD') { const filters = Object.fromEntries(url.searchParams.entries()); const items = await store.listItems({ ...filters, appId: filters.appId || context?.appId }, context); if (req.method === 'HEAD') { res.writeHead(200, {'content-type': 'application/json; charset=utf-8'}); res.end(); return; } sendJson(res, 200, {items}); return; } const item = await store.createItem(await readJsonBody(req), context); const profile = await store.getEntityProfile(item.ownerEntityId, {includePrivate: true}); sendJson(res, 201, {item: ownerViewItem(item, profile)}, {'cache-control': 'no-store'}); return; } if (url.pathname === '/search') { if (!methodAllowed(req, res, ['GET', 'HEAD', 'OPTIONS'])) { return; } const filters = Object.fromEntries(url.searchParams.entries()); const items = await store.listItems({ ...filters, q: filters.q || '', appId: filters.appId || context?.appId }, context); if (req.method === 'HEAD') { res.writeHead(200, {'content-type': 'application/json; charset=utf-8'}); res.end(); return; } sendJson(res, 200, {items}); return; } const itemMatch = url.pathname.match(/^\/items\/([^/]+)$/); if (itemMatch) { if (!methodAllowed(req, res, ['GET', 'HEAD', 'PATCH', 'DELETE', 'OPTIONS'])) { return; } const itemId = decodeURIComponent(itemMatch[1]); if (req.method === 'PATCH') { const item = await store.patchItem(itemId, await readJsonBody(req), context); const profile = await store.getEntityProfile(item.ownerEntityId, {includePrivate: true}); sendJson(res, 200, {item: ownerViewItem(item, profile)}, {'cache-control': 'no-store'}); return; } if (req.method === 'DELETE') { const item = await store.deleteItem(itemId, context); const profile = await store.getEntityProfile(item.ownerEntityId, {includePrivate: true}); sendJson(res, 200, {item: ownerViewItem(item, profile)}, {'cache-control': 'no-store'}); return; } const item = await store.getItem(itemId); if (!item || !itemIsVisible(item, context?.identityId || null)) { sendJson(res, 404, {error: 'unknown index item'}); return; } const profile = await store.getEntityProfile(item.ownerEntityId, {includePrivate: true}); if (req.method === 'HEAD') { res.writeHead(200, {'content-type': 'application/json; charset=utf-8'}); res.end(); return; } sendJson(res, 200, { item: context?.identityId === item.ownerEntityId ? ownerViewItem(item, profile) : publicItem(item, profile) }); return; } if (url.pathname === '/entities/me') { if (!methodAllowed(req, res, ['PUT', 'PATCH', 'OPTIONS'])) { return; } const profile = await store.updateEntityProfile(await readJsonBody(req), context); sendJson(res, 200, {entity: publicEntityProfile(profile, {viewerEntityId: context?.identityId})}, { 'cache-control': 'no-store' }); return; } const entityProfileMatch = url.pathname.match(/^\/entities\/([^/]+)\/profile$/); if (entityProfileMatch) { if (!methodAllowed(req, res, ['GET', 'HEAD', 'OPTIONS'])) { return; } const entityId = decodeURIComponent(entityProfileMatch[1]); const profile = await store.getEntityProfile(entityId, { viewerEntityId: context?.identityId || null }); if (!profile) { sendJson(res, 404, {error: 'unknown indexed entity'}); return; } if (req.method === 'HEAD') { res.writeHead(200, {'content-type': 'application/json; charset=utf-8'}); res.end(); return; } sendJson(res, 200, {entity: profile}); return; } const entityItemsMatch = url.pathname.match(/^\/entities\/([^/]+)\/items$/); if (entityItemsMatch) { if (!methodAllowed(req, res, ['GET', 'HEAD', 'OPTIONS'])) { return; } const entityId = decodeURIComponent(entityItemsMatch[1]); const filters = Object.fromEntries(url.searchParams.entries()); const items = await store.listItems({ ...filters, entityId, appId: filters.appId || context?.appId }, context); if (req.method === 'HEAD') { res.writeHead(200, {'content-type': 'application/json; charset=utf-8'}); res.end(); return; } sendJson(res, 200, {items}); return; } sendJson(res, 404, { error: 'unknown index10 route', routes: routesList() }); } catch (error) { const status = Number(error?.statusCode || error?.status || 500); sendJson(res, status >= 400 && status <= 599 ? status : 500, { error: status >= 500 ? 'internal server error' : error.message }, status === 401 ? {'www-authenticate': 'Bearer'} : {}); } }); } export async function startServer({ host = DEFAULT_HOST, port = DEFAULT_PORT, ...options } = {}) { const server = createIndexServer(options); await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, host, resolve); }); const address = server.address(); const actualHost = address.address === '::' ? '127.0.0.1' : address.address; return { server, host: actualHost, port: address.port, baseUrl: `http://${actualHost}:${address.port}`, close: () => new Promise((resolve, reject) => { server.close((err) => (err ? reject(err) : resolve())); }) }; } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { startServer().then(({host, port}) => { console.log(`[${new Date().toISOString()}] index10 listening on http://${host}:${port}`); }).catch((error) => { console.error(error); process.exit(1); }); }