Initial Version of sitemap.xml spec
This commit is contained in:
60
src/logger.js
Normal file
60
src/logger.js
Normal file
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Structured Logging Utility
|
||||
* Provides severity-based logging with JSON output
|
||||
*
|
||||
* @module logger
|
||||
*/
|
||||
|
||||
// Save reference to original console.log before it gets replaced
|
||||
const originalConsoleLog = globalThis.console.log.bind(globalThis.console);
|
||||
|
||||
/**
|
||||
* Log levels (in order of severity)
|
||||
*/
|
||||
const LOG_LEVELS = {
|
||||
DEBUG: 0,
|
||||
INFO: 1,
|
||||
WARN: 2,
|
||||
ERROR: 3
|
||||
};
|
||||
|
||||
/**
|
||||
* Get configured log level from global config
|
||||
* @returns {number} Log level threshold
|
||||
*/
|
||||
function getLogLevel() {
|
||||
const configLevel = global.config?.logging?.level || 'INFO';
|
||||
return LOG_LEVELS[configLevel.toUpperCase()] ?? LOG_LEVELS.INFO;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a message with structured metadata
|
||||
* @param {string} level - Log level (DEBUG|INFO|WARN|ERROR)
|
||||
* @param {string} message - Log message
|
||||
* @param {Object} meta - Additional metadata
|
||||
*/
|
||||
export function log(level, message, meta = {}) {
|
||||
const levelValue = LOG_LEVELS[level] ?? LOG_LEVELS.INFO;
|
||||
const threshold = getLogLevel();
|
||||
|
||||
// Only log if level meets or exceeds threshold
|
||||
if (levelValue >= threshold) {
|
||||
const entry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
level,
|
||||
message,
|
||||
...meta
|
||||
};
|
||||
originalConsoleLog(JSON.stringify(entry));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Console-like logging interface
|
||||
* Exported as 'console' to match standard console API
|
||||
*/
|
||||
export const logger = {
|
||||
debug: (message, meta) => log('DEBUG', message, meta),
|
||||
info: (message, meta) => log('INFO', message, meta),
|
||||
error: (message, meta) => log('ERROR', message, meta)
|
||||
};
|
||||
775
src/proxy.js
Normal file
775
src/proxy.js
Normal file
@@ -0,0 +1,775 @@
|
||||
/**
|
||||
* Google Drive Sitemap Adapter Proxy
|
||||
*
|
||||
* MONOLITHIC HTTP request handler - ALL functionality in this single file.
|
||||
* Architecture: Server.js delegates ALL requests to proxy.handleRequest(req, res)
|
||||
* Authentication: Service Account (JWT-based) inline
|
||||
*
|
||||
* Globals provided by server.js:
|
||||
* - console: Custom loggern
|
||||
* - crypto: Node.js crypto module (can't use 'crypto' - Web Crypto API conflict)
|
||||
* - config: Infrastructure settings (server port, logging level)
|
||||
* - axios: HTTP client
|
||||
* - uuidv4: UUID generator
|
||||
* - jwt: JSON Web Token library
|
||||
* - xmlBuilder: XML document builder
|
||||
* - globalThis['google_drive_settings']: Consolidated settings (from global/google_drive_settings.json)
|
||||
* - serviceAccount: Service account credentials
|
||||
* - scopes: OAuth2 scopes array
|
||||
* - driveQuery: Drive API query filter
|
||||
* - sitemap: Sitemap configuration (maxUrls)
|
||||
*
|
||||
* Structure:
|
||||
* Section 1: Authentication (Service Account JWT)
|
||||
* Section 2: Utility Functions
|
||||
* Section 3: XML Utilities
|
||||
* Section 4: Request Queue (FIFO)
|
||||
* Section 5: Drive API Client
|
||||
* Section 6: Sitemap Generation
|
||||
* Section 7: Request Handling & Routing
|
||||
*
|
||||
* @module proxy
|
||||
*/
|
||||
|
||||
// NO IMPORTS - ALL dependencies provided as globals by server.js
|
||||
|
||||
// =============================================================================
|
||||
// Section 1: Authentication (Service Account JWT)
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Cached access token for Drive API
|
||||
* @private
|
||||
*/
|
||||
let accessTokenCache = null;
|
||||
let tokenExpiryTime = null;
|
||||
|
||||
/**
|
||||
* Create JWT token for Google Service Account authentication
|
||||
* Uses RS256 algorithm with service account private key
|
||||
*
|
||||
* @param {Object} credentials - Service account credentials
|
||||
* @returns {string} Signed JWT token
|
||||
*/
|
||||
function createServiceAccountJWT(credentials, scopes) {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const expiry = now + 3600; // 1 hour
|
||||
|
||||
const payload = {
|
||||
iss: credentials.client_email,
|
||||
scope: scopes.join(' '),
|
||||
aud: 'https://oauth2.googleapis.com/token',
|
||||
exp: expiry,
|
||||
iat: now
|
||||
};
|
||||
|
||||
return jwt.sign(payload, credentials.private_key, { algorithm: 'RS256' });
|
||||
}
|
||||
|
||||
/**
|
||||
* Exchange JWT for access token
|
||||
*
|
||||
* @param {string} jwtToken - Signed JWT token
|
||||
* @returns {Promise<string>} Access token
|
||||
*/
|
||||
async function getAccessToken(jwtToken) {
|
||||
const response = await axios.post('https://oauth2.googleapis.com/token', {
|
||||
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
|
||||
assertion: jwtToken
|
||||
}, {
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }
|
||||
});
|
||||
|
||||
return response.data.access_token;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize Google OAuth Service Account client
|
||||
* Uses credentials from global object (loaded by server.js from global/ directory)
|
||||
*
|
||||
* @returns {Promise<string>} Access token for Drive API
|
||||
* @throws {Error} If credentials are invalid or not loaded
|
||||
*/
|
||||
async function initializeServiceAccount() {
|
||||
try {
|
||||
// Load settings from consolidated global object
|
||||
const settings = globalThis['google_drive_settings'];
|
||||
|
||||
if (!settings) {
|
||||
throw new Error('Google Drive settings not found in globalThis["google_drive_settings"]. Ensure server.js loaded global/google_drive_settings.json');
|
||||
}
|
||||
|
||||
// Validate service account structure
|
||||
if (!settings.serviceAccount || !settings.serviceAccount.client_email || !settings.serviceAccount.private_key) {
|
||||
throw new Error('Invalid service account key format - missing serviceAccount.client_email or serviceAccount.private_key');
|
||||
}
|
||||
|
||||
// Default scopes if not specified
|
||||
const scopes = settings.scopes || ['https://www.googleapis.com/auth/drive.readonly'];
|
||||
|
||||
// Create JWT token
|
||||
const jwtToken = createServiceAccountJWT(settings.serviceAccount, scopes);
|
||||
|
||||
// Exchange JWT for access token
|
||||
const accessToken = await getAccessToken(jwtToken);
|
||||
|
||||
console.info('Service account authenticated successfully', {
|
||||
email: settings.serviceAccount.client_email
|
||||
});
|
||||
|
||||
return accessToken;
|
||||
|
||||
} catch (error) {
|
||||
console.error('Service account authentication failed', {
|
||||
error: error.message
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create cached access token
|
||||
* Singleton pattern to avoid multiple authentications
|
||||
*
|
||||
* @returns {Promise<string>} Access token for Drive API
|
||||
*/
|
||||
async function getAccessTokenCached() {
|
||||
const now = Date.now();
|
||||
|
||||
// Return cached token if still valid (with 5 minute buffer)
|
||||
if (accessTokenCache && tokenExpiryTime && now < (tokenExpiryTime - 300000)) {
|
||||
return accessTokenCache;
|
||||
}
|
||||
|
||||
// Get new token
|
||||
accessTokenCache = await initializeServiceAccount();
|
||||
tokenExpiryTime = now + 3600000; // 1 hour from now
|
||||
|
||||
return accessTokenCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear cached access token (for testing)
|
||||
*/
|
||||
function clearAuthCache() {
|
||||
accessTokenCache = null;
|
||||
tokenExpiryTime = null;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Section 2: Utility Functions
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Generate a unique request ID for tracing
|
||||
* Uses UUID v4 for uniqueness
|
||||
*
|
||||
* @returns {string} Request ID in format: req_<uuid>
|
||||
*/
|
||||
function generateRequestId() {
|
||||
return `req_${crypto.randomUUID()}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate document ID format
|
||||
* Google Drive IDs are alphanumeric with hyphens and underscores
|
||||
*
|
||||
* @param {string} id - Document ID to validate
|
||||
* @returns {boolean} True if valid
|
||||
*/
|
||||
function validateDocumentId(id) {
|
||||
if (!id || typeof id !== 'string') {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Google Drive IDs are typically 8-128 characters
|
||||
// Characters: a-z, A-Z, 0-9, -, _
|
||||
const pattern = /^[a-zA-Z0-9_-]{8,128}$/;
|
||||
return pattern.test(id);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Section 3: XML Utilities
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Escape special XML characters
|
||||
* Prevents XML injection and ensures valid XML output
|
||||
*
|
||||
* @param {string} str - String to escape
|
||||
* @returns {string} Escaped string safe for XML
|
||||
*/
|
||||
function escapeXml(str) {
|
||||
if (!str) return '';
|
||||
|
||||
return str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Section 4: Request Queue (FIFO)
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* FIFO Queue for request processing
|
||||
*
|
||||
* Ensures sequential processing - only one request executes at a time.
|
||||
* Prevents concurrent Drive API operations per specification clarification #7.
|
||||
*/
|
||||
class RequestQueue {
|
||||
constructor() {
|
||||
this.queue = [];
|
||||
this.processing = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add request to queue and start processing
|
||||
*
|
||||
* @param {Function} handler - Async function to execute
|
||||
* @returns {Promise} Resolves when handler completes
|
||||
*/
|
||||
async enqueue(handler) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.queue.push({ handler, resolve, reject });
|
||||
|
||||
console.debug('Request enqueued', {
|
||||
queueLength: this.queue.length,
|
||||
processing: this.processing
|
||||
});
|
||||
|
||||
// Start processing if not already processing
|
||||
if (!this.processing) {
|
||||
this._processNext();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Process next request in queue
|
||||
* @private
|
||||
*/
|
||||
async _processNext() {
|
||||
if (this.queue.length === 0) {
|
||||
this.processing = false;
|
||||
console.debug('Queue empty, stopping processing');
|
||||
return;
|
||||
}
|
||||
|
||||
this.processing = true;
|
||||
const { handler, resolve, reject } = this.queue.shift();
|
||||
|
||||
console.debug('Processing next request', {
|
||||
remainingInQueue: this.queue.length
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await handler();
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
} finally {
|
||||
// Process next request
|
||||
this._processNext();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current queue length
|
||||
* @returns {number}
|
||||
*/
|
||||
get length() {
|
||||
return this.queue.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if queue is processing
|
||||
* @returns {boolean}
|
||||
*/
|
||||
get isProcessing() {
|
||||
return this.processing;
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
const requestQueue = new RequestQueue();
|
||||
|
||||
// =============================================================================
|
||||
// Section 5: Drive API Client
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Custom error for document count exceeding limit
|
||||
*/
|
||||
class DocumentCountExceededError extends Error {
|
||||
constructor(count, limit) {
|
||||
super(`Document count ${count} exceeds limit of ${limit}`);
|
||||
this.name = 'DocumentCountExceededError';
|
||||
this.count = count;
|
||||
this.limit = limit;
|
||||
this.statusCode = 413;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Query documents from Google Drive with pagination
|
||||
*
|
||||
* Enforces 50k document limit per sitemap protocol specification.
|
||||
* If count exceeds limit, throws DocumentCountExceededError.
|
||||
*
|
||||
* @param {Object} options - Query options
|
||||
* @param {string} options.query - Drive API query filter
|
||||
* @param {string} options.fields - Fields to retrieve
|
||||
* @param {number} options.pageSize - Page size for pagination
|
||||
* @param {number} options.maxDocuments - Maximum documents allowed (default: 50000)
|
||||
* @returns {Promise<Array>} Array of document objects
|
||||
* @throws {DocumentCountExceededError} If document count > maxDocuments
|
||||
* @throws {Error} If Drive API request fails
|
||||
*/
|
||||
async function queryDocuments(options = {}) {
|
||||
const {
|
||||
query = 'trashed = false',
|
||||
fields = 'nextPageToken,files(id,name,mimeType,modifiedTime)',
|
||||
pageSize = 100,
|
||||
maxDocuments = 50000
|
||||
} = options;
|
||||
|
||||
const allFiles = [];
|
||||
let pageToken = null;
|
||||
|
||||
console.debug('Starting Drive API query', {
|
||||
query,
|
||||
pageSize,
|
||||
maxDocuments
|
||||
});
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
const accessToken = await getAccessTokenCached();
|
||||
|
||||
do {
|
||||
// Build query parameters
|
||||
const params = new URLSearchParams({
|
||||
q: query,
|
||||
pageSize: pageSize.toString(),
|
||||
fields
|
||||
});
|
||||
|
||||
if (pageToken) {
|
||||
params.append('pageToken', pageToken);
|
||||
}
|
||||
|
||||
// Make direct HTTP call to Drive API
|
||||
const response = await axios.get(
|
||||
`https://www.googleapis.com/drive/v3/files?${params.toString()}`,
|
||||
{
|
||||
headers: {
|
||||
'Authorization': `Bearer ${accessToken}`,
|
||||
'Accept': 'application/json'
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const files = response.data.files || [];
|
||||
allFiles.push(...files);
|
||||
|
||||
console.debug('Drive API page retrieved', {
|
||||
pageFiles: files.length,
|
||||
totalFiles: allFiles.length,
|
||||
hasMore: !!response.data.nextPageToken
|
||||
});
|
||||
|
||||
// Check if we've exceeded the limit BEFORE fetching more
|
||||
if (allFiles.length > maxDocuments) {
|
||||
console.error('Document count exceeds limit', {
|
||||
count: allFiles.length,
|
||||
limit: maxDocuments
|
||||
});
|
||||
throw new DocumentCountExceededError(allFiles.length, maxDocuments);
|
||||
}
|
||||
|
||||
pageToken = response.data.nextPageToken;
|
||||
|
||||
} while (pageToken);
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
console.info('Drive API query completed', {
|
||||
documentCount: allFiles.length,
|
||||
duration
|
||||
});
|
||||
|
||||
return allFiles;
|
||||
|
||||
} catch (error) {
|
||||
// Re-throw DocumentCountExceededError as-is
|
||||
if (error instanceof DocumentCountExceededError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Log and re-throw other errors
|
||||
console.error('Drive API query failed', {
|
||||
error: error.message,
|
||||
code: error.code,
|
||||
statusCode: error.response?.status
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Map Drive API error to HTTP status code and retry info
|
||||
*
|
||||
* Per specification:
|
||||
* - 429: Rate limit - include Retry-After header
|
||||
* - 503: Service unavailable - NO RETRY (fail immediately)
|
||||
* - 401: Authentication failed
|
||||
* - 500: Other errors
|
||||
*
|
||||
* @param {Error} error - Drive API error
|
||||
* @returns {Object} { statusCode, retryAfter? }
|
||||
*/
|
||||
function mapDriveErrorToHttp(error) {
|
||||
// Handle DocumentCountExceededError
|
||||
if (error instanceof DocumentCountExceededError) {
|
||||
return { statusCode: 413 };
|
||||
}
|
||||
|
||||
// Extract status code from Drive API error
|
||||
const statusCode = error.response?.status || error.code || 500;
|
||||
|
||||
// Handle rate limiting (429)
|
||||
if (statusCode === 429) {
|
||||
// Extract Retry-After from response headers if present
|
||||
const retryAfter = error.response?.headers?.['retry-after'];
|
||||
const retryAfterSeconds = retryAfter ? parseInt(retryAfter, 10) : 60;
|
||||
|
||||
return {
|
||||
statusCode: 429,
|
||||
retryAfter: retryAfterSeconds
|
||||
};
|
||||
}
|
||||
|
||||
// Handle service unavailable (503) - NO RETRY per spec
|
||||
if (statusCode === 503) {
|
||||
return { statusCode: 503 };
|
||||
}
|
||||
|
||||
// Handle authentication errors
|
||||
if (statusCode === 401 || statusCode === 403) {
|
||||
return { statusCode: statusCode };
|
||||
}
|
||||
|
||||
// All other errors map to 500
|
||||
return { statusCode: 500 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate document count against limit
|
||||
*
|
||||
* @param {number} count - Document count
|
||||
* @param {number} limit - Maximum allowed (default: 50000)
|
||||
* @throws {DocumentCountExceededError} If count > limit
|
||||
*/
|
||||
function validateDocumentCount(count, limit = 50000) {
|
||||
if (count > limit) {
|
||||
throw new DocumentCountExceededError(count, limit);
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Section 6: Sitemap Generation
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Transform Drive document to sitemap entry
|
||||
*
|
||||
* Creates RESTful URL in format: {baseUrl}/documents/{documentId}
|
||||
* Per specification clarification #2.
|
||||
*
|
||||
* @param {Object} document - Drive API document
|
||||
* @param {string} document.id - Document ID
|
||||
* @param {string} document.modifiedTime - ISO 8601 timestamp
|
||||
* @param {string} baseUrl - Base URL for the adapter
|
||||
* @returns {Object} Sitemap entry { loc, lastmod }
|
||||
*/
|
||||
function toSitemapEntry(document, baseUrl) {
|
||||
if (!document || !document.id) {
|
||||
console.error('Invalid document for sitemap entry', { document });
|
||||
return null;
|
||||
}
|
||||
|
||||
// RESTful URL format: /documents/{documentId}
|
||||
const loc = `${baseUrl}/documents/${encodeURIComponent(document.id)}`;
|
||||
|
||||
// Format lastmod as ISO 8601 date (YYYY-MM-DD)
|
||||
let lastmod;
|
||||
if (document.modifiedTime) {
|
||||
try {
|
||||
const date = new Date(document.modifiedTime);
|
||||
lastmod = date.toISOString().split('T')[0]; // Extract YYYY-MM-DD
|
||||
} catch (error) {
|
||||
console.error('Invalid modifiedTime for document', {
|
||||
documentId: document.id,
|
||||
modifiedTime: document.modifiedTime
|
||||
});
|
||||
lastmod = new Date().toISOString().split('T')[0]; // Fallback to today
|
||||
}
|
||||
} else {
|
||||
lastmod = new Date().toISOString().split('T')[0]; // Fallback to today
|
||||
}
|
||||
|
||||
return { loc, lastmod };
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform array of Drive documents to sitemap entries
|
||||
*
|
||||
* @param {Array<Object>} documents - Array of Drive API documents
|
||||
* @param {string} baseUrl - Base URL for the adapter
|
||||
* @returns {Array<Object>} Array of sitemap entries
|
||||
*/
|
||||
function transformDocumentsToSitemapEntries(documents, baseUrl) {
|
||||
if (!Array.isArray(documents)) {
|
||||
console.error('Documents must be an array', { documents });
|
||||
return [];
|
||||
}
|
||||
|
||||
return documents
|
||||
.map(doc => toSitemapEntry(doc, baseUrl))
|
||||
.filter(entry => entry !== null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate XML sitemap from sitemap entries
|
||||
*
|
||||
* Handles empty sitemap (0 documents) case - returns valid XML with empty urlset.
|
||||
*
|
||||
* @param {Array<Object>} sitemapEntries - Array of { loc, lastmod } objects
|
||||
* @returns {string} Complete XML sitemap string
|
||||
*/
|
||||
function generateSitemapXML(sitemapEntries) {
|
||||
let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
|
||||
xml += '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n';
|
||||
|
||||
// Handle empty sitemap - valid XML with no <url> elements
|
||||
if (!sitemapEntries || sitemapEntries.length === 0) {
|
||||
xml += '</urlset>';
|
||||
return xml;
|
||||
}
|
||||
|
||||
for (const entry of sitemapEntries) {
|
||||
xml += ' <url>\n';
|
||||
xml += ` <loc>${escapeXml(entry.loc)}</loc>\n`;
|
||||
xml += ` <lastmod>${escapeXml(entry.lastmod)}</lastmod>\n`;
|
||||
xml += ' </url>\n';
|
||||
}
|
||||
|
||||
xml += '</urlset>';
|
||||
|
||||
return xml;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main sitemap generation function
|
||||
*
|
||||
* Combines document transformation and XML generation.
|
||||
*
|
||||
* @param {Array<Object>} documents - Array of Drive API documents
|
||||
* @param {string} baseUrl - Base URL for the adapter
|
||||
* @returns {string} Complete XML sitemap
|
||||
*/
|
||||
function generateSitemap(documents, baseUrl) {
|
||||
const entries = transformDocumentsToSitemapEntries(documents, baseUrl);
|
||||
return generateSitemapXML(entries);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Section 7: Request Handling & Routing
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Parse route from request
|
||||
* @param {string} method - HTTP method
|
||||
* @param {string} url - Request URL
|
||||
* @returns {Object} Route info or error
|
||||
*/
|
||||
function parseRoute(method, url) {
|
||||
if (method !== 'GET') {
|
||||
return { route: null, error: 'Method not allowed', statusCode: 405 };
|
||||
}
|
||||
|
||||
const urlObj = new URL(url, 'http://localhost');
|
||||
const path = urlObj.pathname;
|
||||
|
||||
// Match any path containing 'sitemap.xml'
|
||||
if (path.includes('sitemap.xml')) {
|
||||
return { route: 'sitemap' };
|
||||
}
|
||||
|
||||
// All other paths return 404
|
||||
return { route: null, error: 'Not found', statusCode: 404 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle sitemap generation request
|
||||
* Wrapped in FIFO queue to ensure sequential processing.
|
||||
*
|
||||
* @param {Object} res - HTTP response object
|
||||
* @param {string} requestId - Request ID for tracing
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async function handleSitemapRequest(res, requestId) {
|
||||
try {
|
||||
// Get configuration from consolidated global settings
|
||||
const settings = globalThis['google_drive_settings'] || {};
|
||||
const maxUrls = settings.sitemap?.maxUrls || 50000;
|
||||
const query = settings.driveQuery || 'trashed = false';
|
||||
|
||||
// Query documents from Drive API
|
||||
// This will throw DocumentCountExceededError if exceeds maxUrls limit
|
||||
const documents = await queryDocuments({
|
||||
query: query,
|
||||
maxDocuments: maxUrls
|
||||
});
|
||||
|
||||
// Generate sitemap XML with RESTful URLs
|
||||
const xml = generateSitemap(documents, settings.proxyScriptEndPoint);
|
||||
|
||||
// Send successful response
|
||||
res.statusCode = 200;
|
||||
res.setHeader('Content-Type', 'application/xml; charset=utf-8');
|
||||
res.setHeader('X-Request-Id', requestId);
|
||||
res.setHeader('X-Document-Count', documents.length.toString());
|
||||
res.end(xml);
|
||||
|
||||
console.info('Sitemap generated successfully', {
|
||||
requestId,
|
||||
documentCount: documents.length
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
// Map Drive API error to HTTP status code
|
||||
const errorResponse = mapDriveErrorToHttp(error);
|
||||
|
||||
res.statusCode = errorResponse.statusCode;
|
||||
|
||||
// Add Retry-After header for rate limiting (429)
|
||||
if (errorResponse.retryAfter) {
|
||||
res.setHeader('Retry-After', errorResponse.retryAfter.toString());
|
||||
}
|
||||
|
||||
// Per specification: error responses have NO body
|
||||
res.end();
|
||||
|
||||
console.error('Sitemap generation failed', {
|
||||
requestId,
|
||||
error: error.message,
|
||||
statusCode: errorResponse.statusCode,
|
||||
retryAfter: errorResponse.retryAfter
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle all HTTP requests
|
||||
* Main entry point called by server.js
|
||||
*
|
||||
* @param {Object} req - HTTP request object
|
||||
* @param {Object} res - HTTP response object
|
||||
*/
|
||||
export async function handleRequest(req, res) {
|
||||
const requestId = generateRequestId();
|
||||
const startTime = Date.now();
|
||||
|
||||
console.info('Request received', {
|
||||
requestId,
|
||||
method: req.method,
|
||||
url: req.url
|
||||
});
|
||||
|
||||
try {
|
||||
// Parse route
|
||||
const routeResult = parseRoute(req.method, req.url);
|
||||
|
||||
if (!routeResult.route) {
|
||||
res.statusCode = routeResult.statusCode;
|
||||
res.end(); // Empty body per spec
|
||||
|
||||
console.error('Route not found', {
|
||||
requestId,
|
||||
url: req.url,
|
||||
statusCode: routeResult.statusCode
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle sitemap route with FIFO queue
|
||||
// Per specification: queue concurrent requests, process sequentially
|
||||
if (routeResult.route === 'sitemap') {
|
||||
await requestQueue.enqueue(async () => {
|
||||
await handleSitemapRequest(res, requestId);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
res.statusCode = 500;
|
||||
res.end();
|
||||
|
||||
console.error('Request handler error', {
|
||||
requestId,
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
|
||||
} finally {
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
console.info('Request completed', {
|
||||
requestId,
|
||||
statusCode: res.statusCode,
|
||||
duration
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Exports for Testing
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Internal functions exported for unit testing only
|
||||
* DO NOT use these in production code - use handleRequest() instead
|
||||
*/
|
||||
export {
|
||||
// Authentication
|
||||
getAccessTokenCached,
|
||||
clearAuthCache,
|
||||
|
||||
// Utilities
|
||||
generateRequestId,
|
||||
validateDocumentId,
|
||||
escapeXml,
|
||||
|
||||
// Drive API Client
|
||||
DocumentCountExceededError,
|
||||
queryDocuments,
|
||||
mapDriveErrorToHttp,
|
||||
validateDocumentCount,
|
||||
|
||||
// Sitemap Generation
|
||||
toSitemapEntry,
|
||||
transformDocumentsToSitemapEntries,
|
||||
generateSitemapXML,
|
||||
generateSitemap,
|
||||
|
||||
// Request Queue
|
||||
requestQueue
|
||||
};
|
||||
212
src/server.js
Normal file
212
src/server.js
Normal file
@@ -0,0 +1,212 @@
|
||||
import http from 'node:http';
|
||||
import { join } from 'node:path';
|
||||
import { readFileSync, readdirSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname } from 'node:path';
|
||||
import axios from 'axios';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { create as xmlBuilder } from 'xmlbuilder2';
|
||||
import { logger } from './logger.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Note: crypto is already available as globalThis.crypto (Web Crypto API)
|
||||
// No need to import or set it - Node.js provides it by default
|
||||
|
||||
// Replace global console with custom logger
|
||||
globalThis.console = logger;
|
||||
|
||||
// Make libraries available globally for proxy.js
|
||||
globalThis.axios = axios;
|
||||
globalThis.uuidv4 = uuidv4;
|
||||
globalThis.jwt = jwt;
|
||||
globalThis.xmlBuilder = xmlBuilder;
|
||||
|
||||
/**
|
||||
* Load all JSON files from global/ directory and make them available as global objects
|
||||
* Pattern: global/filename.json -> globalThis['filename']
|
||||
*/
|
||||
function loadGlobalObjects() {
|
||||
const globalDir = join(__dirname, '..', 'global');
|
||||
|
||||
try {
|
||||
const files = readdirSync(globalDir).filter(f => f.endsWith('.json') && !f.endsWith('.example'));
|
||||
|
||||
files.forEach(file => {
|
||||
const objectName = file.replace('.json', '');
|
||||
const filePath = join(globalDir, file);
|
||||
|
||||
try {
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
const data = JSON.parse(content);
|
||||
globalThis[objectName] = data;
|
||||
logger.info(`Loaded global object: ${objectName}`, {
|
||||
file: file,
|
||||
keys: Object.keys(data)
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error(`Failed to load global object from ${file}`, {
|
||||
error: error.message
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`Loaded ${files.length} global objects from ${globalDir}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to load global objects', {
|
||||
directory: globalDir,
|
||||
error: error.message
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load configuration from config/default.json
|
||||
* Merges with environment variables (ENV takes precedence)
|
||||
*
|
||||
* @returns {Object} Configuration object
|
||||
*/
|
||||
function loadConfig() {
|
||||
const configPath = join(__dirname, '..', 'config', 'default.json');
|
||||
const configData = readFileSync(configPath, 'utf-8');
|
||||
const config = JSON.parse(configData);
|
||||
|
||||
// Merge environment variables (ENV vars take precedence)
|
||||
config.server.port = process.env.PORT ? parseInt(process.env.PORT, 10) : config.server.port;
|
||||
config.server.host = process.env.HOST || config.server.host;
|
||||
config.logging.level = process.env.LOG_LEVEL || config.logging.level;
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate configuration
|
||||
* @param {Object} config - Configuration object
|
||||
* @throws {Error} If configuration is invalid
|
||||
*/
|
||||
function validateConfig(config) {
|
||||
const errors = [];
|
||||
|
||||
// Validate server configuration
|
||||
if (!config.server.port || config.server.port < 1 || config.server.port > 65535) {
|
||||
errors.push('Invalid server.port (must be 1-65535)');
|
||||
}
|
||||
|
||||
// Validate consolidated Google Drive settings from global
|
||||
const settings = globalThis['google_drive_settings'];
|
||||
if (!settings) {
|
||||
errors.push('Missing google_drive_settings in global/ directory (required for all functionality)');
|
||||
} else {
|
||||
// Validate service account
|
||||
if (!settings.serviceAccount) {
|
||||
errors.push('Missing serviceAccount in google_drive_settings');
|
||||
} else {
|
||||
if (!settings.serviceAccount.client_email || !settings.serviceAccount.private_key) {
|
||||
errors.push('Invalid serviceAccount format - missing client_email or private_key');
|
||||
}
|
||||
}
|
||||
|
||||
// Validate scopes (optional, will use default if missing)
|
||||
if (settings.scopes) {
|
||||
if (!Array.isArray(settings.scopes) || settings.scopes.length === 0) {
|
||||
errors.push('Invalid scopes (must be a non-empty array)');
|
||||
}
|
||||
} else {
|
||||
logger.warn('No scopes found in google_drive_settings - using default: ["https://www.googleapis.com/auth/drive.readonly"]');
|
||||
}
|
||||
|
||||
// Validate sitemap config (optional)
|
||||
if (settings.sitemap) {
|
||||
if (settings.sitemap.maxUrls && (settings.sitemap.maxUrls < 1 || settings.sitemap.maxUrls > 50000)) {
|
||||
errors.push('Invalid sitemap.maxUrls (must be 1-50000)');
|
||||
}
|
||||
} else {
|
||||
logger.warn('No sitemap config found in google_drive_settings - using default maxUrls: 50000');
|
||||
}
|
||||
|
||||
// Validate drive query (optional)
|
||||
if (settings.driveQuery) {
|
||||
if (typeof settings.driveQuery !== 'string') {
|
||||
errors.push('Invalid driveQuery (must be a string)');
|
||||
}
|
||||
} else {
|
||||
logger.warn('No driveQuery found in google_drive_settings - using default: "trashed = false"');
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(`Configuration validation failed:\n${errors.join('\n')}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the HTTP server
|
||||
*/
|
||||
async function startServer() {
|
||||
try {
|
||||
// Load configuration into global.config
|
||||
global.config = loadConfig();
|
||||
|
||||
// Load global objects from global/ directory (e.g., service account keys)
|
||||
loadGlobalObjects();
|
||||
|
||||
logger.info('Starting Proxy Script Server...');
|
||||
logger.info(`Configuration loaded: ${JSON.stringify({
|
||||
port: global.config.server.port,
|
||||
host: global.config.server.host,
|
||||
logLevel: global.config.logging.level
|
||||
})}`);
|
||||
|
||||
// Validate configuration
|
||||
validateConfig(global.config);
|
||||
logger.info('Configuration validated successfully');
|
||||
|
||||
// Import proxy after global.config is set
|
||||
const { handleRequest } = await import('./proxy.js');
|
||||
|
||||
// Create HTTP server that delegates all requests to proxy
|
||||
const server = http.createServer((req, res) => {
|
||||
handleRequest(req, res);
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
const shutdown = () => {
|
||||
logger.info('\nShutting down gracefully...');
|
||||
server.close(() => {
|
||||
logger.info('Server closed');
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
// Force shutdown after 10 seconds
|
||||
setTimeout(() => {
|
||||
logger.error('Forced shutdown after timeout');
|
||||
process.exit(1);
|
||||
}, 10000);
|
||||
};
|
||||
|
||||
process.on('SIGTERM', shutdown);
|
||||
process.on('SIGINT', shutdown);
|
||||
|
||||
// Start listening
|
||||
server.listen(global.config.server.port, global.config.server.host, () => {
|
||||
logger.info('Server listening', {
|
||||
port: global.config.server.port,
|
||||
host: global.config.server.host,
|
||||
});
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to start server', {
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Start the server
|
||||
startServer();
|
||||
Reference in New Issue
Block a user