const requestPayload = {
    url: url,
    onlyMainContent: true,
    formats: [{
        type: "json",
        schema: {
            type: "object",
            required: [],
            properties: {
                title: {
                    type: "string"
                },
                author: {
                    type: "string"
                },
                date: {
                    type: "string"
                },
                formatted_content: {
                    type: "string",
                    description: "Main article content with basic formatting, no ads or promotional content, no links to sources or references. Use \n\n to separate paragraphs."
                },
                summary: {
                    type: "string",
                    description: "Only the gist of the text as one plain sentence (≤25 words). Convey the single most important idea and why it matters."
                },
                primary_tag: {
                    type: "string"
                },
                secondary_tags: {
                    type: "string"
                },
                reading_time: {
                    type: "string"
                },
                images: {
                    type: "array",
                    items: {
                        type: "string"
                    }
                },
                thumbnail: {
                    type: "string"
                },
            }
        }
    }]
};

app.log.info(`Firecrawl request payload: ${JSON.stringify(requestPayload, null, 2)}`);

const response = await httpClient.post(`${FIRECRAWL_API_URL}/v2/scrape`, requestPayload, {
    headers: {
        'Authorization': `Bearer ${process.env.FIRECRAWL_API_KEY}`,
        'Content-Type': 'application/json'
    },
    timeout: 120000,
    responseType: 'json', // Override the default arraybuffer responseType
    transformResponse: [(data) => {
        // Ensure we get proper JSON parsing
        if (typeof data === 'string') {
            try {
                return JSON.parse(data);
            } catch (e) {
                return data;
            }
        }
        return data;
    }]
});