Serverless Scripting Example: Send Logs to Logging Services

Overview

You can use this document to learn how to send logs to third-party logging services. Specifically, this document provides examples for a self-hosted ElasticSearch cluster and the LogDNA SaaS. 

Before you begin, you must enable serverless scripting for your stack. To learn more, see Enable Serverless Scripting and Create a Script.

View a sample script for ElasticSearch

In the script below, update the following strings: 

• ELASTIC_URL 
  • Replace this string with your ElasticSearch cluster ingestion URL.
• YOURBASICAUTH 
  • Replace this string with your basic authentication credentials.
  • To learn more, see Generate basic authentication credentials.

Review the following sample script: 

let requests = [];

addEventListener('fetch', event => {
    event.respondWith(fetchAndLog(event));
});

async function fetchAndLog(event) {
    const response = await fetch(event.request);
    requests.push(getRequestData(event.request, response));
    return response;
}

async function getRequestData(request, response) {
    var data = {
        'timestamp': Date.now(),
        'x-sp-client-ip': request.headers.get("x-sp-client-ip"),

    };
    console.log(JSON.stringify(data));

    var url = "ELASTIC_URL";
    let myHeaders = new Headers();
    myHeaders.append("Content-Type", "application/json");
    myHeaders.append(
        "Authorization",
        "Basic YOURBASICAUTH"
    );

    await fetch(url, {
        method: "POST",
        body: JSON.stringify(data),
        headers: myHeaders
    });
} 

View a sample script for LogDNA

In the script below, update the following strings: 

• YOUR APP NAME 
  • Replace this string with your application name so that you can identify your application in the logs. 
• YOURBASICAUTH 
  • Replace this string with your basic authentication credentials. 
  • To learn more, see Generate basic authentication credentials.
• domain.com 
  • Replace this string with the domain that serves your requests.

Review the following sample script: 

let requests = [];
let workerInception, workerId, requestStartTime, requestEndTime;
let batchIsRunning = false;
const maxRequestsPerBatch = 150;

addEventListener("fetch", event => {
  event.respondWith(logRequests(event));
});

async function logRequests(event) {
  if (!batchIsRunning) {
    handleBatch(event);
  }
  if (requests.length >= maxRequestsPerBatch) {
    postRequests();
  }
  requestStartTime = Date.now();
  if (!workerInception) workerInception = Date.now();
  if (!workerId) workerId = makeid(6);
  const response = await fetch(event.request);
  requestEndTime = Date.now();
  requests.push(getRequestData(event.request, response));
  return response;
}

async function handleBatch(event) {
  batchIsRunning = true;
  await sleep(10000);
  try {
    if (requests.length) postRequests();
  } catch (e) {}
  requests = [];
  batchIsRunning = false;
}

function sleep(ms) {
  return new Promise(resolve => {
    setTimeout(resolve, ms);
  });
}

function getRequestData(request, re) {
  let data = {
    app: "YOUR APP NAME",
    timestamp: Date.now(),
    meta: {
      ua: request.headers.get("user-agent"),
      referer: request.headers.get("Referer") || "empty",
      ip: request.headers.get("x-sp-client-ip"),
      countryCode: (request.headers.get("x-sp-client-geo-countryCode") || {})
        .country,
      workerInception: workerInception,
      workerId: workerId,
      url: request.url,
      method: request.method,
      x_forwarded_for: request.headers.get("x_forwarded_for") || "0.0.0.0",
      status: (re || {}).status,
      originTime: requestEndTime - requestStartTime
    }
  };
  data.line =
    data.meta.status +
    " " +
    data.meta.countryCode +
    " " +
    data.meta.originTime +
    "ms" +
    " " +
    data.meta.ip +
    " " +
    data.meta.url;
  return data;
}

async function postRequests() {
  let data = JSON.stringify({ lines: requests });
  const hostname = "domain.com";

  let myHeaders = new Headers();
  myHeaders.append("Content-Type", "application/json; charset=UTF-8");
  myHeaders.append(
    "Authorization",
    "Basic YOURBASICAUTH"
  );
  try {
    const res = await fetch(
      "https://logs.logdna.com/logs/ingest?tag=worker&hostname=" + hostname,
      {
        method: "POST",
        headers: myHeaders,
        body: data
      }
    );
    requests = [];
  } catch (err) {
    // console.log(err.stack || err);
  }
}

function makeid(len) {
  let text = "";
  const possible = "ABCDEFGHIJKLMNPQRSTUVWXYZ0123456789";
  for (let i = 0; i < len; i++)
    text += possible.charAt(Math.floor(Math.random() * possible.length));
  return text;
}

Generate basic authentication credentials 

Basic authentication is a method used to authenticate with a remote web server with a username and password. To achieve this, pass an authorization header that contains the method (basic) and the username and password, joined by a :, which is then base64 encoded.

For example, if your username is stackpath and your password is logging, then you would need to base64 encode the string stackpath:logging, which gives the output of c3RhY2twYXRoOmxvZ2dpbmc=. This information would be passed in the request via the Authorization header as below:

Authorization: Basic c3RhY2twYXRoOmxvZ2dpbmc=

Review the following examples to learn how to generate a base64-encoded string. StackPath recommends that you do not use an online service to encode this string. 

For MacOS / Linux users: 

From a terminal, execute the following script and replace stackpath:logging with your credentials:

echo -n 'stackpath:logging' | openssl base64

For Windows users:

From a command prompt, execute the following script and replace stackpath:logging with your credentials:

powershell "[convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes(\"stackpath:logging\"))"