Skip to content

Commit

Permalink
Add Prettier and apply to all project
Browse files Browse the repository at this point in the history
  • Loading branch information
victorshinya committed Jul 12, 2020
1 parent 58f4ca5 commit 97499ab
Show file tree
Hide file tree
Showing 9 changed files with 143 additions and 102 deletions.
37 changes: 19 additions & 18 deletions .eslintrc.json
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
{
"extends": [
"eslint:recommended",
"plugin:import/errors",
"plugin:import/warnings",
"airbnb-base"
],
"parserOptions": {
"ecmaVersion": 11
},
"rules": {
"no-console": "off",
"new-cap": [
"error",
{
"newIsCapExceptions": ["Buffer.from"]
}
]
}
"extends": [
"eslint:recommended",
"plugin:import/errors",
"plugin:import/warnings",
"airbnb-base",
"prettier"
],
"parserOptions": {
"ecmaVersion": 11
},
"rules": {
"no-console": "off",
"new-cap": [
"error",
{
"newIsCapExceptions": ["Buffer.from"]
}
]
}
}
3 changes: 3 additions & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Ignore artifacts:
build
coverage
1 change: 1 addition & 0 deletions .prettierrc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
23 changes: 11 additions & 12 deletions CODE_OF_CONDUCT.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

# Contributor Covenant Code of Conduct

## Our Pledge
Expand All @@ -18,23 +17,23 @@ diverse, inclusive, and healthy community.
Examples of behavior that contributes to a positive environment for our
community include:

* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
- Demonstrating empathy and kindness toward other people
- Being respectful of differing opinions, viewpoints, and experiences
- Giving and gracefully accepting constructive feedback
- Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
* Focusing on what is best not just for us as individuals, but for the
- Focusing on what is best not just for us as individuals, but for the
overall community

Examples of unacceptable behavior include:

* The use of sexualized language or imagery, and sexual attention or
- The use of sexualized language or imagery, and sexual attention or
advances of any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or email
- Trolling, insulting or derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or email
address, without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
- Other conduct which could reasonably be considered inappropriate in a
professional setting

## Enforcement Responsibilities
Expand Down Expand Up @@ -107,7 +106,7 @@ Violating these terms may lead to a permanent ban.
### 4. Permanent Ban

**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.

**Consequence**: A permanent ban from any sort of public interaction within
Expand Down
40 changes: 19 additions & 21 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,25 +51,25 @@ In order to start the script on a VM, with a recursive mode, you have to uncomme

```js
async function main() {
console.time("LogDNA-COS")
const response = await downloadAndSend()
console.log(`DEBUG: downloadAndSend = ${JSON.stringify(response.message)}`)
console.timeEnd("LogDNA-COS")
// DEBUG::
// switch (response.status) {
// case 200:
// console.log(`DEBUG: Fetch new log file`)
// await main()
// break
// case 204:
// console.log(`DEBUG: Wait 3 minutes to fetch new log file on COS Bucket`)
// await new Promise(r => setTimeout(r, 180000))
// await main()
// break
// default:
// console.log(`DEBUG: Uncommon behavior`)
// break
// }
console.time("LogDNA-COS");
const response = await downloadAndSend();
console.log(`DEBUG: downloadAndSend = ${JSON.stringify(response.message)}`);
console.timeEnd("LogDNA-COS");
// DEBUG::
// switch (response.status) {
// case 200:
// console.log(`DEBUG: Fetch new log file`)
// await main()
// break
// case 204:
// console.log(`DEBUG: Wait 3 minutes to fetch new log file on COS Bucket`)
// await new Promise(r => setTimeout(r, 180000))
// await main()
// break
// default:
// console.log(`DEBUG: Uncommon behavior`)
// break
// }
}

// DEBUG::
Expand All @@ -86,8 +86,6 @@ Tip: Use [pm2](https://www.npmjs.com/package/pm2) to run the script on backgroun

## Troubleshooting



## LICENSE

Copyright 2020 Victor Shinya
Expand Down
97 changes: 57 additions & 40 deletions handler.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,35 +16,35 @@
*
*/
// https://www.npmjs.com/package/dotenv
require('dotenv').config();
require("dotenv").config();
// https://www.npmjs.com/package/ibm-cos-sdk
const { S3 } = require('ibm-cos-sdk');
const { S3 } = require("ibm-cos-sdk");
// https://www.npmjs.com/package/request
const request = require('request-promise').defaults({ forever: true });
const request = require("request-promise").defaults({ forever: true });
// https://nodejs.org/api/zlib.html
const { unzip } = require('zlib');
const { unzip } = require("zlib");
// https://nodejs.org/api/util.html
const util = require('util');
const util = require("util");
/**
*
* IBM CLOUD OBJECT STORAGE
* Endpoint access -> Endpoint, API Key and Instance ID
*
*/
const cos = new S3({
endpoint: process.env.COS_ENDPOINT || '{endpoint}',
apiKeyId: process.env.COS_APIKEY || '{apiKeyId}',
ibmAuthEndpoint: 'https://iam.cloud.ibm.com/identity/token',
serviceInstanceId: process.env.COS_INSTANCEID || '{serviceInstanceId}',
endpoint: process.env.COS_ENDPOINT || "{endpoint}",
apiKeyId: process.env.COS_APIKEY || "{apiKeyId}",
ibmAuthEndpoint: "https://iam.cloud.ibm.com/identity/token",
serviceInstanceId: process.env.COS_INSTANCEID || "{serviceInstanceId}",
});
/**
*
* IBM CLOUD OBJECT STORAGE
* Using "From-To" logic with all logs
*
*/
const BUCKET_RECEIVER = process.env.COS_BUCKET_RECEIVER || '{bucketReceiver}';
const BUCKET_ARCHIVE = process.env.COS_BUCKET_ARCHIVE || '{bucketArchive}';
const BUCKET_RECEIVER = process.env.COS_BUCKET_RECEIVER || "{bucketReceiver}";
const BUCKET_ARCHIVE = process.env.COS_BUCKET_ARCHIVE || "{bucketArchive}";
/**
*
* IBM CLOUD OBJECT STORAGE
Expand All @@ -59,8 +59,8 @@ const MAX_KEYS = 1;
* API Key and Hostname to send the logs
*
*/
const INGESTION_KEY = process.env.LOGDNA_INGESTION_KEY || '{ingestionKey}';
const HOSTNAME = process.env.LOGDNA_HOSTNAME || '{host}';
const INGESTION_KEY = process.env.LOGDNA_INGESTION_KEY || "{ingestionKey}";
const HOSTNAME = process.env.LOGDNA_HOSTNAME || "{host}";
/**
*
* PACKAGE PER REQUEST
Expand All @@ -72,11 +72,19 @@ const LOGS = 20000;

async function uploadAndDeleteBucket(fileName) {
try {
console.log('DEBUG: Uploading the log file');
await cos.copyObject({ Bucket: BUCKET_ARCHIVE, CopySource: `${BUCKET_RECEIVER}/${fileName}`, Key: fileName }).promise();
console.log('DEBUG: Deleting the log file');
await cos.deleteObject({ Bucket: BUCKET_RECEIVER, Key: fileName }).promise();
return { status: 200, message: 'Update and delete log file DONE' };
console.log("DEBUG: Uploading the log file");
await cos
.copyObject({
Bucket: BUCKET_ARCHIVE,
CopySource: `${BUCKET_RECEIVER}/${fileName}`,
Key: fileName,
})
.promise();
console.log("DEBUG: Deleting the log file");
await cos
.deleteObject({ Bucket: BUCKET_RECEIVER, Key: fileName })
.promise();
return { status: 200, message: "Update and delete log file DONE" };
} catch (e) {
console.error(e);
return e;
Expand All @@ -85,22 +93,24 @@ async function uploadAndDeleteBucket(fileName) {

function sendLogDNA(json) {
return request({
method: 'POST',
method: "POST",
url: `https://logs.us-south.logging.cloud.ibm.com/logs/ingest?hostname=${HOSTNAME}`,
body: json,
auth: {
user: INGESTION_KEY,
},
headers: { 'Content-Type': 'application/json' },
headers: { "Content-Type": "application/json" },
json: true,
timeout: 18000,
agent: false,
pool: { maxSockets: 200 },
}).then((response) => response).catch(async (e) => {
console.error(e);
console.log('Retrying to send package');
return sendLogDNA(json);
});
})
.then((response) => response)
.catch(async (e) => {
console.error(e);
console.log("Retrying to send package");
return sendLogDNA(json);
});
}

function split(buffer, tag) {
Expand All @@ -109,21 +119,28 @@ function split(buffer, tag) {

while (newBuffer.indexOf(tag) > -1) {
lines.push(newBuffer.slice(0, newBuffer.indexOf(tag)));
newBuffer = newBuffer.slice(newBuffer.indexOf(tag) + tag.length, newBuffer.length);
newBuffer = newBuffer.slice(
newBuffer.indexOf(tag) + tag.length,
newBuffer.length
);
}
lines.push(newBuffer);
return lines;
}

async function downloadAndSend() {
try {
const lo = await cos.listObjectsV2({ Bucket: BUCKET_RECEIVER, MaxKeys: MAX_KEYS }).promise();
const lo = await cos
.listObjectsV2({ Bucket: BUCKET_RECEIVER, MaxKeys: MAX_KEYS })
.promise();
if (lo.Contents.length === 0) {
// Empty Bucket, return a HTTP status code 204 'No Content'
return { status: 204, message: 'No new log file on COS Bucket' };
return { status: 204, message: "No new log file on COS Bucket" };
}
console.log(`DEBUG: log file = ${lo.Contents[0].Key}`);
const o = await cos.getObject({ Bucket: BUCKET_RECEIVER, Key: lo.Contents[0].Key }).promise();
const o = await cos
.getObject({ Bucket: BUCKET_RECEIVER, Key: lo.Contents[0].Key })
.promise();
const buffer = Buffer.from(o.Body);
console.log(`DEBUG: Buffer length = ${buffer.length}`);
if (buffer.length <= 28) {
Expand All @@ -132,35 +149,35 @@ async function downloadAndSend() {
}
const unzipPromise = util.promisify(unzip);
const newBuffer = await unzipPromise(buffer);
const tag = new Buffer.from('}');
const tag = new Buffer.from("}");
const sa = split(newBuffer, tag);
sa.pop();
const fj = { lines: [] };
/* eslint-disable no-await-in-loop */
for (let i = 0; i < sa.length; i += 1) {
sa[i] += '}';
sa[i] += "}";
const json = JSON.parse(sa[i]);
fj.lines.push({
timestamp: new Date().getTime(),
line: '[AUTOMATIC] LOG FROM IBM CLOUD INTERNET SERVICE',
app: 'logdna-cos',
level: 'INFO',
line: "[AUTOMATIC] LOG FROM IBM CLOUD INTERNET SERVICE",
app: "logdna-cos",
level: "INFO",
meta: {
customfield: json,
},
});
if (i % LOGS === 0 || i === (sa.length - 1)) {
console.log(`DEBUG: Sending package = ${(i / LOGS + 1)}`);
if (i % LOGS === 0 || i === sa.length - 1) {
console.log(`DEBUG: Sending package = ${i / LOGS + 1}`);
const response = await sendLogDNA(fj);
console.log(`DEBUG: sendLogDNA response = ${JSON.stringify(response)}`);
// Example response body = {"status":"ok","batchID":""}
if (response && response.status === 'ok') {
if (response && response.status === "ok") {
fj.lines = [];
}
}
}
/* eslint-enable no-await-in-loop */
console.log('DEBUG: uploadAndDeleteBucket');
console.log("DEBUG: uploadAndDeleteBucket");
return await uploadAndDeleteBucket(lo.Contents[0].Key);
} catch (e) {
console.error(e);
Expand All @@ -169,10 +186,10 @@ async function downloadAndSend() {
}

async function main() {
console.time('LogDNA-COS');
console.time("LogDNA-COS");
const response = await downloadAndSend();
console.log(`DEBUG: downloadAndSend = ${JSON.stringify(response.message)}`);
console.timeEnd('LogDNA-COS');
console.timeEnd("LogDNA-COS");
// DEBUG::
// switch (response.status) {
// case 200:
Expand Down
21 changes: 21 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 97499ab

Please sign in to comment.