「🏗️」 wip: ELK workingggggggggggggggggggggggggggggggggggg

Justfile

@@ -13,7 +13,7 @@ set dotenv-load
 
 # To launch all apis
 @apis:
-	node src/dev.js
+	node src/start.js
 
 # To launch the front end
 @front:

docker/ELK/compose.yml

@@ -1,4 +1,4 @@
 include:
   - ./logstash/compose.yml
-  # - ./kibana/compose.yml
-  # - ./elasticsearch/compose.yml
+  - ./kibana/compose.yml
+  - ./elasticsearch/compose.yml

docker/ELK/elasticsearch/Dockerfile

@@ -1,3 +1,4 @@
-FROM		docker.elastic.co/elasticsearch/elasticsearch-wolfi:9.0.3
+FROM		docker.elastic.co/elasticsearch/elasticsearch-wolfi:9.0.4
 
 COPY		--chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/
+COPY		--chown=elasticsearch:elasticsearch jvm.options /usr/share/elasticsearch/config/jvm.options.d/custom.options

docker/ELK/elasticsearch/compose.yml

@@ -6,5 +6,5 @@ services:
       context: .
     environment:
       - LOG_LEVEL=info
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-    mem_limit: 1gb
+    networks:
+      - elk

docker/ELK/elasticsearch/elasticsearch.yml

@@ -0,0 +1,9 @@
+cluster.name: docker-cluster
+
+node.name: transcendence-elasticsearch
+
+discovery.type: single-node
+
+xpack.security.enabled: false
+
+network.host: 0.0.0.0

docker/ELK/elasticsearch/jvm.options

@@ -0,0 +1,2 @@
+-Xms1g
+-Xmx1g

docker/ELK/kibana/Dockerfile

@@ -1 +1,3 @@
-FROM		kibana:9.0.3
+FROM		docker.elastic.co/kibana/kibana-wolfi:9.0.4
+
+COPY		--chmod=777 kibana.yml /etc/kibana/kibana.yml

docker/ELK/kibana/compose.yml

@@ -0,0 +1,12 @@
+services:
+  kibana:
+    container_name: transcendence-kibana
+    build:
+      dockerfile: Dockerfile
+      context: .
+    ports:
+      - ${ELK_PORT}:5601
+    environment:
+      - LOG_LEVEL=info
+    networks:
+      - elk

docker/ELK/kibana/kibana.yml

@@ -0,0 +1,6 @@
+server.name: kibana
+server.host: "0.0.0.0"
+
+elasticsearch.hosts: ["https://transcendence-elasticsearch:9200"]
+
+telemetry.enabled: false

docker/ELK/logstash/Dockerfile

@@ -1,4 +1,4 @@
-FROM		docker.elastic.co/logstash/logstash-wolfi:9.0.3
+FROM		docker.elastic.co/logstash/logstash-wolfi:9.0.4
 
 RUN			rm -f /usr/share/logstash/pipeline/logstash.conf
 

docker/ELK/logstash/compose.yml

@@ -12,3 +12,4 @@ services:
       - LOG_LEVEL=info
     networks:
       - elk
+      - logstash

docker/api-base/compose.yml

@@ -14,6 +14,7 @@ services:
     environment:
       - TZ=Europe/Paris
       - API_TARGET=user
+      - LOG_FILE_PATH=/var/log/log.log
       - JWT_SECRET=${JWT_SECRET}
     restart: unless-stopped
   auth-api:
@@ -31,5 +32,6 @@ services:
     environment:
       - TZ=Europe/Paris
       - API_TARGET=auth
+      - LOG_FILE_PATH=/var/log/log.log
       - JWT_SECRET=${JWT_SECRET}
     restart: unless-stopped

docker/docker-compose.yml

@@ -3,7 +3,7 @@ name: ft_transcendence
 include:
   - ./volumes.yml
   - ./networks.yml
-  - ./monitoring/compose.yml
+  # - ./monitoring/compose.yml
   - ./api-base/compose.yml
   - ./front/compose.yml
   - ./ELK/compose.yml

docker/front/Dockerfile

@@ -31,7 +31,5 @@ RUN			touch /var/log/front/err.log /var/log/front/log.log
 RUN			chmod -R 777 /var/log/front
 USER		nginx
 
-
-
 EXPOSE		80 443
 STOPSIGNAL	SIGINT

docker/front/compose.yml

@@ -9,7 +9,6 @@ services:
     volumes:
       - log-nginx:/var/log/front
     environment:
-      - SERVER_NAME=localhost
       - TZ=Europe/Paris
     depends_on:
       user-api:

docker/monitoring/prometheus/config/prometheus.yml

@@ -24,7 +24,7 @@ scrape_configs:
     static_configs:
       - targets: ['node-exporter:9100']
 
-  - job_name: 'nodejs'
-    static_configs:
-      - targets: ['transcendence-api-auth:3000']
-      - targets: ['transcendence-api-user:3000']
+  # - job_name: 'nodejs'
+  #   static_configs:
+  #     - targets: ['transcendence-api-auth:3000']
+  #     - targets: ['transcendence-api-user:3000']

docker/networks.yml

@@ -9,3 +9,5 @@ networks:
     name: transcendence-prom-exporter
   elk:
     name: transcendence-elk
+  logstash:
+    name: transcendence-logstash

package.json

@@ -10,6 +10,8 @@
 		"fastify": "^5.4.0",
 		"fastify-cli": "^7.4.0",
 		"google-auth-library": "^10.1.0",
+		"pino": "^9.7.0",
+		"pino-logstash": "^1.0.0",
 		"prom-client": "^15.1.3",
 		"solhint": "^6.0.0"
 	},

pnpm-lock.yaml

@@ -38,6 +38,12 @@ importers:
       google-auth-library:
         specifier: ^10.1.0
         version: 10.1.0
+      pino:
+        specifier: ^9.7.0
+        version: 9.7.0
+      pino-logstash:
+        specifier: ^1.0.0
+        version: 1.0.0
       prom-client:
         specifier: ^15.1.3
         version: 15.1.3
@@ -1328,6 +1334,9 @@ packages:
   pino-abstract-transport@2.0.0:
     resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==}
 
+  pino-logstash@1.0.0:
+    resolution: {integrity: sha512-v8UCUxGROClKZW6mB0GumsfCi3gnmupMNDzh86TX9Oab9ijDjIW3vJdMjlcNaAyJK9GiJxUdLtI6amzLhpuARg==}
+
   pino-pretty@13.0.0:
     resolution: {integrity: sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA==}
     hasBin: true
@@ -2785,6 +2794,8 @@ snapshots:
     dependencies:
       split2: 4.2.0
 
+  pino-logstash@1.0.0: {}
+
   pino-pretty@13.0.0:
     dependencies:
       colorette: 2.0.20

src/api/auth/default.js

@@ -1,6 +1,5 @@
 import fastifyJWT from '@fastify/jwt';
 import fastifyCookie from '@fastify/cookie';
-import client from 'prom-client';
 
 import { register } from './register.js';
 import { login } from './login.js';
@@ -14,7 +13,6 @@ import { totpVerify } from './totpVerify.js';
 
 const saltRounds = 10;
 export const appName = process.env.APP_NAME || 'knl_meowscendence';
-const collectDefaultMetrics = client.collectDefaultMetrics
 
 authDB.prepareDB();
 
@@ -24,29 +22,6 @@ authDB.prepareDB();
  */
 export default async function(fastify, options) {
 
-	collectDefaultMetrics({ labels: { service: "auth-api" } })
-	client.register.setDefaultLabels({ service: "auth-api" })
-
-	const httpRequestCounter = new client.Counter({
-		name: 'http_requests_total',
-		help: 'Total number of HTTP requests',
-		labelNames: ['method', 'route', 'status_code'],
-	})
-
-	fastify.addHook('onResponse', (req, res, done) => {
-		httpRequestCounter.inc({
-			method: req.method,
-			route: req.routerPath || req.url,
-			status_code: res.statusCode,
-		})
-		done()
-	})
-	fastify.get('/metrics', async (req, reply) => {
-		reply
-			.header('Content-Type', client.register.contentType)
-			.send(await client.register.metrics())
-	})
-
 	fastify.register(fastifyJWT, {
 		secret: process.env.JWT_SECRET || '123456789101112131415161718192021',
 		cookie: {

src/api/user/default.js

@@ -1,10 +1,8 @@
 import fastifyJWT from '@fastify/jwt';
 import fastifyCookie from '@fastify/cookie';
 import Database from 'better-sqlite3';
-import client from 'prom-client';
 
 var env = process.env.NODE_ENV || 'development';
-const collectDefaultMetrics = client.collectDefaultMetrics
 
 let database;
 
@@ -58,31 +56,6 @@ const deleteFriends = database.prepare('DELETE FROM friends WHERE username = ?;'
  * @param {import('fastify').FastifyPluginOptions}	options
  */
 export default async function(fastify, options) {
-
-	collectDefaultMetrics({ labels: { service: "auth-api" } })
-	client.register.setDefaultLabels({ service: "auth-api" })
-
-	const httpRequestCounter = new client.Counter({
-		name: 'http_requests_total',
-		help: 'Total number of HTTP requests',
-		labelNames: ['method', 'route', 'status_code'],
-	})
-
-	fastify.addHook('onResponse', (req, res, done) => {
-		httpRequestCounter.inc({
-			method: req.method,
-			route: req.routerPath || req.url,
-			status_code: res.statusCode,
-		})
-		done()
-	})
-	fastify.get('/metrics', async (req, reply) => {
-		reply
-			.header('Content-Type', client.register.contentType)
-			.send(await client.register.metrics())
-	})
-
-
 	fastify.register(fastifyJWT, {
 		secret: process.env.JWT_SECRET || '123456789101112131415161718192021',
 		cookie: {

src/start.js

@@ -1,53 +1,71 @@
 import Fastify from 'fastify';
 import authApi from './api/auth/default.js';
 import userApi from './api/user/default.js';
+import fs from 'fs';
+import path from 'path';
 
-const loggerOption = {
-	transport: {
-		target: 'pino-pretty',
-		options: {
-			colorize: true,
-			translateTime: 'HH:MM:ss',
-			ignore: 'pid,hostname'
-		}
-	},
-	file: process.env.LOG_TARGET || ""
+const isProduction = process.env.NODE_ENV === 'production';
+const logFilePath = process.env.LOG_FILE_PATH || './logs/api.log';
+
+const loggerOption = () => {
+  if (!isProduction) {
+    return {
+      transport: {
+        target: 'pino-pretty',
+        options: {
+          colorize: true,
+          translateTime: 'HH:MM:ss',
+          ignore: 'pid,hostname',
+        },
+      },
+    };
+  } else {
+    // Make sure the directory exists
+    const logDir = path.dirname(logFilePath);
+    fs.mkdirSync(logDir, { recursive: true });
+
+    const logStream = fs.createWriteStream(logFilePath, { flags: 'a' }); // append mode
+    return {
+      level: 'info',
+      stream: logStream,
+    };
+  }
 };
 
-function sigHandle(signal) {
-	process.exit(0);
-}
-
-process.on('SIGINT', sigHandle);
-
 async function start() {
 	const target = process.env.API_TARGET || 'all';
 
+	const servers = [];
+
 	if (target === 'auth' || target === 'all') {
-		const auth = Fastify({ logger: loggerOption });
+		const auth = Fastify({ logger: loggerOption('auth') });
 		auth.register(authApi);
-		if (target !== 'all') {
-			await auth.listen({ port: 3000, host: '0.0.0.0' });
-			console.log('Auth API listening on http://0.0.0.0:3000');
-		}
-		else {
-			await auth.listen({ port: 3001, host: '127.0.0.1'});
-			console.log('Auth API listening on http://localhost:3001');
-		}
+		const port = target === 'all' ? 3001 : 3000;
+		const host = target === 'all' ? '127.0.0.1' : '0.0.0.0';
+		await auth.listen({ port, host });
+		console.log(`Auth API listening on http://${host}:${port}`);
+		servers.push(auth);
 	}
 
 	if (target === 'user' || target === 'all') {
-		const user = Fastify({ logger: loggerOption });
+		const user = Fastify({ logger: loggerOption('user') });
 		user.register(userApi);
-		if (target !== 'all') {
-			await user.listen({ port: 3000, host: '0.0.0.0' });
-			console.log('User API listening on http://0.0.0.0:3000');
-		}
-		else {
-			await user.listen({ port: 3002, host: '127.0.0.1'});
-			console.log('User API listening on http://localhost:3002');
-		}
+		const port = target === 'all' ? 3002 : 3000;
+		const host = target === 'all' ? '127.0.0.1' : '0.0.0.0';
+		await user.listen({ port, host });
+		console.log(`User API listening on http://${host}:${port}`);
+		servers.push(user);
 	}
+
+	// Graceful shutdown on SIGINT
+	process.on('SIGINT', async () => {
+		console.log('SIGINT received, closing servers...');
+		await Promise.all(servers.map((srv) => srv.close()));
+		process.exit(0);
+	});
 }
 
-start().catch(console.error);
+start().catch((err) => {
+	console.error(err);
+	process.exit(1);
+});