diff --git a/Justfile b/Justfile index 6a884a6..a5736f1 100644 --- a/Justfile +++ b/Justfile @@ -13,7 +13,7 @@ set dotenv-load # To launch all apis @apis: - node src/dev.js + node src/start.js # To launch the front end @front: diff --git a/docker/ELK/compose.yml b/docker/ELK/compose.yml index 16ab019..46182c7 100644 --- a/docker/ELK/compose.yml +++ b/docker/ELK/compose.yml @@ -1,4 +1,4 @@ include: - ./logstash/compose.yml - # - ./kibana/compose.yml - # - ./elasticsearch/compose.yml + - ./kibana/compose.yml + - ./elasticsearch/compose.yml diff --git a/docker/ELK/elasticsearch/Dockerfile b/docker/ELK/elasticsearch/Dockerfile index d5cee85..74b087a 100644 --- a/docker/ELK/elasticsearch/Dockerfile +++ b/docker/ELK/elasticsearch/Dockerfile @@ -1,3 +1,4 @@ -FROM docker.elastic.co/elasticsearch/elasticsearch-wolfi:9.0.3 +FROM docker.elastic.co/elasticsearch/elasticsearch-wolfi:9.0.4 COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/ +COPY --chown=elasticsearch:elasticsearch jvm.options /usr/share/elasticsearch/config/jvm.options.d/custom.options diff --git a/docker/ELK/elasticsearch/compose.yml b/docker/ELK/elasticsearch/compose.yml index 6341ac2..36ab9d2 100644 --- a/docker/ELK/elasticsearch/compose.yml +++ b/docker/ELK/elasticsearch/compose.yml @@ -6,5 +6,5 @@ services: context: . environment: - LOG_LEVEL=info - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - mem_limit: 1gb + networks: + - elk diff --git a/docker/ELK/elasticsearch/elasticsearch.yml b/docker/ELK/elasticsearch/elasticsearch.yml index e69de29..30b5f04 100644 --- a/docker/ELK/elasticsearch/elasticsearch.yml +++ b/docker/ELK/elasticsearch/elasticsearch.yml @@ -0,0 +1,9 @@ +cluster.name: docker-cluster + +node.name: transcendence-elasticsearch + +discovery.type: single-node + +xpack.security.enabled: false + +network.host: 0.0.0.0 diff --git a/docker/ELK/elasticsearch/jvm.options b/docker/ELK/elasticsearch/jvm.options new file mode 100644 index 0000000..e292d3f --- /dev/null +++ b/docker/ELK/elasticsearch/jvm.options @@ -0,0 +1,2 @@ +-Xms1g +-Xmx1g diff --git a/docker/ELK/kibana/Dockerfile b/docker/ELK/kibana/Dockerfile index 78fcc00..0ab577c 100644 --- a/docker/ELK/kibana/Dockerfile +++ b/docker/ELK/kibana/Dockerfile @@ -1 +1,3 @@ -FROM kibana:9.0.3 +FROM docker.elastic.co/kibana/kibana-wolfi:9.0.4 + +COPY --chmod=777 kibana.yml /etc/kibana/kibana.yml diff --git a/docker/ELK/kibana/compose.yml b/docker/ELK/kibana/compose.yml index e69de29..7a4cb52 100644 --- a/docker/ELK/kibana/compose.yml +++ b/docker/ELK/kibana/compose.yml @@ -0,0 +1,12 @@ +services: + kibana: + container_name: transcendence-kibana + build: + dockerfile: Dockerfile + context: . + ports: + - ${ELK_PORT}:5601 + environment: + - LOG_LEVEL=info + networks: + - elk diff --git a/docker/ELK/kibana/kibana.yml b/docker/ELK/kibana/kibana.yml new file mode 100644 index 0000000..a24e16a --- /dev/null +++ b/docker/ELK/kibana/kibana.yml @@ -0,0 +1,6 @@ +server.name: kibana +server.host: "0.0.0.0" + +elasticsearch.hosts: ["https://transcendence-elasticsearch:9200"] + +telemetry.enabled: false diff --git a/docker/ELK/logstash/Dockerfile b/docker/ELK/logstash/Dockerfile index 03cb881..58d1167 100644 --- a/docker/ELK/logstash/Dockerfile +++ b/docker/ELK/logstash/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash-wolfi:9.0.3 +FROM docker.elastic.co/logstash/logstash-wolfi:9.0.4 RUN rm -f /usr/share/logstash/pipeline/logstash.conf diff --git a/docker/ELK/logstash/compose.yml b/docker/ELK/logstash/compose.yml index 700e695..be01856 100644 --- a/docker/ELK/logstash/compose.yml +++ b/docker/ELK/logstash/compose.yml @@ -12,3 +12,4 @@ services: - LOG_LEVEL=info networks: - elk + - logstash diff --git a/docker/api-base/compose.yml b/docker/api-base/compose.yml index df8ce09..9d33dc7 100644 --- a/docker/api-base/compose.yml +++ b/docker/api-base/compose.yml @@ -14,6 +14,7 @@ services: environment: - TZ=Europe/Paris - API_TARGET=user + - LOG_FILE_PATH=/var/log/log.log - JWT_SECRET=${JWT_SECRET} restart: unless-stopped auth-api: @@ -31,5 +32,6 @@ services: environment: - TZ=Europe/Paris - API_TARGET=auth + - LOG_FILE_PATH=/var/log/log.log - JWT_SECRET=${JWT_SECRET} restart: unless-stopped diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9e49825..77718d0 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -3,7 +3,7 @@ name: ft_transcendence include: - ./volumes.yml - ./networks.yml - - ./monitoring/compose.yml + # - ./monitoring/compose.yml - ./api-base/compose.yml - ./front/compose.yml - ./ELK/compose.yml diff --git a/docker/front/Dockerfile b/docker/front/Dockerfile index c900cf6..a60b3e9 100644 --- a/docker/front/Dockerfile +++ b/docker/front/Dockerfile @@ -31,7 +31,5 @@ RUN touch /var/log/front/err.log /var/log/front/log.log RUN chmod -R 777 /var/log/front USER nginx - - EXPOSE 80 443 STOPSIGNAL SIGINT diff --git a/docker/front/compose.yml b/docker/front/compose.yml index 593bf8b..22b500e 100644 --- a/docker/front/compose.yml +++ b/docker/front/compose.yml @@ -9,7 +9,6 @@ services: volumes: - log-nginx:/var/log/front environment: - - SERVER_NAME=localhost - TZ=Europe/Paris depends_on: user-api: diff --git a/docker/monitoring/prometheus/config/prometheus.yml b/docker/monitoring/prometheus/config/prometheus.yml index 84ec958..e3789e5 100644 --- a/docker/monitoring/prometheus/config/prometheus.yml +++ b/docker/monitoring/prometheus/config/prometheus.yml @@ -24,7 +24,7 @@ scrape_configs: static_configs: - targets: ['node-exporter:9100'] - - job_name: 'nodejs' - static_configs: - - targets: ['transcendence-api-auth:3000'] - - targets: ['transcendence-api-user:3000'] + # - job_name: 'nodejs' + # static_configs: + # - targets: ['transcendence-api-auth:3000'] + # - targets: ['transcendence-api-user:3000'] diff --git a/docker/networks.yml b/docker/networks.yml index 7e28b60..65a1c1e 100644 --- a/docker/networks.yml +++ b/docker/networks.yml @@ -9,3 +9,5 @@ networks: name: transcendence-prom-exporter elk: name: transcendence-elk + logstash: + name: transcendence-logstash diff --git a/package.json b/package.json index 149bcc0..c980981 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "fastify": "^5.4.0", "fastify-cli": "^7.4.0", "google-auth-library": "^10.1.0", + "pino": "^9.7.0", + "pino-logstash": "^1.0.0", "prom-client": "^15.1.3", "solhint": "^6.0.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8c12749..ef4dd58 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -38,6 +38,12 @@ importers: google-auth-library: specifier: ^10.1.0 version: 10.1.0 + pino: + specifier: ^9.7.0 + version: 9.7.0 + pino-logstash: + specifier: ^1.0.0 + version: 1.0.0 prom-client: specifier: ^15.1.3 version: 15.1.3 @@ -1328,6 +1334,9 @@ packages: pino-abstract-transport@2.0.0: resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==} + pino-logstash@1.0.0: + resolution: {integrity: sha512-v8UCUxGROClKZW6mB0GumsfCi3gnmupMNDzh86TX9Oab9ijDjIW3vJdMjlcNaAyJK9GiJxUdLtI6amzLhpuARg==} + pino-pretty@13.0.0: resolution: {integrity: sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA==} hasBin: true @@ -2785,6 +2794,8 @@ snapshots: dependencies: split2: 4.2.0 + pino-logstash@1.0.0: {} + pino-pretty@13.0.0: dependencies: colorette: 2.0.20 diff --git a/src/api/auth/default.js b/src/api/auth/default.js index b8a9063..6a15651 100644 --- a/src/api/auth/default.js +++ b/src/api/auth/default.js @@ -1,6 +1,5 @@ import fastifyJWT from '@fastify/jwt'; import fastifyCookie from '@fastify/cookie'; -import client from 'prom-client'; import { register } from './register.js'; import { login } from './login.js'; @@ -14,7 +13,6 @@ import { totpVerify } from './totpVerify.js'; const saltRounds = 10; export const appName = process.env.APP_NAME || 'knl_meowscendence'; -const collectDefaultMetrics = client.collectDefaultMetrics authDB.prepareDB(); @@ -24,29 +22,6 @@ authDB.prepareDB(); */ export default async function(fastify, options) { - collectDefaultMetrics({ labels: { service: "auth-api" } }) - client.register.setDefaultLabels({ service: "auth-api" }) - - const httpRequestCounter = new client.Counter({ - name: 'http_requests_total', - help: 'Total number of HTTP requests', - labelNames: ['method', 'route', 'status_code'], - }) - - fastify.addHook('onResponse', (req, res, done) => { - httpRequestCounter.inc({ - method: req.method, - route: req.routerPath || req.url, - status_code: res.statusCode, - }) - done() - }) - fastify.get('/metrics', async (req, reply) => { - reply - .header('Content-Type', client.register.contentType) - .send(await client.register.metrics()) - }) - fastify.register(fastifyJWT, { secret: process.env.JWT_SECRET || '123456789101112131415161718192021', cookie: { diff --git a/src/api/user/default.js b/src/api/user/default.js index 34ef89d..a5efe65 100644 --- a/src/api/user/default.js +++ b/src/api/user/default.js @@ -1,10 +1,8 @@ import fastifyJWT from '@fastify/jwt'; import fastifyCookie from '@fastify/cookie'; import Database from 'better-sqlite3'; -import client from 'prom-client'; var env = process.env.NODE_ENV || 'development'; -const collectDefaultMetrics = client.collectDefaultMetrics let database; @@ -58,31 +56,6 @@ const deleteFriends = database.prepare('DELETE FROM friends WHERE username = ?;' * @param {import('fastify').FastifyPluginOptions} options */ export default async function(fastify, options) { - - collectDefaultMetrics({ labels: { service: "auth-api" } }) - client.register.setDefaultLabels({ service: "auth-api" }) - - const httpRequestCounter = new client.Counter({ - name: 'http_requests_total', - help: 'Total number of HTTP requests', - labelNames: ['method', 'route', 'status_code'], - }) - - fastify.addHook('onResponse', (req, res, done) => { - httpRequestCounter.inc({ - method: req.method, - route: req.routerPath || req.url, - status_code: res.statusCode, - }) - done() - }) - fastify.get('/metrics', async (req, reply) => { - reply - .header('Content-Type', client.register.contentType) - .send(await client.register.metrics()) - }) - - fastify.register(fastifyJWT, { secret: process.env.JWT_SECRET || '123456789101112131415161718192021', cookie: { diff --git a/src/start.js b/src/start.js index 3a0e40a..e76f7d9 100644 --- a/src/start.js +++ b/src/start.js @@ -1,53 +1,71 @@ import Fastify from 'fastify'; import authApi from './api/auth/default.js'; import userApi from './api/user/default.js'; +import fs from 'fs'; +import path from 'path'; -const loggerOption = { - transport: { - target: 'pino-pretty', - options: { - colorize: true, - translateTime: 'HH:MM:ss', - ignore: 'pid,hostname' - } - }, - file: process.env.LOG_TARGET || "" +const isProduction = process.env.NODE_ENV === 'production'; +const logFilePath = process.env.LOG_FILE_PATH || './logs/api.log'; + +const loggerOption = () => { + if (!isProduction) { + return { + transport: { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'HH:MM:ss', + ignore: 'pid,hostname', + }, + }, + }; + } else { + // Make sure the directory exists + const logDir = path.dirname(logFilePath); + fs.mkdirSync(logDir, { recursive: true }); + + const logStream = fs.createWriteStream(logFilePath, { flags: 'a' }); // append mode + return { + level: 'info', + stream: logStream, + }; + } }; -function sigHandle(signal) { - process.exit(0); -} - -process.on('SIGINT', sigHandle); - async function start() { const target = process.env.API_TARGET || 'all'; + const servers = []; + if (target === 'auth' || target === 'all') { - const auth = Fastify({ logger: loggerOption }); + const auth = Fastify({ logger: loggerOption('auth') }); auth.register(authApi); - if (target !== 'all') { - await auth.listen({ port: 3000, host: '0.0.0.0' }); - console.log('Auth API listening on http://0.0.0.0:3000'); - } - else { - await auth.listen({ port: 3001, host: '127.0.0.1'}); - console.log('Auth API listening on http://localhost:3001'); - } + const port = target === 'all' ? 3001 : 3000; + const host = target === 'all' ? '127.0.0.1' : '0.0.0.0'; + await auth.listen({ port, host }); + console.log(`Auth API listening on http://${host}:${port}`); + servers.push(auth); } if (target === 'user' || target === 'all') { - const user = Fastify({ logger: loggerOption }); + const user = Fastify({ logger: loggerOption('user') }); user.register(userApi); - if (target !== 'all') { - await user.listen({ port: 3000, host: '0.0.0.0' }); - console.log('User API listening on http://0.0.0.0:3000'); - } - else { - await user.listen({ port: 3002, host: '127.0.0.1'}); - console.log('User API listening on http://localhost:3002'); - } + const port = target === 'all' ? 3002 : 3000; + const host = target === 'all' ? '127.0.0.1' : '0.0.0.0'; + await user.listen({ port, host }); + console.log(`User API listening on http://${host}:${port}`); + servers.push(user); } + + // Graceful shutdown on SIGINT + process.on('SIGINT', async () => { + console.log('SIGINT received, closing servers...'); + await Promise.all(servers.map((srv) => srv.close())); + process.exit(0); + }); } -start().catch(console.error); +start().catch((err) => { + console.error(err); + process.exit(1); +});