From 6439f8eb92d41d6a2980906ece6847b00eeca2bf Mon Sep 17 00:00:00 2001 From: Klemek Date: Tue, 30 Mar 2021 19:11:31 +0200 Subject: [PATCH] hit-counter --- README.md | 9 ++ package-lock.json | 85 ++++++++++++ package.json | 1 + src/app.js | 91 ++++++++----- src/config.default.json | 10 +- src/hit_counter.js | 45 ++++++- test/app.test.js | 50 +++++++ test/hit_counter.test.js | 281 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 529 insertions(+), 43 deletions(-) create mode 100644 test/hit_counter.test.js diff --git a/README.md b/README.md index 5d42b1b..fa3b851 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,8 @@ Any URL like `/year/month/day/anything/` will redirect to this article (and link activate PlantUML diagram rendering * `fa-diagrams` (default: true) activate fa-diagrams rendering + * `hit_counter` (default: true) + activate /stats endpoints and visitor counting (need an active redis connection) * `home` * `title` (default: GitBlog.md) the title of your blog, **strongly advised to be changed** @@ -332,4 +334,11 @@ Any URL like `/year/month/day/anything/` will redirect to this article (and link specify the output format between svg, html or MathMl (mml) * `speak_text`: (default: true) activate the alternate text in equations +* `hit_counter` + * `unique_visitor_timeout`: (default: 7200000 / 2h) + specify the time (in ms) before a visitor can be accounted again +* `redis` + Options to connect to redis (see [redis options](https://github.com/NodeRedis/node-redis#options-object-properties) for more info) + * `host`: (default: localhost) + * `port`: (default: 6379) diff --git a/package-lock.json b/package-lock.json index 1dad291..9414459 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "ncp": "^2.0.0", "node-prismjs": "^0.1.0", "prismjs": "^1.23.0", + "redis": "^3.0.2", "rss": "^1.2.2", "showdown": "^1.9.1" }, @@ -1897,6 +1898,14 @@ "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==", "optional": true }, + "node_modules/denque": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -6966,6 +6975,48 @@ "node": ">=4" } }, + "node_modules/redis": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz", + "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==", + "dependencies": { + "denque": "^1.4.1", + "redis-commands": "^1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-redis" + } + }, + "node_modules/redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", @@ -10365,6 +10416,11 @@ "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==", "optional": true }, + "denque": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz", + "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==" + }, "depd": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", @@ -14305,6 +14361,35 @@ "util.promisify": "^1.0.0" } }, + "redis": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz", + "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==", + "requires": { + "denque": "^1.4.1", + "redis-commands": "^1.5.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0" + } + }, + "redis-commands": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", + "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==" + }, + "redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=" + }, + "redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=", + "requires": { + "redis-errors": "^1.0.0" + } + }, "regex-not": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", diff --git a/package.json b/package.json index abcada1..12f4a62 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "ncp": "^2.0.0", "node-prismjs": "^0.1.0", "prismjs": "^1.23.0", + "redis": "^3.0.2", "rss": "^1.2.2", "showdown": "^1.9.1" }, diff --git a/src/app.js b/src/app.js index 341128d..5dcea61 100644 --- a/src/app.js +++ b/src/app.js @@ -51,7 +51,16 @@ module.exports = (config) => { let showError; const fw = require('./file_walker')(config); const renderer = require('./renderer')(config); - const hc = require('./hit_counter')(config); + const hc = require('./hit_counter')(config, + () => { + console.log(cons.ok, 'redis connected'); + }, + (err) => { + if (err.code !== 'ECONNREFUSED') { + console.log(cons.warn, 'redis error: ' + err); + } + }, + ); // set view engine from configuration app.set('view engine', config['view_engine']); @@ -159,22 +168,28 @@ module.exports = (config) => { if (err) { showError(req, res, 404); } else { - hc.count(req, '/'); - render(req, res, homePath, - { - articles: Object.values(articles) - .filter(d => !d.draft) - .sort((a, b) => ('' + b.path).localeCompare(a.path)), - }); + hc.count(req, '/', () => { + render(req, res, homePath, + { + articles: Object.values(articles) + .filter(d => !d.draft) + .sort((a, b) => ('' + b.path).localeCompare(a.path)), + }); + }); } }); }); app.get('/stats', (req, res) => { - const data = hc.read('/'); - res.json({ - hits: data.hits, - visitors: data.visitors, - }); + if (config['modules']['hit_counter']) { + hc.read('/', (data) => { + res.json({ + hits: data.hits, + visitors: data.visitors, + }); + }); + } else { + showError(req, res, 404); + } }); //RSS endpoint @@ -251,29 +266,35 @@ module.exports = (config) => { if (!article) { showError(req, res, 404); } else if (req.path.endsWith('stats')) { - const data = hc.read(articlePath); - res.json({ - hits: data.hits, - visitors: data.visitors, - }); - } else { - hc.count(req, articlePath); - renderer.render(article.realPath, (err, html) => { - if (err) { - console.log(cons.error, `failed to render article ${req.path} : ${err}`); - showError(req, res, 500); - } else { - article.content = html; - const templatePath = path.join(config['data_dir'], config['article']['template']); - fs.access(templatePath, fs.constants.R_OK, (err) => { - if (err) { - console.log(cons.error, `no template found at ${templatePath}`); - showError(req, res, 500); - } else { - render(req, res, templatePath, { article: article }); - } + if (config['modules']['hit_counter']) { + hc.read(articlePath, (data) => { + res.json({ + hits: data.hits, + visitors: data.visitors, }); - } + }); + } else { + showError(req, res, 404); + } + } else { + hc.count(req, articlePath, () => { + renderer.render(article.realPath, (err, html) => { + if (err) { + console.log(cons.error, `failed to render article ${req.path} : ${err}`); + showError(req, res, 500); + } else { + article.content = html; + const templatePath = path.join(config['data_dir'], config['article']['template']); + fs.access(templatePath, fs.constants.R_OK, (err) => { + if (err) { + console.log(cons.error, `no template found at ${templatePath}`); + showError(req, res, 500); + } else { + render(req, res, templatePath, { article: article }); + } + }); + } + }); }); } } else { diff --git a/src/config.default.json b/src/config.default.json index f0d0678..49565f9 100644 --- a/src/config.default.json +++ b/src/config.default.json @@ -12,7 +12,8 @@ "prism": true, "mathjax": true, "plantuml": true, - "fa-diagrams": true + "fa-diagrams": true, + "hit_counter": true }, "home": { "title": "GitBlog.md", @@ -58,5 +59,12 @@ }, "plantuml": { "output_format": "svg" + }, + "hit_counter": { + "unique_visitor_timeout": 7200000 + }, + "redis": { + "host": "localhost", + "port": 6379 } } diff --git a/src/hit_counter.js b/src/hit_counter.js index df922f7..e36b22c 100644 --- a/src/hit_counter.js +++ b/src/hit_counter.js @@ -1,13 +1,44 @@ -module.exports = (config) => { - const count = (req, path) => { +const redis = require('redis'); +module.exports = (config, onConnect, onError) => { + const client = config['modules']['hit_counter'] ? redis.createClient(config['redis']) : { connected: false, on: () => { /* ignore */ } }; + + client.on('connect', onConnect); + client.on('error', onError); + + const visitors = {}; + + const count = (req, path, cb) => { + if (!client.connected) { + cb(); + } else { + const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress; + const key = path + ':' + ip; + const now = Date.now(); + const isNewVisitor = (now - (visitors[key] || 0)) > config['hit_counter']['unique_visitor_timeout']; + visitors[key] = now; + client + .multi() + .hincrby(path, 'h', 1) + .hincrby(path, 'v', isNewVisitor ? 1 : 0) + .exec(cb); + } }; - const read = (path) => { - return { - hits: 0, - visitors: 0, - }; + const read = (path, cb) => { + if (!client.connected) { + cb({ + hits: 0, + visitors: 0, + }); + } else { + client.hgetall(path, (_, value) => { + cb({ + hits: value ? value.h || 0 : 0, + visitors: value ? value.v || 0 : 0, + }); + }); + } }; return { diff --git a/test/app.test.js b/test/app.test.js index 96a24ec..b8e0103 100644 --- a/test/app.test.js +++ b/test/app.test.js @@ -17,6 +17,7 @@ config['rss']['endpoint'] = '/rsstest'; config['rss']['length'] = 2; config['home']['error'] = testError; config['article']['template'] = testTemplate; +config['modules']['hit_counter'] = false; const app = require('../src/app')(config); @@ -28,6 +29,7 @@ beforeEach((done, fail) => { config['error_log'] = ''; config['modules']['rss'] = true; config['modules']['webhook'] = true; + config['modules']['hit_counter'] = false; utils.deleteFolderSync(dataDir); fs.mkdirSync(dataDir); @@ -195,6 +197,22 @@ describe('Test root path', () => { }); }, fail); }); + test('404 index no stats', (done) => { + request(app).get('/stats') + .then((response) => { + expect(response.statusCode).toBe(404); + done(); + }); + }); + test('200 index stats', (done) => { + config['modules']['hit_counter'] = true; + request(app).get('/stats') + .then((response) => { + expect(response.statusCode).toBe(200); + expect(response.body).toEqual({ hits: 0, visitors: 0 }); + done(); + }); + }); }); describe('Test RSS feed', () => { @@ -433,6 +451,38 @@ describe('Test articles rendering', () => { }); }, fail); }); + + test('404 article no stats', (done) => { + utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]); + utils.createEmptyFiles([ + path.join(dataDir, '2019', '05', '05', 'index.md'), + path.join(dataDir, testTemplate), + ]); + app.reload(() => { + request(app).get('/2019/05/05/hello/stats') + .then((response) => { + expect(response.statusCode).toBe(404); + done(); + }); + }); + }); + + test('200 index stats', (done) => { + config['modules']['hit_counter'] = true; + utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]); + utils.createEmptyFiles([ + path.join(dataDir, '2019', '05', '05', 'index.md'), + path.join(dataDir, testTemplate), + ]); + app.reload(() => { + request(app).get('/2019/05/05/anything/stats') + .then((response) => { + expect(response.statusCode).toBe(200); + expect(response.body).toEqual({ hits: 0, visitors: 0 }); + done(); + }); + }); + }); }); diff --git a/test/hit_counter.test.js b/test/hit_counter.test.js new file mode 100644 index 0000000..a4afaad --- /dev/null +++ b/test/hit_counter.test.js @@ -0,0 +1,281 @@ +const mockClient = { + options: {}, + connected: true, + on: () => { /* ignore */ }, +}; + +jest.mock('redis', () => { + return { + createClient: (options) => { + mockClient.options = options; + return mockClient; + }, + }; +}); + +const config = { + test: true, + modules: { + hit_counter: true, + }, + redis: { + host: 'test-host', + port: 'test-port', + }, + hit_counter: { + unique_visitor_timeout: -1, + }, +}; + +const hc = require('../src/hit_counter')(config, () => { /* ignore */ }, () => { /* ignore */ }); + +afterAll(() => { + jest.resetModules(); +}); + +test('options passed to redis', () => { + expect(mockClient.options).toEqual(config['redis']); +}); + +describe('read()', () => { + beforeEach(() => { + mockClient.hgetall = (_, cb) => { + cb(); + }; + }); + + test('read path', (done) => { + mockClient.hgetall = (path, cb) => { + expect(path).toBe('/test/path/'); + cb(undefined, { h: 12, v: 34 }); + }; + hc.read('/test/path/', (data) => { + expect(data).toBeDefined(); + expect(data.hits).toBe(12); + expect(data.visitors).toBe(34); + done(); + }); + }); + + test('read path with error', (done) => { + mockClient.hgetall = (path, cb) => { + expect(path).toBe('/test/path/'); + cb('error', undefined); + }; + hc.read('/test/path/', (data) => { + expect(data).toBeDefined(); + expect(data.hits).toBe(0); + expect(data.visitors).toBe(0); + done(); + }); + }); + + test('read path with error 2', (done) => { + mockClient.hgetall = (path, cb) => { + expect(path).toBe('/test/path/'); + cb(undefined, {}); + }; + hc.read('/test/path/', (data) => { + expect(data).toBeDefined(); + expect(data.hits).toBe(0); + expect(data.visitors).toBe(0); + done(); + }); + }); +}); + +describe('count()', () => { + beforeEach(() => { + mockClient.multi = () => mockClient; + mockClient.hincrby = () => mockClient; + mockClient.exec = (cb) => { + cb(); + }; + }); + + test('simple visit', (done) => { + let multiCalled = false; + let execCalled = false; + let hincrbyCalls = []; + mockClient.multi = () => { + multiCalled = true; + return mockClient; + }; + mockClient.hincrby = (hash, key, value) => { + hincrbyCalls.push([ + hash, + key, + value, + ]); + return mockClient; + }; + mockClient.exec = (cb) => { + execCalled = true; + cb(); + }; + hc.count({ + headers: {}, + connection: { remoteAddress: 'test1' }, + }, '/test/path/1', () => { + expect(multiCalled).toBeTruthy(); + expect(hincrbyCalls).toEqual([ + [ + '/test/path/1', + 'h', + 1, + ], + [ + '/test/path/1', + 'v', + 1, + ], + ]); + expect(execCalled).toBeTruthy(); + done(); + }); + }); +}); + +describe('count()', () => { + beforeEach(() => { + mockClient.multi = () => mockClient; + mockClient.hincrby = () => mockClient; + mockClient.exec = (cb) => { + cb(); + }; + config['hit_counter']['unique_visitor_timeout'] = -1; + }); + + test('simple visit', (done) => { + let multiCalled = false; + let execCalled = false; + let hincrbyCalls = []; + mockClient.multi = () => { + multiCalled = true; + return mockClient; + }; + mockClient.hincrby = (hash, key, value) => { + hincrbyCalls.push([ + hash, + key, + value, + ]); + return mockClient; + }; + mockClient.exec = (cb) => { + execCalled = true; + cb(); + }; + hc.count({ + headers: {}, + connection: { remoteAddress: 'test1' }, + }, '/test/path/1', () => { + expect(multiCalled).toBeTruthy(); + expect(hincrbyCalls).toEqual([ + [ + '/test/path/1', + 'h', + 1, + ], + [ + '/test/path/1', + 'v', + 1, + ], + ]); + expect(execCalled).toBeTruthy(); + done(); + }); + }); + + test('re-visit after long time', (done) => { + let hincrbyCalls = []; + mockClient.hincrby = (hash, key, value) => { + hincrbyCalls.push([ + hash, + key, + value, + ]); + return mockClient; + }; + hc.count({ + headers: {}, + connection: { remoteAddress: 'test2' }, + }, '/test/path/2', () => { + hc.count({ + headers: {}, + connection: { remoteAddress: 'test2' }, + }, '/test/path/2', () => { + expect(hincrbyCalls).toEqual([ + [ + '/test/path/2', + 'h', + 1, + ], + [ + '/test/path/2', + 'v', + 1, + ], + [ + '/test/path/2', + 'h', + 1, + ], + [ + '/test/path/2', + 'v', + 1, + ], + ]); + done(); + }); + }); + }); + + test('re-visit after short time', (done) => { + config['hit_counter']['unique_visitor_timeout'] = 10000; + let hincrbyCalls = []; + mockClient.hincrby = (hash, key, value) => { + hincrbyCalls.push([ + hash, + key, + value, + ]); + return mockClient; + }; + hc.count({ + headers: {}, + connection: { remoteAddress: 'test3' }, + }, '/test/path/3', () => { + hc.count({ + headers: {}, + connection: { remoteAddress: 'test3' }, + }, '/test/path/3', () => { + expect(hincrbyCalls).toEqual([ + [ + '/test/path/3', + 'h', + 1, + ], + [ + '/test/path/3', + 'v', + 1, + ], + [ + '/test/path/3', + 'h', + 1, + ], + [ + '/test/path/3', + 'v', + 0, + ], + ]); + done(); + }); + }); + }); +});