hit-counter

README.md

@@ -282,6 +282,8 @@ Any URL like `/year/month/day/anything/` will redirect to this article (and link
      activate PlantUML diagram rendering
    * `fa-diagrams` (default: true)  
      activate fa-diagrams rendering
+   * `hit_counter` (default: true)  
+     activate /stats endpoints and visitor counting (need an active redis connection)
 * `home`
    * `title` (default: GitBlog.md)  
      the title of your blog, **strongly advised to be changed**  
@@ -332,4 +334,11 @@ Any URL like `/year/month/day/anything/` will redirect to this article (and link
      specify the output format between svg, html or MathMl (mml)
    * `speak_text`: (default: true)  
      activate the alternate text in equations
+* `hit_counter`
+   * `unique_visitor_timeout`: (default: 7200000 / 2h)
+     specify the time (in ms) before a visitor can be accounted again
+* `redis`  
+ Options to connect to redis (see [redis options](https://github.com/NodeRedis/node-redis#options-object-properties) for more info)  
+   * `host`: (default: localhost)  
+   * `port`: (default: 6379) 
 

package-lock.json

@@ -19,6 +19,7 @@
         "ncp": "^2.0.0",
         "node-prismjs": "^0.1.0",
         "prismjs": "^1.23.0",
+        "redis": "^3.0.2",
         "rss": "^1.2.2",
         "showdown": "^1.9.1"
       },
@@ -1897,6 +1898,14 @@
       "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==",
       "optional": true
     },
+    "node_modules/denque": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
+      "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==",
+      "engines": {
+        "node": ">=0.10"
+      }
+    },
     "node_modules/depd": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -6966,6 +6975,48 @@
         "node": ">=4"
       }
     },
+    "node_modules/redis": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
+      "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
+      "dependencies": {
+        "denque": "^1.4.1",
+        "redis-commands": "^1.5.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/node-redis"
+      }
+    },
+    "node_modules/redis-commands": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
+      "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
+    },
+    "node_modules/redis-errors": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+      "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/redis-parser": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
+      "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
+      "dependencies": {
+        "redis-errors": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/regex-not": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",
@@ -10365,6 +10416,11 @@
       "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==",
       "optional": true
     },
+    "denque": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
+      "integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ=="
+    },
     "depd": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -14305,6 +14361,35 @@
         "util.promisify": "^1.0.0"
       }
     },
+    "redis": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
+      "integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
+      "requires": {
+        "denque": "^1.4.1",
+        "redis-commands": "^1.5.0",
+        "redis-errors": "^1.2.0",
+        "redis-parser": "^3.0.0"
+      }
+    },
+    "redis-commands": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
+      "integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
+    },
+    "redis-errors": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
+      "integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
+    },
+    "redis-parser": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
+      "integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
+      "requires": {
+        "redis-errors": "^1.0.0"
+      }
+    },
     "regex-not": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",

package.json

@@ -14,6 +14,7 @@
     "ncp": "^2.0.0",
     "node-prismjs": "^0.1.0",
     "prismjs": "^1.23.0",
+    "redis": "^3.0.2",
     "rss": "^1.2.2",
     "showdown": "^1.9.1"
   },

src/app.js

@@ -51,7 +51,16 @@ module.exports = (config) => {
     let showError;
     const fw = require('./file_walker')(config);
     const renderer = require('./renderer')(config);
-    const hc = require('./hit_counter')(config);
+    const hc = require('./hit_counter')(config,
+        () => {
+            console.log(cons.ok, 'redis connected');
+        },
+        (err) => {
+            if (err.code !== 'ECONNREFUSED') {
+                console.log(cons.warn, 'redis error: ' + err);
+            }
+        },
+    );
 
     // set view engine from configuration
     app.set('view engine', config['view_engine']);
@@ -159,23 +168,29 @@ module.exports = (config) => {
             if (err) {
                 showError(req, res, 404);
             } else {
-                hc.count(req, '/');
+                hc.count(req, '/', () => {
                     render(req, res, homePath,
                         {
                             articles: Object.values(articles)
                                 .filter(d => !d.draft)
                                 .sort((a, b) => ('' + b.path).localeCompare(a.path)),
                         });
+                });
             }
         });
     });
     app.get('/stats', (req, res) => {
-        const data = hc.read('/');
+        if (config['modules']['hit_counter']) {
+            hc.read('/', (data) => {
                 res.json({
                     hits: data.hits,
                     visitors: data.visitors,
                 });
             });
+        } else {
+            showError(req, res, 404);
+        }
+    });
 
     //RSS endpoint
     app.get(config['rss']['endpoint'], (req, res) => {
@@ -251,13 +266,18 @@ module.exports = (config) => {
             if (!article) {
                 showError(req, res, 404);
             } else if (req.path.endsWith('stats')) {
-                const data = hc.read(articlePath);
+                if (config['modules']['hit_counter']) {
+                    hc.read(articlePath, (data) => {
                         res.json({
                             hits: data.hits,
                             visitors: data.visitors,
                         });
+                    });
                 } else {
-                hc.count(req, articlePath);
+                    showError(req, res, 404);
+                }
+            } else {
+                hc.count(req, articlePath, () => {
                     renderer.render(article.realPath, (err, html) => {
                         if (err) {
                             console.log(cons.error, `failed to render article ${req.path} : ${err}`);
@@ -275,6 +295,7 @@ module.exports = (config) => {
                             });
                         }
                     });
+                });
             }
         } else {
             next();

src/config.default.json

@@ -12,7 +12,8 @@
     "prism": true,
     "mathjax": true,
     "plantuml": true,
-    "fa-diagrams": true
+    "fa-diagrams": true,
+    "hit_counter": true
   },
   "home": {
     "title": "GitBlog.md",
@@ -58,5 +59,12 @@
   },
   "plantuml": {
     "output_format": "svg"
+  },
+  "hit_counter": {
+    "unique_visitor_timeout": 7200000
+  },
+  "redis": {
+    "host": "localhost",
+    "port": 6379
   }
 }

src/hit_counter.js

@@ -1,13 +1,44 @@
-module.exports = (config) => {
+const redis = require('redis');
-    const count = (req, path) => {
 
+module.exports = (config, onConnect, onError) => {
+    const client = config['modules']['hit_counter'] ? redis.createClient(config['redis']) : { connected: false, on: () => { /* ignore */ } };
+
+    client.on('connect', onConnect);
+    client.on('error', onError);
+
+    const visitors = {};
+
+    const count = (req, path, cb) => {
+        if (!client.connected) {
+            cb();
+        } else {
+            const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
+            const key = path + ':' + ip;
+            const now = Date.now();
+            const isNewVisitor = (now - (visitors[key] || 0)) > config['hit_counter']['unique_visitor_timeout'];
+            visitors[key] = now;
+            client
+                .multi()
+                .hincrby(path, 'h', 1)
+                .hincrby(path, 'v', isNewVisitor ? 1 : 0)
+                .exec(cb);
+        }
     };
 
-    const read = (path) => {
+    const read = (path, cb) => {
-        return {
+        if (!client.connected) {
+            cb({
                 hits: 0,
                 visitors: 0,
-        };
+            });
+        } else {
+            client.hgetall(path, (_, value) => {
+                cb({
+                    hits: value ? value.h || 0 : 0,
+                    visitors: value ? value.v || 0 : 0,
+                });
+            });
+        }
     };
 
     return {

test/app.test.js

@@ -17,6 +17,7 @@ config['rss']['endpoint'] = '/rsstest';
 config['rss']['length'] = 2;
 config['home']['error'] = testError;
 config['article']['template'] = testTemplate;
+config['modules']['hit_counter'] = false;
 
 const app = require('../src/app')(config);
 
@@ -28,6 +29,7 @@ beforeEach((done, fail) => {
     config['error_log'] = '';
     config['modules']['rss'] = true;
     config['modules']['webhook'] = true;
+    config['modules']['hit_counter'] = false;
 
     utils.deleteFolderSync(dataDir);
     fs.mkdirSync(dataDir);
@@ -195,6 +197,22 @@ describe('Test root path', () => {
                 });
         }, fail);
     });
+    test('404 index no stats', (done) => {
+        request(app).get('/stats')
+            .then((response) => {
+                expect(response.statusCode).toBe(404);
+                done();
+            });
+    });
+    test('200 index stats', (done) => {
+        config['modules']['hit_counter'] = true;
+        request(app).get('/stats')
+            .then((response) => {
+                expect(response.statusCode).toBe(200);
+                expect(response.body).toEqual({ hits: 0, visitors: 0 });
+                done();
+            });
+    });
 });
 
 describe('Test RSS feed', () => {
@@ -433,6 +451,38 @@ describe('Test articles rendering', () => {
                 });
         }, fail);
     });
+
+    test('404 article no stats', (done) => {
+        utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]);
+        utils.createEmptyFiles([
+            path.join(dataDir, '2019', '05', '05', 'index.md'),
+            path.join(dataDir, testTemplate),
+        ]);
+        app.reload(() => {
+            request(app).get('/2019/05/05/hello/stats')
+                .then((response) => {
+                    expect(response.statusCode).toBe(404);
+                    done();
+                });
+        });
+    });
+
+    test('200 index stats', (done) => {
+        config['modules']['hit_counter'] = true;
+        utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]);
+        utils.createEmptyFiles([
+            path.join(dataDir, '2019', '05', '05', 'index.md'),
+            path.join(dataDir, testTemplate),
+        ]);
+        app.reload(() => {
+            request(app).get('/2019/05/05/anything/stats')
+                .then((response) => {
+                    expect(response.statusCode).toBe(200);
+                    expect(response.body).toEqual({ hits: 0, visitors: 0 });
+                    done();
+                });
+        });
+    });
 });
 
 

test/hit_counter.test.js

@@ -0,0 +1,281 @@
+const mockClient = {
+    options: {},
+    connected: true,
+    on: () => { /* ignore */ },
+};
+
+jest.mock('redis', () => {
+    return {
+        createClient: (options) => {
+            mockClient.options = options;
+            return mockClient;
+        },
+    };
+});
+
+const config = {
+    test: true,
+    modules: {
+        hit_counter: true,
+    },
+    redis: {
+        host: 'test-host',
+        port: 'test-port',
+    },
+    hit_counter: {
+        unique_visitor_timeout: -1,
+    },
+};
+
+const hc = require('../src/hit_counter')(config, () => { /* ignore */ }, () => { /* ignore */ });
+
+afterAll(() => {
+    jest.resetModules();
+});
+
+test('options passed to redis', () => {
+    expect(mockClient.options).toEqual(config['redis']);
+});
+
+describe('read()', () => {
+    beforeEach(() => {
+        mockClient.hgetall = (_, cb) => {
+            cb();
+        };
+    });
+
+    test('read path', (done) => {
+        mockClient.hgetall = (path, cb) => {
+            expect(path).toBe('/test/path/');
+            cb(undefined, { h: 12, v: 34 });
+        };
+        hc.read('/test/path/', (data) => {
+            expect(data).toBeDefined();
+            expect(data.hits).toBe(12);
+            expect(data.visitors).toBe(34);
+            done();
+        });
+    });
+
+    test('read path with error', (done) => {
+        mockClient.hgetall = (path, cb) => {
+            expect(path).toBe('/test/path/');
+            cb('error', undefined);
+        };
+        hc.read('/test/path/', (data) => {
+            expect(data).toBeDefined();
+            expect(data.hits).toBe(0);
+            expect(data.visitors).toBe(0);
+            done();
+        });
+    });
+
+    test('read path with error 2', (done) => {
+        mockClient.hgetall = (path, cb) => {
+            expect(path).toBe('/test/path/');
+            cb(undefined, {});
+        };
+        hc.read('/test/path/', (data) => {
+            expect(data).toBeDefined();
+            expect(data.hits).toBe(0);
+            expect(data.visitors).toBe(0);
+            done();
+        });
+    });
+});
+
+describe('count()', () => {
+    beforeEach(() => {
+        mockClient.multi = () => mockClient;
+        mockClient.hincrby = () => mockClient;
+        mockClient.exec = (cb) => {
+            cb();
+        };
+    });
+
+    test('simple visit', (done) => {
+        let multiCalled = false;
+        let execCalled = false;
+        let hincrbyCalls = [];
+        mockClient.multi = () => {
+            multiCalled = true;
+            return mockClient;
+        };
+        mockClient.hincrby = (hash, key, value) => {
+            hincrbyCalls.push([
+                hash,
+                key,
+                value,
+            ]);
+            return mockClient;
+        };
+        mockClient.exec = (cb) => {
+            execCalled = true;
+            cb();
+        };
+        hc.count({
+            headers: {},
+            connection: { remoteAddress: 'test1' },
+        }, '/test/path/1', () => {
+            expect(multiCalled).toBeTruthy();
+            expect(hincrbyCalls).toEqual([
+                [
+                    '/test/path/1',
+                    'h',
+                    1,
+                ],
+                [
+                    '/test/path/1',
+                    'v',
+                    1,
+                ],
+            ]);
+            expect(execCalled).toBeTruthy();
+            done();
+        });
+    });
+});
+
+describe('count()', () => {
+    beforeEach(() => {
+        mockClient.multi = () => mockClient;
+        mockClient.hincrby = () => mockClient;
+        mockClient.exec = (cb) => {
+            cb();
+        };
+        config['hit_counter']['unique_visitor_timeout'] = -1;
+    });
+
+    test('simple visit', (done) => {
+        let multiCalled = false;
+        let execCalled = false;
+        let hincrbyCalls = [];
+        mockClient.multi = () => {
+            multiCalled = true;
+            return mockClient;
+        };
+        mockClient.hincrby = (hash, key, value) => {
+            hincrbyCalls.push([
+                hash,
+                key,
+                value,
+            ]);
+            return mockClient;
+        };
+        mockClient.exec = (cb) => {
+            execCalled = true;
+            cb();
+        };
+        hc.count({
+            headers: {},
+            connection: { remoteAddress: 'test1' },
+        }, '/test/path/1', () => {
+            expect(multiCalled).toBeTruthy();
+            expect(hincrbyCalls).toEqual([
+                [
+                    '/test/path/1',
+                    'h',
+                    1,
+                ],
+                [
+                    '/test/path/1',
+                    'v',
+                    1,
+                ],
+            ]);
+            expect(execCalled).toBeTruthy();
+            done();
+        });
+    });
+
+    test('re-visit after long time', (done) => {
+        let hincrbyCalls = [];
+        mockClient.hincrby = (hash, key, value) => {
+            hincrbyCalls.push([
+                hash,
+                key,
+                value,
+            ]);
+            return mockClient;
+        };
+        hc.count({
+            headers: {},
+            connection: { remoteAddress: 'test2' },
+        }, '/test/path/2', () => {
+            hc.count({
+                headers: {},
+                connection: { remoteAddress: 'test2' },
+            }, '/test/path/2', () => {
+                expect(hincrbyCalls).toEqual([
+                    [
+                        '/test/path/2',
+                        'h',
+                        1,
+                    ],
+                    [
+                        '/test/path/2',
+                        'v',
+                        1,
+                    ],
+                    [
+                        '/test/path/2',
+                        'h',
+                        1,
+                    ],
+                    [
+                        '/test/path/2',
+                        'v',
+                        1,
+                    ],
+                ]);
+                done();
+            });
+        });
+    });
+
+    test('re-visit after short time', (done) => {
+        config['hit_counter']['unique_visitor_timeout'] = 10000;
+        let hincrbyCalls = [];
+        mockClient.hincrby = (hash, key, value) => {
+            hincrbyCalls.push([
+                hash,
+                key,
+                value,
+            ]);
+            return mockClient;
+        };
+        hc.count({
+            headers: {},
+            connection: { remoteAddress: 'test3' },
+        }, '/test/path/3', () => {
+            hc.count({
+                headers: {},
+                connection: { remoteAddress: 'test3' },
+            }, '/test/path/3', () => {
+                expect(hincrbyCalls).toEqual([
+                    [
+                        '/test/path/3',
+                        'h',
+                        1,
+                    ],
+                    [
+                        '/test/path/3',
+                        'v',
+                        1,
+                    ],
+                    [
+                        '/test/path/3',
+                        'h',
+                        1,
+                    ],
+                    [
+                        '/test/path/3',
+                        'v',
+                        0,
+                    ],
+                ]);
+                done();
+            });
+        });
+    });
+});