klemek/GitBlog.md
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hit-counter

Browse Source
This commit is contained in:
Klemek
2021-03-30 19:11:31 +02:00
parent 88cb7ce30f
commit 6439f8eb92
8 changed files with 529 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+9
View File
@@ -282,6 +282,8 @@ Any URL like `/year/month/day/anything/` will redirect to this article (and link
activate PlantUML diagram rendering
* `fa-diagrams` (default: true)
activate fa-diagrams rendering
* `hit_counter` (default: true)
activate /stats endpoints and visitor counting (need an active redis connection)
* `home`
* `title` (default: GitBlog.md)
the title of your blog, **strongly advised to be changed**
@@ -332,4 +334,11 @@ Any URL like `/year/month/day/anything/` will redirect to this article (and link
specify the output format between svg, html or MathMl (mml)
* `speak_text`: (default: true)
activate the alternate text in equations
* `hit_counter`
* `unique_visitor_timeout`: (default: 7200000 / 2h)
specify the time (in ms) before a visitor can be accounted again
* `redis`
Options to connect to redis (see [redis options](https://github.com/NodeRedis/node-redis#options-object-properties) for more info)
* `host`: (default: localhost)
* `port`: (default: 6379)
package-lock.json
Generated
+85
View File
@@ -19,6 +19,7 @@
"ncp": "^2.0.0",
"node-prismjs": "^0.1.0",
"prismjs": "^1.23.0",
"redis": "^3.0.2",
"rss": "^1.2.2",
"showdown": "^1.9.1"
},
@@ -1897,6 +1898,14 @@
"integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==",
"optional": true
},
"node_modules/denque": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==",
"engines": {
"node": ">=0.10"
}
},
"node_modules/depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -6966,6 +6975,48 @@
"node": ">=4"
}
},
"node_modules/redis": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
"integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
"dependencies": {
"denque": "^1.4.1",
"redis-commands": "^1.5.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0"
},
"engines": {
"node": ">=6"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/node-redis"
}
},
"node_modules/redis-commands": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
},
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=",
"engines": {
"node": ">=4"
}
},
"node_modules/redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
"dependencies": {
"redis-errors": "^1.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/regex-not": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",
@@ -10365,6 +10416,11 @@
"integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==",
"optional": true
},
"denque": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.5.0.tgz",
"integrity": "sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ=="
},
"depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
@@ -14305,6 +14361,35 @@
"util.promisify": "^1.0.0"
}
},
"redis": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/redis/-/redis-3.0.2.tgz",
"integrity": "sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==",
"requires": {
"denque": "^1.4.1",
"redis-commands": "^1.5.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0"
}
},
"redis-commands": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz",
"integrity": "sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ=="
},
"redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha1-62LSrbFeTq9GEMBK/hUpOEJQq60="
},
"redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha1-tm2CjNyv5rS4pCin3vTGvKwxyLQ=",
"requires": {
"redis-errors": "^1.0.0"
}
},
"regex-not": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",
package.json
+1
View File
@@ -14,6 +14,7 @@
"ncp": "^2.0.0",
"node-prismjs": "^0.1.0",
"prismjs": "^1.23.0",
"redis": "^3.0.2",
"rss": "^1.2.2",
"showdown": "^1.9.1"
},
src/app.js
+56 -35
View File
@@ -51,7 +51,16 @@ module.exports = (config) => {
let showError;
const fw = require('./file_walker')(config);
const renderer = require('./renderer')(config);
const hc = require('./hit_counter')(config);
const hc = require('./hit_counter')(config,
() => {
console.log(cons.ok, 'redis connected');
},
(err) => {
if (err.code !== 'ECONNREFUSED') {
console.log(cons.warn, 'redis error: ' + err);
}
},
);
// set view engine from configuration
app.set('view engine', config['view_engine']);
@@ -159,22 +168,28 @@ module.exports = (config) => {
if (err) {
showError(req, res, 404);
} else {
hc.count(req, '/');
render(req, res, homePath,
{
articles: Object.values(articles)
.filter(d => !d.draft)
.sort((a, b) => ('' + b.path).localeCompare(a.path)),
});
hc.count(req, '/', () => {
render(req, res, homePath,
{
articles: Object.values(articles)
.filter(d => !d.draft)
.sort((a, b) => ('' + b.path).localeCompare(a.path)),
});
});
}
});
});
app.get('/stats', (req, res) => {
const data = hc.read('/');
res.json({
hits: data.hits,
visitors: data.visitors,
});
if (config['modules']['hit_counter']) {
hc.read('/', (data) => {
res.json({
hits: data.hits,
visitors: data.visitors,
});
});
} else {
showError(req, res, 404);
}
});
//RSS endpoint
@@ -251,29 +266,35 @@ module.exports = (config) => {
if (!article) {
showError(req, res, 404);
} else if (req.path.endsWith('stats')) {
const data = hc.read(articlePath);
res.json({
hits: data.hits,
visitors: data.visitors,
});
} else {
hc.count(req, articlePath);
renderer.render(article.realPath, (err, html) => {
if (err) {
console.log(cons.error, `failed to render article ${req.path} : ${err}`);
showError(req, res, 500);
} else {
article.content = html;
const templatePath = path.join(config['data_dir'], config['article']['template']);
fs.access(templatePath, fs.constants.R_OK, (err) => {
if (err) {
console.log(cons.error, `no template found at ${templatePath}`);
showError(req, res, 500);
} else {
render(req, res, templatePath, { article: article });
}
if (config['modules']['hit_counter']) {
hc.read(articlePath, (data) => {
res.json({
hits: data.hits,
visitors: data.visitors,
});
}
});
} else {
showError(req, res, 404);
}
} else {
hc.count(req, articlePath, () => {
renderer.render(article.realPath, (err, html) => {
if (err) {
console.log(cons.error, `failed to render article ${req.path} : ${err}`);
showError(req, res, 500);
} else {
article.content = html;
const templatePath = path.join(config['data_dir'], config['article']['template']);
fs.access(templatePath, fs.constants.R_OK, (err) => {
if (err) {
console.log(cons.error, `no template found at ${templatePath}`);
showError(req, res, 500);
} else {
render(req, res, templatePath, { article: article });
}
});
}
});
});
}
} else {
src/config.default.json
+9 -1
View File
@@ -12,7 +12,8 @@
"prism": true,
"mathjax": true,
"plantuml": true,
"fa-diagrams": true
"fa-diagrams": true,
"hit_counter": true
},
"home": {
"title": "GitBlog.md",
@@ -58,5 +59,12 @@
},
"plantuml": {
"output_format": "svg"
},
"hit_counter": {
"unique_visitor_timeout": 7200000
},
"redis": {
"host": "localhost",
"port": 6379
}
}
src/hit_counter.js
+38 -7
View File
@@ -1,13 +1,44 @@
module.exports = (config) => {
const count = (req, path) => {
const redis = require('redis');
module.exports = (config, onConnect, onError) => {
const client = config['modules']['hit_counter'] ? redis.createClient(config['redis']) : { connected: false, on: () => { /* ignore */ } };
client.on('connect', onConnect);
client.on('error', onError);
const visitors = {};
const count = (req, path, cb) => {
if (!client.connected) {
cb();
} else {
const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
const key = path + ':' + ip;
const now = Date.now();
const isNewVisitor = (now - (visitors[key] || 0)) > config['hit_counter']['unique_visitor_timeout'];
visitors[key] = now;
client
.multi()
.hincrby(path, 'h', 1)
.hincrby(path, 'v', isNewVisitor ? 1 : 0)
.exec(cb);
}
};
const read = (path) => {
return {
hits: 0,
visitors: 0,
};
const read = (path, cb) => {
if (!client.connected) {
cb({
hits: 0,
visitors: 0,
});
} else {
client.hgetall(path, (_, value) => {
cb({
hits: value ? value.h || 0 : 0,
visitors: value ? value.v || 0 : 0,
});
});
}
};
return {
test/app.test.js
+50
View File
@@ -17,6 +17,7 @@ config['rss']['endpoint'] = '/rsstest';
config['rss']['length'] = 2;
config['home']['error'] = testError;
config['article']['template'] = testTemplate;
config['modules']['hit_counter'] = false;
const app = require('../src/app')(config);
@@ -28,6 +29,7 @@ beforeEach((done, fail) => {
config['error_log'] = '';
config['modules']['rss'] = true;
config['modules']['webhook'] = true;
config['modules']['hit_counter'] = false;
utils.deleteFolderSync(dataDir);
fs.mkdirSync(dataDir);
@@ -195,6 +197,22 @@ describe('Test root path', () => {
});
}, fail);
});
test('404 index no stats', (done) => {
request(app).get('/stats')
.then((response) => {
expect(response.statusCode).toBe(404);
done();
});
});
test('200 index stats', (done) => {
config['modules']['hit_counter'] = true;
request(app).get('/stats')
.then((response) => {
expect(response.statusCode).toBe(200);
expect(response.body).toEqual({ hits: 0, visitors: 0 });
done();
});
});
});
describe('Test RSS feed', () => {
@@ -433,6 +451,38 @@ describe('Test articles rendering', () => {
});
}, fail);
});
test('404 article no stats', (done) => {
utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]);
utils.createEmptyFiles([
path.join(dataDir, '2019', '05', '05', 'index.md'),
path.join(dataDir, testTemplate),
]);
app.reload(() => {
request(app).get('/2019/05/05/hello/stats')
.then((response) => {
expect(response.statusCode).toBe(404);
done();
});
});
});
test('200 index stats', (done) => {
config['modules']['hit_counter'] = true;
utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]);
utils.createEmptyFiles([
path.join(dataDir, '2019', '05', '05', 'index.md'),
path.join(dataDir, testTemplate),
]);
app.reload(() => {
request(app).get('/2019/05/05/anything/stats')
.then((response) => {
expect(response.statusCode).toBe(200);
expect(response.body).toEqual({ hits: 0, visitors: 0 });
done();
});
});
});
});
test/hit_counter.test.js
+281
View File
@@ -0,0 +1,281 @@
const mockClient = {
options: {},
connected: true,
on: () => { /* ignore */ },
};
jest.mock('redis', () => {
return {
createClient: (options) => {
mockClient.options = options;
return mockClient;
},
};
});
const config = {
test: true,
modules: {
hit_counter: true,
},
redis: {
host: 'test-host',
port: 'test-port',
},
hit_counter: {
unique_visitor_timeout: -1,
},
};
const hc = require('../src/hit_counter')(config, () => { /* ignore */ }, () => { /* ignore */ });
afterAll(() => {
jest.resetModules();
});
test('options passed to redis', () => {
expect(mockClient.options).toEqual(config['redis']);
});
describe('read()', () => {
beforeEach(() => {
mockClient.hgetall = (_, cb) => {
cb();
};
});
test('read path', (done) => {
mockClient.hgetall = (path, cb) => {
expect(path).toBe('/test/path/');
cb(undefined, { h: 12, v: 34 });
};
hc.read('/test/path/', (data) => {
expect(data).toBeDefined();
expect(data.hits).toBe(12);
expect(data.visitors).toBe(34);
done();
});
});
test('read path with error', (done) => {
mockClient.hgetall = (path, cb) => {
expect(path).toBe('/test/path/');
cb('error', undefined);
};
hc.read('/test/path/', (data) => {
expect(data).toBeDefined();
expect(data.hits).toBe(0);
expect(data.visitors).toBe(0);
done();
});
});
test('read path with error 2', (done) => {
mockClient.hgetall = (path, cb) => {
expect(path).toBe('/test/path/');
cb(undefined, {});
};
hc.read('/test/path/', (data) => {
expect(data).toBeDefined();
expect(data.hits).toBe(0);
expect(data.visitors).toBe(0);
done();
});
});
});
describe('count()', () => {
beforeEach(() => {
mockClient.multi = () => mockClient;
mockClient.hincrby = () => mockClient;
mockClient.exec = (cb) => {
cb();
};
});
test('simple visit', (done) => {
let multiCalled = false;
let execCalled = false;
let hincrbyCalls = [];
mockClient.multi = () => {
multiCalled = true;
return mockClient;
};
mockClient.hincrby = (hash, key, value) => {
hincrbyCalls.push([
hash,
key,
value,
]);
return mockClient;
};
mockClient.exec = (cb) => {
execCalled = true;
cb();
};
hc.count({
headers: {},
connection: { remoteAddress: 'test1' },
}, '/test/path/1', () => {
expect(multiCalled).toBeTruthy();
expect(hincrbyCalls).toEqual([
[
'/test/path/1',
'h',
1,
],
[
'/test/path/1',
'v',
1,
],
]);
expect(execCalled).toBeTruthy();
done();
});
});
});
describe('count()', () => {
beforeEach(() => {
mockClient.multi = () => mockClient;
mockClient.hincrby = () => mockClient;
mockClient.exec = (cb) => {
cb();
};
config['hit_counter']['unique_visitor_timeout'] = -1;
});
test('simple visit', (done) => {
let multiCalled = false;
let execCalled = false;
let hincrbyCalls = [];
mockClient.multi = () => {
multiCalled = true;
return mockClient;
};
mockClient.hincrby = (hash, key, value) => {
hincrbyCalls.push([
hash,
key,
value,
]);
return mockClient;
};
mockClient.exec = (cb) => {
execCalled = true;
cb();
};
hc.count({
headers: {},
connection: { remoteAddress: 'test1' },
}, '/test/path/1', () => {
expect(multiCalled).toBeTruthy();
expect(hincrbyCalls).toEqual([
[
'/test/path/1',
'h',
1,
],
[
'/test/path/1',
'v',
1,
],
]);
expect(execCalled).toBeTruthy();
done();
});
});
test('re-visit after long time', (done) => {
let hincrbyCalls = [];
mockClient.hincrby = (hash, key, value) => {
hincrbyCalls.push([
hash,
key,
value,
]);
return mockClient;
};
hc.count({
headers: {},
connection: { remoteAddress: 'test2' },
}, '/test/path/2', () => {
hc.count({
headers: {},
connection: { remoteAddress: 'test2' },
}, '/test/path/2', () => {
expect(hincrbyCalls).toEqual([
[
'/test/path/2',
'h',
1,
],
[
'/test/path/2',
'v',
1,
],
[
'/test/path/2',
'h',
1,
],
[
'/test/path/2',
'v',
1,
],
]);
done();
});
});
});
test('re-visit after short time', (done) => {
config['hit_counter']['unique_visitor_timeout'] = 10000;
let hincrbyCalls = [];
mockClient.hincrby = (hash, key, value) => {
hincrbyCalls.push([
hash,
key,
value,
]);
return mockClient;
};
hc.count({
headers: {},
connection: { remoteAddress: 'test3' },
}, '/test/path/3', () => {
hc.count({
headers: {},
connection: { remoteAddress: 'test3' },
}, '/test/path/3', () => {
expect(hincrbyCalls).toEqual([
[
'/test/path/3',
'h',
1,
],
[
'/test/path/3',
'v',
1,
],
[
'/test/path/3',
'h',
1,
],
[
'/test/path/3',
'v',
0,
],
]);
done();
});
});
});
});