From 20b4d0353f087c7fe9c26fb1909410d0f5ebccf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20GOUIN?= Date: Wed, 19 Jun 2019 16:27:49 +0200 Subject: [PATCH] file_walker.js detect articles in data file tree --- .gitignore | 3 +- package.json | 2 +- ...efault_config.json => config.default.json} | 0 src/config.js | 2 +- src/file_walker.js | 55 ++++++ test/app.test.js | 5 +- test/config.test.json | 3 - test/file_walker.test.js | 165 ++++++++++++++++++ 8 files changed, 227 insertions(+), 8 deletions(-) rename src/{default_config.json => config.default.json} (100%) create mode 100644 src/file_walker.js delete mode 100644 test/config.test.json create mode 100644 test/file_walker.test.js diff --git a/.gitignore b/.gitignore index 8dd596e..aba5ca9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ /node_modules /config.json /config.example.json -/data \ No newline at end of file +/data +/test_data \ No newline at end of file diff --git a/package.json b/package.json index b05eae5..ee88950 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "scripts": { "start": "node src/server.js", "test": "jest --silent", - "install": "shx mkdir ./data && shx cp -rf ./sample_data/home/* ./data && shx mkdir -p ./data/$(date +%Y)/$(date +%m)/$(date +%d) && shx cp -rf ./sample_data/article/* ./data/$(date +%Y)/$(date +%m)/$(date +%d) && shx cp -f ./src/default_config.json ./config.example.json" + "install": "shx mkdir ./data && shx cp -rf ./sample_data/home/* ./data && shx mkdir -p ./data/$(date +%Y)/$(date +%m)/$(date +%d) && shx cp -rf ./sample_data/article/* ./data/$(date +%Y)/$(date +%m)/$(date +%d) && shx cp -f src/config.default.json ./config.example.json" }, "repository": { "type": "git", diff --git a/src/default_config.json b/src/config.default.json similarity index 100% rename from src/default_config.json rename to src/config.default.json diff --git a/src/config.js b/src/config.js index 42f18b3..30ec53d 100644 --- a/src/config.js +++ b/src/config.js @@ -1,4 +1,4 @@ -const refConfig = require('./default_config.json'); +const refConfig = require('./config.default.json'); const fs = require('fs'); const merge = function (ref, src) { diff --git a/src/file_walker.js b/src/file_walker.js new file mode 100644 index 0000000..134a170 --- /dev/null +++ b/src/file_walker.js @@ -0,0 +1,55 @@ +const fs = require('fs'); + +const fileTree = (path, cb) => { + let list = []; + let remaining = 0; + fs.readdir(path, {withFileTypes: true}, (err, items) => { + if (err) + return cb(err); + items.forEach((item) => { + if (item.isDirectory()) { + remaining++; + fileTree(`${path}/${item.name}`, (err, out) => { + if (err) + return cb(err); + list.push(...out); + remaining--; + if (remaining === 0) + cb(null, list); + }); + } else { + list.push(`${path}/${item.name}`); + } + }); + if (remaining === 0) + cb(null, list); + }); +}; + +module.exports = (config) => { + return { + fileTree: config['test'] ? fileTree : undefined, + fetchArticles: (cb) => { + fileTree(config['data_dir'], (err, fileList) => { + if (err) + return cb(err); + const paths = fileList + .map(path => path.substr(config['data_dir'].length)) + .filter(path => path.indexOf(config['article']['index']) === path.length - config['article']['index'].length) + .map(path => path.substr(0, path.length - config['article']['index'].length)) + .map(path => path.match(/^\/(\d{4})\/(\d{2})\/(\d{2})\/$/)) + .filter(path => path && path.length > 1); + const list = []; + paths.forEach(path => { + list.push({ + path: config['data_dir'] + path[0] + config['article']['index'], + year: parseInt(path[1]), + month: parseInt(path[2]), + day: parseInt(path[3]) + }); + }); + cb(null, list); + }); + } + }; +}; \ No newline at end of file diff --git a/test/app.test.js b/test/app.test.js index 6f24b00..8e33e19 100644 --- a/test/app.test.js +++ b/test/app.test.js @@ -1,7 +1,8 @@ /* jshint -W117 */ const request = require('supertest'); -const config = require('./config.test.json'); -const app = require('../src/app')(config); +const app = require('../src/app')({ + +}); describe('Test root path', () => { test('GET / 200', done => { diff --git a/test/config.test.json b/test/config.test.json deleted file mode 100644 index 0e0dcd2..0000000 --- a/test/config.test.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - -} \ No newline at end of file diff --git a/test/file_walker.test.js b/test/file_walker.test.js new file mode 100644 index 0000000..10da942 --- /dev/null +++ b/test/file_walker.test.js @@ -0,0 +1,165 @@ +/* jshint -W117 */ +const fs = require('fs'); + +const dataDir = './test_data'; +const testIndex = 'testindex.md'; + +const config = { + 'test': true, + 'data_dir': dataDir, + 'article': { + 'index': testIndex + } +}; + +const fw = require('../src/file_walker')(config); + +const deleteFolderSync = (path) => { + if(!fs.existsSync(path)) + return; + fs.readdirSync(path, {withFileTypes: true}).forEach((item) => { + if (item.isDirectory()) + deleteFolderSync(`${path}/${item.name}`); + else + fs.unlinkSync(`${path}/${item.name}`); + }); + fs.rmdirSync(path); +}; + +beforeEach(() => { + deleteFolderSync(dataDir); + fs.mkdirSync(dataDir); +}); + +afterAll(() => { + if (fs.existsSync(dataDir)) { + deleteFolderSync(dataDir); + } +}); + +const createEmptyDirs = list => list.forEach(path => fs.mkdirSync(path, {recursive: true})); +const createEmptyFiles = list => list.forEach(file => fs.writeFileSync(file, '')); + +describe('Test function fileTree', () => { + test('empty root', (done) => { + fw.fileTree(dataDir, (err, list) => { + expect(err).toBeNull(); + expect(list).toBeDefined(); + expect(list.length).toBe(0); + done(); + }); + }); + test('empty folders', (done) => { + createEmptyDirs([ + `${dataDir}/test/test`, + `${dataDir}/test/test2`, + `${dataDir}/test2` + ]); + fw.fileTree(dataDir, (err, list) => { + expect(err).toBeNull(); + expect(list).toBeDefined(); + expect(list.length).toBe(0); + done(); + }); + }); + test('simple files', (done) => { + const fileList = [ + `${dataDir}/f1.txt`, + `${dataDir}/f2.txt` + ]; + createEmptyFiles(fileList); + fw.fileTree(dataDir, (err, list) => { + expect(err).toBeNull(); + expect(list).toBeDefined(); + expect(list.length).toBe(fileList.length); + expect(list).toEqual(expect.arrayContaining(fileList)); + done(); + }); + }); + test('nested files', (done) => { + createEmptyDirs([ + `${dataDir}/test/test`, + `${dataDir}/test2` + ]); + const fileList = [ + `${dataDir}/f1.txt`, + `${dataDir}/test/f2.txt`, + `${dataDir}/test/test/f3.txt`, + `${dataDir}/test2/f4.txt` + ]; + createEmptyFiles(fileList); + fw.fileTree(dataDir, (err, list) => { + expect(err).toBeNull(); + expect(list).toBeDefined(); + expect(list.length).toBe(fileList.length); + expect(list).toEqual(expect.arrayContaining(fileList)); + done(); + }); + }); + test('invalid root', (done) => { + fw.fileTree('invalid root', (err, list) => { + expect(err).not.toBeNull(); + expect(list).not.toBeDefined(); + done(); + }); + }); +}); + +describe('Test article fetching', () => { + test('invalid data dir', (done) => { + config['data_dir'] = 'invalid root'; + fw.fetchArticles((err,list) => { + expect(err).not.toBeNull(); + expect(list).not.toBeDefined(); + config['data_dir'] = dataDir; + done(); + }); + }); + test('empty data dir', (done) => { + fw.fetchArticles((err,list) => { + expect(err).toBeNull(); + expect(list).toBeDefined(); + expect(list.length).toBe(0); + done(); + }); + }); + test('misplaced index file', (done) => { + createEmptyDirs([ + `${dataDir}/test/test`, + `${dataDir}/2019/05/05`, + ]); + createEmptyFiles([ + `${dataDir}/${testIndex}`, + `${dataDir}/test/test/${testIndex}`, + `${dataDir}/2019/05/${testIndex}`, + ]); + fw.fetchArticles((err,list) => { + expect(err).toBeNull(); + expect(list).toBeDefined(); + expect(list.length).toBe(0); + done(); + }); + }); + test('correct index file', (done) => { + createEmptyDirs([ + `${dataDir}/2019/05/05`, + ]); + const fileList = [ + `${dataDir}/2019/05/05/${testIndex}`, + ]; + createEmptyFiles(fileList); + fw.fetchArticles((err,list) => { + expect(err).toBeNull(); + expect(list).toBeDefined(); + expect(list.length).toBe(1); + expect(list[0]).toEqual({ + path: fileList[0], + year:2019, + month:5, + day:5 + }); + done(); + }); + }); +}); +