file_walker.js detect articles in data file tree

.gitignore

@@ -2,4 +2,5 @@
 /node_modules
 /config.json
 /config.example.json
-/data
+/data
+/test_data

package.json

@@ -20,7 +20,7 @@
   "scripts": {
     "start": "node src/server.js",
     "test": "jest --silent",
-    "install": "shx mkdir ./data && shx cp -rf ./sample_data/home/* ./data && shx mkdir -p ./data/$(date +%Y)/$(date +%m)/$(date +%d) && shx cp -rf ./sample_data/article/* ./data/$(date +%Y)/$(date +%m)/$(date +%d) && shx cp -f ./src/default_config.json ./config.example.json"
+    "install": "shx mkdir ./data && shx cp -rf ./sample_data/home/* ./data && shx mkdir -p ./data/$(date +%Y)/$(date +%m)/$(date +%d) && shx cp -rf ./sample_data/article/* ./data/$(date +%Y)/$(date +%m)/$(date +%d) && shx cp -f src/config.default.json ./config.example.json"
   },
   "repository": {
     "type": "git",

src/config.js

@@ -1,4 +1,4 @@
-const refConfig = require('./default_config.json');
+const refConfig = require('./config.default.json');
 const fs = require('fs');
 
 const merge = function (ref, src) {

src/file_walker.js

@@ -0,0 +1,55 @@
+const fs = require('fs');
+
+const fileTree = (path, cb) => {
+    let list = [];
+    let remaining = 0;
+    fs.readdir(path, {withFileTypes: true}, (err, items) => {
+        if (err)
+            return cb(err);
+        items.forEach((item) => {
+            if (item.isDirectory()) {
+                remaining++;
+                fileTree(`${path}/${item.name}`, (err, out) => {
+                    if (err)
+                        return cb(err);
+                    list.push(...out);
+                    remaining--;
+                    if (remaining === 0)
+                        cb(null, list);
+                });
+            } else {
+                list.push(`${path}/${item.name}`);
+            }
+        });
+        if (remaining === 0)
+            cb(null, list);
+    });
+};
+
+module.exports = (config) => {
+    return {
+        fileTree: config['test'] ? fileTree : undefined,
+        fetchArticles: (cb) => {
+            fileTree(config['data_dir'], (err, fileList) => {
+                if (err)
+                    return cb(err);
+                const paths = fileList
+                    .map(path => path.substr(config['data_dir'].length))
+                    .filter(path => path.indexOf(config['article']['index']) === path.length - config['article']['index'].length)
+                    .map(path => path.substr(0, path.length - config['article']['index'].length))
+                    .map(path => path.match(/^\/(\d{4})\/(\d{2})\/(\d{2})\/$/))
+                    .filter(path => path && path.length > 1);
+                const list = [];
+                paths.forEach(path => {
+                    list.push({
+                        path: config['data_dir'] + path[0] + config['article']['index'],
+                        year: parseInt(path[1]),
+                        month: parseInt(path[2]),
+                        day: parseInt(path[3])
+                    });
+                });
+                cb(null, list);
+            });
+        }
+    };
+};

test/app.test.js

@@ -1,7 +1,8 @@
 /* jshint -W117 */
 const request = require('supertest');
-const config = require('./config.test.json');
+const app = require('../src/app')({
-const app = require('../src/app')(config);
+
+});
 
 describe('Test root path', () => {
     test('GET / 200', done => {

test/config.test.json

@@ -1,3 +0,0 @@
-{
-
-}

test/file_walker.test.js

@@ -0,0 +1,165 @@
+/* jshint -W117 */
+const fs = require('fs');
+
+const dataDir = './test_data';
+const testIndex = 'testindex.md';
+
+const config = {
+    'test': true,
+    'data_dir': dataDir,
+    'article': {
+        'index': testIndex
+    }
+};
+
+const fw = require('../src/file_walker')(config);
+
+const deleteFolderSync = (path) => {
+    if(!fs.existsSync(path))
+        return;
+    fs.readdirSync(path, {withFileTypes: true}).forEach((item) => {
+        if (item.isDirectory())
+            deleteFolderSync(`${path}/${item.name}`);
+        else
+            fs.unlinkSync(`${path}/${item.name}`);
+    });
+    fs.rmdirSync(path);
+};
+
+beforeEach(() => {
+    deleteFolderSync(dataDir);
+    fs.mkdirSync(dataDir);
+});
+
+afterAll(() => {
+    if (fs.existsSync(dataDir)) {
+        deleteFolderSync(dataDir);
+    }
+});
+
+const createEmptyDirs = list => list.forEach(path => fs.mkdirSync(path, {recursive: true}));
+const createEmptyFiles = list => list.forEach(file => fs.writeFileSync(file, ''));
+
+describe('Test function fileTree', () => {
+    test('empty root', (done) => {
+        fw.fileTree(dataDir, (err, list) => {
+            expect(err).toBeNull();
+            expect(list).toBeDefined();
+            expect(list.length).toBe(0);
+            done();
+        });
+    });
+    test('empty folders', (done) => {
+        createEmptyDirs([
+            `${dataDir}/test/test`,
+            `${dataDir}/test/test2`,
+            `${dataDir}/test2`
+        ]);
+        fw.fileTree(dataDir, (err, list) => {
+            expect(err).toBeNull();
+            expect(list).toBeDefined();
+            expect(list.length).toBe(0);
+            done();
+        });
+    });
+    test('simple files', (done) => {
+        const fileList = [
+            `${dataDir}/f1.txt`,
+            `${dataDir}/f2.txt`
+        ];
+        createEmptyFiles(fileList);
+        fw.fileTree(dataDir, (err, list) => {
+            expect(err).toBeNull();
+            expect(list).toBeDefined();
+            expect(list.length).toBe(fileList.length);
+            expect(list).toEqual(expect.arrayContaining(fileList));
+            done();
+        });
+    });
+    test('nested files', (done) => {
+        createEmptyDirs([
+            `${dataDir}/test/test`,
+            `${dataDir}/test2`
+        ]);
+        const fileList = [
+            `${dataDir}/f1.txt`,
+            `${dataDir}/test/f2.txt`,
+            `${dataDir}/test/test/f3.txt`,
+            `${dataDir}/test2/f4.txt`
+        ];
+        createEmptyFiles(fileList);
+        fw.fileTree(dataDir, (err, list) => {
+            expect(err).toBeNull();
+            expect(list).toBeDefined();
+            expect(list.length).toBe(fileList.length);
+            expect(list).toEqual(expect.arrayContaining(fileList));
+            done();
+        });
+    });
+    test('invalid root', (done) => {
+        fw.fileTree('invalid root', (err, list) => {
+            expect(err).not.toBeNull();
+            expect(list).not.toBeDefined();
+            done();
+        });
+    });
+});
+
+describe('Test article fetching', () => {
+    test('invalid data dir', (done) => {
+        config['data_dir'] = 'invalid root';
+        fw.fetchArticles((err,list) => {
+            expect(err).not.toBeNull();
+            expect(list).not.toBeDefined();
+            config['data_dir'] = dataDir;
+            done();
+        });
+    });
+    test('empty data dir', (done) => {
+        fw.fetchArticles((err,list) => {
+            expect(err).toBeNull();
+            expect(list).toBeDefined();
+            expect(list.length).toBe(0);
+            done();
+        });
+    });
+    test('misplaced index file', (done) => {
+        createEmptyDirs([
+            `${dataDir}/test/test`,
+            `${dataDir}/2019/05/05`,
+        ]);
+        createEmptyFiles([
+            `${dataDir}/${testIndex}`,
+            `${dataDir}/test/test/${testIndex}`,
+            `${dataDir}/2019/05/${testIndex}`,
+        ]);
+        fw.fetchArticles((err,list) => {
+            expect(err).toBeNull();
+            expect(list).toBeDefined();
+            expect(list.length).toBe(0);
+            done();
+        });
+    });
+    test('correct index file', (done) => {
+        createEmptyDirs([
+            `${dataDir}/2019/05/05`,
+        ]);
+        const fileList = [
+            `${dataDir}/2019/05/05/${testIndex}`,
+        ];
+        createEmptyFiles(fileList);
+        fw.fetchArticles((err,list) => {
+            expect(err).toBeNull();
+            expect(list).toBeDefined();
+            expect(list.length).toBe(1);
+            expect(list[0]).toEqual({
+               path: fileList[0],
+                year:2019,
+                month:5,
+                day:5
+            });
+            done();
+        });
+    });
+});
+