fix: upgrade body-parser from 1.19.0 to 1.20.0

Snyk has created this PR to upgrade body-parser from 1.19.0 to 1.20.0.

See this package in npm:
https://www.npmjs.com/package/body-parser

See this project in Snyk:
https://app.snyk.io/org/klemek/project/81af650f-8c8e-4116-b289-95415c85d8a7?utm_source=github&utm_medium=referral&page=upgrade-pr

.github/workflows/docker.yml

@@ -0,0 +1,34 @@
+name: Docker
+
+on: ["push", "pull_request"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      - name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          push: false
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -2,6 +2,7 @@
 /node_modules
 /config.json
 /config.example.json
+/robots_list.json
 /data
 /data/*
 /test_data

Dockerfile

@@ -1,12 +1,16 @@
-FROM node:14
+FROM node:15
 
 # Create app directory
 WORKDIR /usr/src/app
 
+VOLUME [ "/usr/src/app/data" ]
+
 # Install app dependencies
 # A wildcard is used to ensure both package.json AND package-lock.json are copied
 # where available (npm@5+)
 COPY package*.json ./
+COPY src/postinstall.js ./src/postinstall.js
+COPY src/config.default.json ./src/config.default.json
 
 RUN npm install
 # If you are building your code for production

README.md

@@ -336,6 +336,11 @@ Any URL like `/year/month/day/anything/` will redirect to this article (and link
 * `hit_counter`
    * `unique_visitor_timeout`: (default: 7200000 / 2h)
      specify the time (in ms) before a visitor can be accounted again
+* `robots`
+   * `list_url`: (default: https://raw.githubusercontent.com/atmire/COUNTER-Robots/master/COUNTER_Robots_list.json)
+     url to fetch for web crawlers patterns
+   * `list_file`: (default: robots_list.json)
+     file to store web crawlers patterns
 * `redis`  
  Options to connect to redis (see [redis options](https://github.com/NodeRedis/node-redis#options-object-properties) for more info)  
    * `host`: (default: localhost)  

package-lock.json

@@ -1,16 +1,17 @@
 {
   "name": "gitblog.md",
-  "version": "1.2.8",
+  "version": "1.3.3",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
-      "version": "1.2.8",
+      "name": "gitblog.md",
+      "version": "1.3.3",
       "hasInstallScript": true,
       "license": "ISC",
       "dependencies": {
         "@iarna/toml": "^2.2.3",
-        "body-parser": "^1.19.0",
+        "body-parser": "^1.20.0",
         "ejs": "^3.1.6",
         "express": "^4.17.1",
         "express-rate-limit": "^5.0.0",
@@ -1285,25 +1286,111 @@
       }
     },
     "node_modules/body-parser": {
-      "version": "1.19.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
-      "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
+      "version": "1.20.0",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
+      "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==",
       "dependencies": {
-        "bytes": "3.1.0",
+        "bytes": "3.1.2",
         "content-type": "~1.0.4",
         "debug": "2.6.9",
-        "depd": "~1.1.2",
-        "http-errors": "1.7.2",
+        "depd": "2.0.0",
+        "destroy": "1.2.0",
+        "http-errors": "2.0.0",
         "iconv-lite": "0.4.24",
-        "on-finished": "~2.3.0",
-        "qs": "6.7.0",
-        "raw-body": "2.4.0",
-        "type-is": "~1.6.17"
+        "on-finished": "2.4.1",
+        "qs": "6.10.3",
+        "raw-body": "2.5.1",
+        "type-is": "~1.6.18",
+        "unpipe": "1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8",
+        "npm": "1.2.8000 || >= 1.4.16"
+      }
+    },
+    "node_modules/body-parser/node_modules/depd": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
+      "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/body-parser/node_modules/destroy": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
+      "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==",
+      "engines": {
+        "node": ">= 0.8",
+        "npm": "1.2.8000 || >= 1.4.16"
+      }
+    },
+    "node_modules/body-parser/node_modules/http-errors": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
+      "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
+      "dependencies": {
+        "depd": "2.0.0",
+        "inherits": "2.0.4",
+        "setprototypeof": "1.2.0",
+        "statuses": "2.0.1",
+        "toidentifier": "1.0.1"
       },
       "engines": {
         "node": ">= 0.8"
       }
     },
+    "node_modules/body-parser/node_modules/inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+    },
+    "node_modules/body-parser/node_modules/on-finished": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
+      "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
+      "dependencies": {
+        "ee-first": "1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/body-parser/node_modules/qs": {
+      "version": "6.10.3",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
+      "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
+      "dependencies": {
+        "side-channel": "^1.0.4"
+      },
+      "engines": {
+        "node": ">=0.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/body-parser/node_modules/setprototypeof": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
+    },
+    "node_modules/body-parser/node_modules/statuses": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+      "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/body-parser/node_modules/toidentifier": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+      "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
+      "engines": {
+        "node": ">=0.6"
+      }
+    },
     "node_modules/brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -1382,9 +1469,9 @@
       "dev": true
     },
     "node_modules/bytes": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
-      "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==",
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+      "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
       "engines": {
         "node": ">= 0.8"
       }
@@ -1413,7 +1500,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
       "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
-      "dev": true,
       "dependencies": {
         "function-bind": "^1.1.1",
         "get-intrinsic": "^1.0.2"
@@ -2685,6 +2771,48 @@
       "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.0.0.tgz",
       "integrity": "sha512-dhT57wqxfqmkOi4HM7NuT4Gd7gbUgSK2ocG27Y6lwm8lbOAw9XQfeANawGq8wLDtlGPO1ZgDj0HmKsykTxfFAg=="
     },
+    "node_modules/express/node_modules/body-parser": {
+      "version": "1.19.0",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
+      "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
+      "dependencies": {
+        "bytes": "3.1.0",
+        "content-type": "~1.0.4",
+        "debug": "2.6.9",
+        "depd": "~1.1.2",
+        "http-errors": "1.7.2",
+        "iconv-lite": "0.4.24",
+        "on-finished": "~2.3.0",
+        "qs": "6.7.0",
+        "raw-body": "2.4.0",
+        "type-is": "~1.6.17"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/express/node_modules/bytes": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
+      "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/express/node_modules/raw-body": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
+      "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
+      "dependencies": {
+        "bytes": "3.1.0",
+        "http-errors": "1.7.2",
+        "iconv-lite": "0.4.24",
+        "unpipe": "1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
     "node_modules/extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@@ -3873,8 +4001,7 @@
     "node_modules/function-bind": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
-      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
-      "dev": true
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
     "node_modules/functional-red-black-tree": {
       "version": "1.0.1",
@@ -3894,7 +4021,6 @@
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz",
       "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==",
-      "dev": true,
       "dependencies": {
         "function-bind": "^1.1.1",
         "has": "^1.0.3",
@@ -4072,7 +4198,6 @@
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
       "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
-      "dev": true,
       "dependencies": {
         "function-bind": "^1.1.1"
       },
@@ -4092,7 +4217,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
       "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
-      "dev": true,
       "engines": {
         "node": ">= 0.4"
       },
@@ -6329,10 +6453,12 @@
       }
     },
     "node_modules/object-inspect": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.6.0.tgz",
-      "integrity": "sha512-GJzfBZ6DgDAmnuaM3104jR4s1Myxr3Y3zfIyN4z3UdqN69oSRacNK8UhnobDdC+7J2AHCjGwxQubNJfE70SXXQ==",
-      "dev": true
+      "version": "1.12.2",
+      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz",
+      "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==",
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
     },
     "node_modules/object-keys": {
       "version": "1.1.1",
@@ -6845,12 +6971,12 @@
       }
     },
     "node_modules/raw-body": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
-      "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
+      "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
       "dependencies": {
-        "bytes": "3.1.0",
-        "http-errors": "1.7.2",
+        "bytes": "3.1.2",
+        "http-errors": "2.0.0",
         "iconv-lite": "0.4.24",
         "unpipe": "1.0.0"
       },
@@ -6858,6 +6984,55 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/raw-body/node_modules/depd": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
+      "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/raw-body/node_modules/http-errors": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
+      "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
+      "dependencies": {
+        "depd": "2.0.0",
+        "inherits": "2.0.4",
+        "setprototypeof": "1.2.0",
+        "statuses": "2.0.1",
+        "toidentifier": "1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/raw-body/node_modules/inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+    },
+    "node_modules/raw-body/node_modules/setprototypeof": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
+    },
+    "node_modules/raw-body/node_modules/statuses": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+      "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/raw-body/node_modules/toidentifier": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+      "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
+      "engines": {
+        "node": ">=0.6"
+      }
+    },
     "node_modules/react-is": {
       "version": "16.9.0",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.9.0.tgz",
@@ -7519,6 +7694,19 @@
         "showdown": "bin/showdown.js"
       }
     },
+    "node_modules/side-channel": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
+      "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+      "dependencies": {
+        "call-bind": "^1.0.0",
+        "get-intrinsic": "^1.0.2",
+        "object-inspect": "^1.9.0"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/signal-exit": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
@@ -9904,20 +10092,82 @@
       }
     },
     "body-parser": {
-      "version": "1.19.0",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
-      "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
+      "version": "1.20.0",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
+      "integrity": "sha512-DfJ+q6EPcGKZD1QWUjSpqp+Q7bDQTsQIF4zfUAtZ6qk+H/3/QRhg9CEp39ss+/T2vw0+HaidC0ecJj/DRLIaKg==",
       "requires": {
-        "bytes": "3.1.0",
+        "bytes": "3.1.2",
         "content-type": "~1.0.4",
         "debug": "2.6.9",
-        "depd": "~1.1.2",
-        "http-errors": "1.7.2",
+        "depd": "2.0.0",
+        "destroy": "1.2.0",
+        "http-errors": "2.0.0",
         "iconv-lite": "0.4.24",
-        "on-finished": "~2.3.0",
-        "qs": "6.7.0",
-        "raw-body": "2.4.0",
-        "type-is": "~1.6.17"
+        "on-finished": "2.4.1",
+        "qs": "6.10.3",
+        "raw-body": "2.5.1",
+        "type-is": "~1.6.18",
+        "unpipe": "1.0.0"
+      },
+      "dependencies": {
+        "depd": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
+          "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="
+        },
+        "destroy": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
+          "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg=="
+        },
+        "http-errors": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
+          "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
+          "requires": {
+            "depd": "2.0.0",
+            "inherits": "2.0.4",
+            "setprototypeof": "1.2.0",
+            "statuses": "2.0.1",
+            "toidentifier": "1.0.1"
+          }
+        },
+        "inherits": {
+          "version": "2.0.4",
+          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+          "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+        },
+        "on-finished": {
+          "version": "2.4.1",
+          "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
+          "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
+          "requires": {
+            "ee-first": "1.1.1"
+          }
+        },
+        "qs": {
+          "version": "6.10.3",
+          "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz",
+          "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==",
+          "requires": {
+            "side-channel": "^1.0.4"
+          }
+        },
+        "setprototypeof": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+          "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
+        },
+        "statuses": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+          "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="
+        },
+        "toidentifier": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+          "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
+        }
       }
     },
     "brace-expansion": {
@@ -9996,9 +10246,9 @@
       "dev": true
     },
     "bytes": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
-      "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+      "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="
     },
     "cache-base": {
       "version": "1.0.1",
@@ -10021,7 +10271,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
       "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
-      "dev": true,
       "requires": {
         "function-bind": "^1.1.1",
         "get-intrinsic": "^1.0.2"
@@ -11000,6 +11249,41 @@
         "type-is": "~1.6.18",
         "utils-merge": "1.0.1",
         "vary": "~1.1.2"
+      },
+      "dependencies": {
+        "body-parser": {
+          "version": "1.19.0",
+          "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
+          "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
+          "requires": {
+            "bytes": "3.1.0",
+            "content-type": "~1.0.4",
+            "debug": "2.6.9",
+            "depd": "~1.1.2",
+            "http-errors": "1.7.2",
+            "iconv-lite": "0.4.24",
+            "on-finished": "~2.3.0",
+            "qs": "6.7.0",
+            "raw-body": "2.4.0",
+            "type-is": "~1.6.17"
+          }
+        },
+        "bytes": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
+          "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
+        },
+        "raw-body": {
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
+          "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
+          "requires": {
+            "bytes": "3.1.0",
+            "http-errors": "1.7.2",
+            "iconv-lite": "0.4.24",
+            "unpipe": "1.0.0"
+          }
+        }
       }
     },
     "express-rate-limit": {
@@ -11901,8 +12185,7 @@
     "function-bind": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
-      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==",
-      "dev": true
+      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
     },
     "functional-red-black-tree": {
       "version": "1.0.1",
@@ -11919,7 +12202,6 @@
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz",
       "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==",
-      "dev": true,
       "requires": {
         "function-bind": "^1.1.1",
         "has": "^1.0.3",
@@ -12055,7 +12337,6 @@
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
       "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
-      "dev": true,
       "requires": {
         "function-bind": "^1.1.1"
       }
@@ -12068,8 +12349,7 @@
     "has-symbols": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz",
-      "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==",
-      "dev": true
+      "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw=="
     },
     "has-value": {
       "version": "1.0.0",
@@ -13871,10 +14151,9 @@
       }
     },
     "object-inspect": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.6.0.tgz",
-      "integrity": "sha512-GJzfBZ6DgDAmnuaM3104jR4s1Myxr3Y3zfIyN4z3UdqN69oSRacNK8UhnobDdC+7J2AHCjGwxQubNJfE70SXXQ==",
-      "dev": true
+      "version": "1.12.2",
+      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.2.tgz",
+      "integrity": "sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ=="
     },
     "object-keys": {
       "version": "1.1.1",
@@ -14259,14 +14538,53 @@
       "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
     },
     "raw-body": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
-      "integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
+      "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
       "requires": {
-        "bytes": "3.1.0",
-        "http-errors": "1.7.2",
+        "bytes": "3.1.2",
+        "http-errors": "2.0.0",
         "iconv-lite": "0.4.24",
         "unpipe": "1.0.0"
+      },
+      "dependencies": {
+        "depd": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
+          "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="
+        },
+        "http-errors": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
+          "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==",
+          "requires": {
+            "depd": "2.0.0",
+            "inherits": "2.0.4",
+            "setprototypeof": "1.2.0",
+            "statuses": "2.0.1",
+            "toidentifier": "1.0.1"
+          }
+        },
+        "inherits": {
+          "version": "2.0.4",
+          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+          "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
+        },
+        "setprototypeof": {
+          "version": "1.2.0",
+          "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+          "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
+        },
+        "statuses": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+          "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="
+        },
+        "toidentifier": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+          "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
+        }
       }
     },
     "react-is": {
@@ -14788,6 +15106,16 @@
         "yargs": "^14.2"
       }
     },
+    "side-channel": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
+      "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+      "requires": {
+        "call-bind": "^1.0.0",
+        "get-intrinsic": "^1.0.2",
+        "object-inspect": "^1.9.0"
+      }
+    },
     "signal-exit": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",

package.json

@@ -1,11 +1,11 @@
 {
   "name": "gitblog.md",
-  "version": "1.3.0",
+  "version": "1.3.3",
   "description": "A static blog using Markdown pulled from your git repository.",
   "main": "src/server.js",
   "dependencies": {
     "@iarna/toml": "^2.2.3",
-    "body-parser": "^1.19.0",
+    "body-parser": "^1.20.0",
     "ejs": "^3.1.6",
     "express": "^4.17.1",
     "express-rate-limit": "^5.0.0",

src/app.js

@@ -61,6 +61,23 @@ module.exports = (config) => {
             }
         },
     );
+    const botDetector = require('./bot_detector')(config);
+    botDetector.load((status, err) => {
+        switch (status) {
+        case botDetector.status.FETCH_OK:
+            console.log(cons.ok, 'fetched robots list');
+            break;
+        case botDetector.status.FETCH_ERROR:
+            console.error(cons.error, 'error fetching robots list : ' + err);
+            break;
+        case botDetector.status.READ_OK:
+            console.log(cons.ok, `read robots list: ${botDetector.count}`);
+            break;
+        case botDetector.status.READ_ERROR:
+            console.error(cons.error, 'error reading robots list : ' + err);
+            break;
+        }
+    });
 
     // set view engine from configuration
     app.set('view engine', config['view_engine']);
@@ -145,6 +162,9 @@ module.exports = (config) => {
     });
     app.use(limiter);
 
+    //detect robots
+    app.use(botDetector.handle);
+
     //log request at result end
     app.use((req, res, next) => {
         if (config['access_log']) {
@@ -168,7 +188,7 @@ module.exports = (config) => {
             if (err) {
                 showError(req, res, 404);
             } else {
-                hc.count(req, '/', () => {
+                hc.count(req, '/', req.isRobot, () => {
                     render(req, res, homePath,
                         {
                             articles: Object.values(articles)
@@ -181,12 +201,25 @@ module.exports = (config) => {
     });
     app.get('/stats', (req, res) => {
         if (config['modules']['hit_counter']) {
-            hc.read('/', (data) => {
-                res.json({
-                    hits: data.hits,
-                    visitors: data.visitors,
+            if (req.query['all']) {
+                const keys = Object.keys(articles).filter(key => !articles[key].draft);
+                keys.unshift('/');
+                const read = (i, outputData) => {
+                    if (i >= keys.length) {
+                        res.json(outputData);
+                    } else {
+                        hc.read(keys[i], (data) => {
+                            outputData.push(data);
+                            read(i + 1, outputData);
+                        });
+                    }
+                };
+                read(0, []);
+            } else {
+                hc.read('/', (data) => {
+                    res.json(data);
                 });
-            });
+            }
         } else {
             showError(req, res, 404);
         }
@@ -268,16 +301,13 @@ module.exports = (config) => {
             } else if (req.path.endsWith('stats')) {
                 if (config['modules']['hit_counter']) {
                     hc.read(articlePath, (data) => {
-                        res.json({
-                            hits: data.hits,
-                            visitors: data.visitors,
-                        });
+                        res.json(data);
                     });
                 } else {
                     showError(req, res, 404);
                 }
             } else {
-                hc.count(req, articlePath, () => {
+                hc.count(req, articlePath, req.isRobot, () => {
                     renderer.render(article.realPath, (err, html) => {
                         if (err) {
                             console.log(cons.error, `failed to render article ${req.path} : ${err}`);

src/bot_detector.js

@@ -0,0 +1,66 @@
+const https = require('https');
+const fs = require('fs');
+
+module.exports = (config) => {
+    const _this = {
+        status: {
+            FETCH_OK: 1,
+            FETCH_ERROR: 2,
+            READ_OK: 3,
+            READ_ERROR: 4,
+        },
+        count: [],
+        regex: null,
+    };
+
+    const fetchList = (cb) => {
+        https.get(config['robots']['list_url'], (res) => {
+            if (res.statusCode !== 200) {
+                cb(res.statusCode);
+            } else {
+                const file = fs.createWriteStream(config['robots']['list_file']);
+                res.pipe(file);
+                file.on('finish', () => {
+                    file.close(cb);
+                });
+            }
+        }).on('error', (err) => {
+            cb(err.message);
+        });
+    };
+
+    const readFile = (cb) => {
+        fs.readFile(config['robots']['list_file'], { encoding: 'utf-8' }, (err, data) => {
+            if (err) {
+                cb(err, undefined);
+            } else {
+                try {
+                    cb(undefined, JSON.parse(data));
+                } catch (err) {
+                    cb(err, undefined);
+                }
+            }
+        });
+    };
+
+    _this.load = (cb) => {
+        fetchList((err) => {
+            cb(err ? _this.status.FETCH_ERROR : _this.status.FETCH_OK, err);
+            readFile((err, data) => {
+                if (!err) {
+                    _this.count = data.length;
+                    _this.regex = new RegExp('(' + data.filter(v => v['pattern']).map(v => v['pattern'])
+                        .join('|') + ')');
+                }
+                cb(err ? _this.status.READ_ERROR : _this.status.READ_OK, err);
+            });
+        });
+    };
+
+    _this.handle = (req, res, next) => {
+        req.isRobot = !!((req.headers['user-agent'] || '').match(_this.regex));
+        next();
+    };
+
+    return _this;
+};

src/config.default.json

@@ -63,6 +63,10 @@
   "hit_counter": {
     "unique_visitor_timeout": 7200000
   },
+  "robots": {
+    "list_url": "https://raw.githubusercontent.com/atmire/COUNTER-Robots/master/COUNTER_Robots_list.json",
+    "list_file": "robots_list.json"
+  },
   "redis": {
     "host": "localhost",
     "port": 6379

src/hit_counter.js

@@ -8,15 +8,15 @@ module.exports = (config, onConnect, onError) => {
 
     const visitors = {};
 
-    const count = (req, path, cb) => {
-        if (!client.connected) {
+    const count = (req, path, disable, cb) => {
+        if (!client.connected || disable) {
             cb();
         } else {
             const ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress;
-            const key = path + ':' + ip;
+            visitors[path] = (visitors[path] || {});
             const now = Date.now();
-            const isNewVisitor = (now - (visitors[key] || 0)) > config['hit_counter']['unique_visitor_timeout'];
-            visitors[key] = now;
+            const isNewVisitor = (now - (visitors[path][ip] || 0)) > config['hit_counter']['unique_visitor_timeout'];
+            visitors[path][ip] = now;
             client
                 .multi()
                 .hincrby(path, 'h', 1)
@@ -25,17 +25,35 @@ module.exports = (config, onConnect, onError) => {
         }
     };
 
+    const cleanVisitors = (path) => {
+        visitors[path] = (visitors[path] || {});
+        const now = Date.now();
+        let count = 0;
+        for (let ip in visitors[path]) {
+            if ((now - visitors[path][ip]) > config['hit_counter']['unique_visitor_timeout']) {
+                delete visitors[path][ip];
+            } else {
+                count++;
+            }
+        }
+        return count;
+    };
+
     const read = (path, cb) => {
         if (!client.connected) {
             cb({
+                path: path,
                 hits: 0,
-                visitors: 0,
+                total_visitors: 0,
+                current_visitors: cleanVisitors(path),
             });
         } else {
             client.hgetall(path, (_, value) => {
                 cb({
-                    hits: value ? value.h || 0 : 0,
-                    visitors: value ? value.v || 0 : 0,
+                    path: path,
+                    hits: value ? parseInt(value.h) || 0 : 0,
+                    total_visitors: value ? parseInt(value.v) || 0 : 0,
+                    current_visitors: cleanVisitors(path),
                 });
             });
         }

test/app.test.js

@@ -197,22 +197,6 @@ describe('Test root path', () => {
                 });
         }, fail);
     });
-    test('404 index no stats', (done) => {
-        request(app).get('/stats')
-            .then((response) => {
-                expect(response.statusCode).toBe(404);
-                done();
-            });
-    });
-    test('200 index stats', (done) => {
-        config['modules']['hit_counter'] = true;
-        request(app).get('/stats')
-            .then((response) => {
-                expect(response.statusCode).toBe(200);
-                expect(response.body).toEqual({ hits: 0, visitors: 0 });
-                done();
-            });
-    });
 });
 
 describe('Test RSS feed', () => {
@@ -451,41 +435,8 @@ describe('Test articles rendering', () => {
                 });
         }, fail);
     });
-
-    test('404 article no stats', (done) => {
-        utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]);
-        utils.createEmptyFiles([
-            path.join(dataDir, '2019', '05', '05', 'index.md'),
-            path.join(dataDir, testTemplate),
-        ]);
-        app.reload(() => {
-            request(app).get('/2019/05/05/hello/stats')
-                .then((response) => {
-                    expect(response.statusCode).toBe(404);
-                    done();
-                });
-        });
-    });
-
-    test('200 index stats', (done) => {
-        config['modules']['hit_counter'] = true;
-        utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]);
-        utils.createEmptyFiles([
-            path.join(dataDir, '2019', '05', '05', 'index.md'),
-            path.join(dataDir, testTemplate),
-        ]);
-        app.reload(() => {
-            request(app).get('/2019/05/05/anything/stats')
-                .then((response) => {
-                    expect(response.statusCode).toBe(200);
-                    expect(response.body).toEqual({ hits: 0, visitors: 0 });
-                    done();
-                });
-        });
-    });
 });
 
-
 describe('Test static files', () => {
     test('404 invalid file no error page', (done) => {
         request(app).get('/somefile.txt')
@@ -566,3 +517,113 @@ describe('Test other requests', () => {
             });
     });
 });
+
+
+describe('Test stats', () => {
+    test('404 index no stats', (done) => {
+        request(app).get('/stats')
+            .then((response) => {
+                expect(response.statusCode).toBe(404);
+                done();
+            });
+    });
+    test('200 index stats', (done) => {
+        config['modules']['hit_counter'] = true;
+        request(app).get('/stats')
+            .then((response) => {
+                expect(response.statusCode).toBe(200);
+                expect(response.body).toEqual({
+                    path: '/',
+                    hits: 0,
+                    total_visitors: 0,
+                    current_visitors: 0,
+                });
+                done();
+            });
+    });
+    test('200 index stats all no article', (done) => {
+        config['modules']['hit_counter'] = true;
+        app.reload(() => {
+            request(app).get('/stats?all=true')
+                .then((response) => {
+                    expect(response.statusCode).toBe(200);
+                    expect(response.body).toEqual([
+                        {
+                            path: '/',
+                            hits: 0,
+                            total_visitors: 0,
+                            current_visitors: 0,
+                        },
+                    ]);
+                    done();
+                });
+        });
+    });
+    test('200 index stats all 2 article 1 drafted', (done) => {
+        config['modules']['hit_counter'] = true;
+        utils.createEmptyDirs([
+            path.join(dataDir, '2019', '05', '05'),
+            path.join(dataDir, '2019', '04', '05'),
+        ]);
+        utils.createEmptyFiles([
+            path.join(dataDir, '2019', '05', '05', 'index.md'),
+            path.join(dataDir, '2019', '04', '05', 'draft.md'),
+        ]);
+        app.reload(() => {
+            request(app).get('/stats?all=true')
+                .then((response) => {
+                    expect(response.statusCode).toBe(200);
+                    expect(response.body).toEqual([
+                        {
+                            path: '/',
+                            hits: 0,
+                            total_visitors: 0,
+                            current_visitors: 0,
+                        },
+                        {
+                            path: '2019/05/05',
+                            hits: 0,
+                            total_visitors: 0,
+                            current_visitors: 0,
+                        },
+                    ]);
+                    done();
+                });
+        });
+    });
+    test('404 article no stats', (done) => {
+        utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]);
+        utils.createEmptyFiles([
+            path.join(dataDir, '2019', '05', '05', 'index.md'),
+            path.join(dataDir, testTemplate),
+        ]);
+        app.reload(() => {
+            request(app).get('/2019/05/05/hello/stats')
+                .then((response) => {
+                    expect(response.statusCode).toBe(404);
+                    done();
+                });
+        });
+    });
+    test('200 article stats', (done) => {
+        config['modules']['hit_counter'] = true;
+        utils.createEmptyDirs([ path.join(dataDir, '2019', '05', '05') ]);
+        utils.createEmptyFiles([
+            path.join(dataDir, '2019', '05', '05', 'index.md'),
+            path.join(dataDir, testTemplate),
+        ]);
+        app.reload(() => {
+            request(app).get('/2019/05/05/anything/stats')
+                .then((response) => {
+                    expect(response.statusCode).toBe(200);
+                    expect(response.body).toEqual({
+                        path: '2019/05/05',
+                        hits: 0,
+                        total_visitors: 0,
+                        current_visitors: 0,
+                    });
+                    done();
+                });
+        });
+    });
+});

test/bot_detector.test.js

@@ -0,0 +1,116 @@
+const fs = require('fs');
+const utils = require('./test_utils');
+
+const dataDir = 'test_data';
+
+const config = {
+    robots: {
+        list_url: '',
+        list_file: `${dataDir}/robots_list.json`,
+    },
+};
+
+
+beforeAll(() => {
+    utils.deleteFolderSync(dataDir);
+    fs.mkdirSync(dataDir);
+});
+
+afterAll(() => {
+    if (fs.existsSync(dataDir)) {
+        utils.deleteFolderSync(dataDir);
+    }
+});
+
+const botDetector = require('../src/bot_detector')(config);
+
+describe('load()', () => {
+    test('success', (done) => {
+        config.robots = {
+            list_url: 'https://raw.githubusercontent.com/atmire/COUNTER-Robots/master/COUNTER_Robots_list.json',
+            list_file: `${dataDir}/robots_list_success.json`,
+        };
+        let count = 0;
+        botDetector.load((status, err) => {
+            expect(err).not.toBeDefined();
+            expect(status).toBe(count === 0 ? botDetector.status.FETCH_OK : botDetector.status.READ_OK);
+            if (count > 0) {
+                done();
+            }
+            count++;
+        });
+    });
+
+    test('fetch and file failure', (done) => {
+        let count = 0;
+        config.robots = {
+            list_url: 'https://blog.klemek.fr/invalid.json',
+            list_file: `${dataDir}/robots_list_fail_1.json`,
+        };
+        botDetector.load((status) => {
+            expect(status).toBe(count === 0 ? botDetector.status.FETCH_ERROR : botDetector.status.READ_ERROR);
+            if (count > 0) {
+                done();
+            }
+            count++;
+        });
+    });
+
+    test('fetch failure and file ok', (done) => {
+        let count = 0;
+        config.robots = {
+            list_url: 'https://blog.klemek.fr/invalid.json',
+            list_file: `${dataDir}/robots_list_fail_2.json`,
+        };
+        fs.writeFile(config.robots.list_file, '[]\n', { encoding: 'utf-8' }, () => {
+            botDetector.load((status) => {
+                expect(status).toBe(count === 0 ? botDetector.status.FETCH_ERROR : botDetector.status.READ_OK);
+                if (count > 0) {
+                    done();
+                }
+                count++;
+            });
+        });
+    });
+});
+
+
+describe('handle()', () => {
+    beforeAll((done) => {
+        config.robots = {
+            list_url: 'https://blog.klemek.fr/invalid.json',
+            list_file: `${dataDir}/robots_list_fake.json`,
+        };
+        fs.writeFile(config.robots.list_file, '[{"pattern":"bot"}]\n', { encoding: 'utf-8' }, () => {
+            botDetector.load((status) => {
+                if (status !== botDetector.status.FETCH_ERROR) {
+                    done();
+                }
+            });
+        });
+    });
+
+    test('not bot', (done) => {
+        const req = {
+            headers: {
+                'user-agent': 'my user agent',
+            },
+        };
+        botDetector.handle(req, null, () => {
+            expect(req.isRobot).toBeFalsy();
+            done();
+        });
+    });
+
+    test('bot', (done) => {
+        const req = {
+            headers: {
+                'user-agent': 'bot',
+            },
+        };
+        botDetector.handle(req, null, () => {
+            expect(req.isRobot).toBeTruthy();
+            done();
+        });
+    });
+});

test/hit_counter.test.js

@@ -37,14 +37,21 @@ test('options passed to redis', () => {
     expect(mockClient.options).toEqual(config['redis']);
 });
 
-describe('read()', () => {
-    beforeEach(() => {
-        mockClient.hgetall = (_, cb) => {
-            cb();
-        };
-    });
 
-    test('read path', (done) => {
+beforeEach(() => {
+    mockClient.hgetall = (_, cb) => {
+        cb();
+    };
+    mockClient.multi = () => mockClient;
+    mockClient.hincrby = () => mockClient;
+    mockClient.exec = (cb) => {
+        cb();
+    };
+    config['hit_counter']['unique_visitor_timeout'] = -1;
+});
+
+describe('read()', () => {
+    test('normal', (done) => {
         mockClient.hgetall = (path, cb) => {
             expect(path).toBe('/test/path/');
             cb(undefined, { h: 12, v: 34 });
@@ -52,12 +59,13 @@ describe('read()', () => {
         hc.read('/test/path/', (data) => {
             expect(data).toBeDefined();
             expect(data.hits).toBe(12);
-            expect(data.visitors).toBe(34);
+            expect(data.total_visitors).toBe(34);
+            expect(data.current_visitors).toBe(0);
             done();
         });
     });
 
-    test('read path with error', (done) => {
+    test('with error', (done) => {
         mockClient.hgetall = (path, cb) => {
             expect(path).toBe('/test/path/');
             cb('error', undefined);
@@ -65,12 +73,13 @@ describe('read()', () => {
         hc.read('/test/path/', (data) => {
             expect(data).toBeDefined();
             expect(data.hits).toBe(0);
-            expect(data.visitors).toBe(0);
+            expect(data.total_visitors).toBe(0);
+            expect(data.current_visitors).toBe(0);
             done();
         });
     });
 
-    test('read path with error 2', (done) => {
+    test('with error 2', (done) => {
         mockClient.hgetall = (path, cb) => {
             expect(path).toBe('/test/path/');
             cb(undefined, {});
@@ -78,22 +87,41 @@ describe('read()', () => {
         hc.read('/test/path/', (data) => {
             expect(data).toBeDefined();
             expect(data.hits).toBe(0);
-            expect(data.visitors).toBe(0);
+            expect(data.total_visitors).toBe(0);
+            expect(data.current_visitors).toBe(0);
             done();
         });
     });
+
+    test('1 visitor', (done) => {
+        config['hit_counter']['unique_visitor_timeout'] = 1000;
+        hc.count({
+            headers: {},
+            connection: { remoteAddress: 'test1' },
+        }, '/test/path/5', false, () => {
+            hc.read('/test/path/5', (data) => {
+                expect(data).toBeDefined();
+                expect(data.current_visitors).toBe(1);
+                done();
+            });
+        });
+    });
+
+    test('cleaned old visitor', (done) => {
+        hc.count({
+            headers: {},
+            connection: { remoteAddress: 'test1' },
+        }, '/test/path/5', false, () => {
+            hc.read('/test/path/5', (data) => {
+                expect(data).toBeDefined();
+                expect(data.current_visitors).toBe(0);
+                done();
+            });
+        });
+    });
 });
 
 describe('count()', () => {
-    beforeEach(() => {
-        mockClient.multi = () => mockClient;
-        mockClient.hincrby = () => mockClient;
-        mockClient.exec = (cb) => {
-            cb();
-        };
-        config['hit_counter']['unique_visitor_timeout'] = -1;
-    });
-
     test('simple visit', (done) => {
         let multiCalled = false;
         let execCalled = false;
@@ -117,7 +145,7 @@ describe('count()', () => {
         hc.count({
             headers: {},
             connection: { remoteAddress: 'test1' },
-        }, '/test/path/1', () => {
+        }, '/test/path/1', false, () => {
             expect(multiCalled).toBeTruthy();
             expect(hincrbyCalls).toEqual([
                 [
@@ -149,11 +177,11 @@ describe('count()', () => {
         hc.count({
             headers: {},
             connection: { remoteAddress: 'test2' },
-        }, '/test/path/2', () => {
+        }, '/test/path/2', false, () => {
             hc.count({
                 headers: {},
                 connection: { remoteAddress: 'test2' },
-            }, '/test/path/2', () => {
+            }, '/test/path/2', false, () => {
                 expect(hincrbyCalls).toEqual([
                     [
                         '/test/path/2',
@@ -195,11 +223,11 @@ describe('count()', () => {
         hc.count({
             headers: {},
             connection: { remoteAddress: 'test3' },
-        }, '/test/path/3', () => {
+        }, '/test/path/3', false, () => {
             hc.count({
                 headers: {},
                 connection: { remoteAddress: 'test3' },
-            }, '/test/path/3', () => {
+            }, '/test/path/3', false, () => {
                 expect(hincrbyCalls).toEqual([
                     [
                         '/test/path/3',