Merge pull request #55 from Klemek/dev

v1.16

README.md

@@ -125,6 +125,11 @@ python3 src/main.py
 
 ## Changelog
 
+* **v1.16**
+  * `%freq graph` graph hours frequency along the week
+  * uses discord new time format
+  * `%freq` now shows quietest day of week and hour of day
+  * improvments and bug fix
 * **v1.15**
   * `nsfw:allow/only` filter nsfw channels
   * `%find` can use regexes

requirements.txt

@@ -2,3 +2,5 @@ discord.py==1.7.0
 python-dotenv==0.15.0
 python-dateutil==2.8.1
 git+git://github.com/Klemek/miniscord.git
+numpy
+matplotlib

src/data_types/frequency.py

@@ -1,10 +1,13 @@
 from typing import List
 from datetime import timedelta
 import calendar
+import matplotlib.pyplot as plt
+import numpy as np
+from io import BytesIO
+import discord
+import time
 
 from utils import (
-    str_date,
-    str_datetime,
     from_now,
     plural,
     percent,
@@ -19,8 +22,7 @@ class Frequency:
         self.dates = []
         self.longest_break = timedelta(seconds=0)
         self.longest_break_start = None
-        self.week = {i: 0 for i in range(7)}
+        self.hours = {i: {j: 0 for j in range(24)} for i in range(7)}
-        self.day = {i: 0 for i in range(24)}
         self.busiest_day = None
         self.busiest_day_count = 0
         self.busiest_hour = None
@@ -33,6 +35,49 @@ class Frequency:
         self.longest_streak_start = None
         self.longest_streak_author = None
 
+    def to_graph(self) -> List[str]:
+        self.dates.sort()
+        delta = self.dates[-1] - self.dates[0]
+        if delta.days == 0:
+            delta = timedelta(days=1)
+        day = {j: sum(self.hours[i][j] for i in range(7)) for j in range(24)}
+        busiest_hour = top_key(day)
+        n_hours = delta.days
+        if self.dates[0].hour <= busiest_hour and self.dates[-1].hour >= busiest_hour:
+            n_hours += 1
+
+        plt.style.use("dark_background")
+
+        fig, ax = plt.subplots()
+
+        fig.patch.set_facecolor("#36393F")
+        ax.patch.set_alpha(0)
+
+        times = range(25)
+
+        for i in range(7):
+            hours = [self.hours[i][hour] * 7 / n_hours for hour in range(24)] + [
+                self.hours[i][0] * 7 / n_hours
+            ]
+            ax.plot(
+                times, hours, label=calendar.day_name[i], linestyle="--", linewidth=0.8
+            )
+
+        hours = [day[hour] / n_hours for hour in range(24)] + [day[0] / n_hours]
+        ax.plot(times, hours, c="r", label="average", linewidth=1.5)
+
+        ax.set_xlabel("hour of day")
+        ax.set_xlim([0, 24])
+        ax.set_ylabel("average messages")
+        ax.legend(framealpha=0)
+
+        with BytesIO() as f:
+            plt.savefig(
+                f, format="png", facecolor=fig.get_facecolor(), edgecolor="none"
+            )
+            f.seek(0)
+            return [discord.File(f, f"{time.time()}-plot.png")]
+
     def to_string(
         self,
         *,
@@ -43,8 +88,14 @@ class Frequency:
         if delta.days == 0:
             delta = timedelta(days=1)
         total_msg = len(self.dates)
-        busiest_weekday = top_key(self.week)
+
-        busiest_hour = top_key(self.day)
+        week = {i: sum(self.hours[i].values()) for i in range(7)}
+        day = {j: sum(self.hours[i][j] for i in range(7)) for j in range(24)}
+
+        busiest_weekday = top_key(week)
+        busiest_hour = top_key(day)
+        quietest_weekday = top_key(week, reverse=True)
+        quietest_hour = top_key(day, reverse=True)
         n_weekdays = delta.days // 7
         if (
             self.dates[0].weekday() <= busiest_weekday
@@ -55,20 +106,26 @@ class Frequency:
         if self.dates[0].hour <= busiest_hour and self.dates[-1].hour >= busiest_hour:
             n_hours += 1
         ret = [
-            f"- **earliest message**: {str_datetime(self.dates[0])} ({from_now(self.dates[0])})",
+            f"- **earliest message**: {from_now(self.dates[0])}",
-            f"- **latest message**: {str_datetime(self.dates[-1])} ({from_now(self.dates[-1])})",
+            f"- **latest message**: {from_now(self.dates[-1])}",
             f"- **messages/day**: {precise(total_msg/delta.days, precision=3)}",
-            f"- **busiest day of week**: {calendar.day_name[busiest_weekday]} (~{precise(self.week[busiest_weekday]/n_weekdays, precision=3)} msg, {percent(self.week[busiest_weekday]/total_msg)})",
+            f"- **busiest day of week**: {calendar.day_name[busiest_weekday]} (~{precise(week[busiest_weekday]/n_weekdays, precision=3)} msg, {percent(week[busiest_weekday]/total_msg)})",
-            f"- **busiest day ever**: {str_date(self.busiest_day)} ({from_now(self.busiest_day)}, {self.busiest_day_count} msg)"
+            f"- **quietest day of week**: {calendar.day_name[quietest_weekday]} (~{precise(week[quietest_weekday]/n_weekdays, precision=3)} msg, {percent(week[quietest_weekday]/total_msg)})"
+            if week[quietest_weekday] > 0
+            else "",
+            f"- **busiest day ever**: {from_now(self.busiest_day)} ({self.busiest_day_count} msg)"
             if self.busiest_day is not None
             else "",
             f"- **messages/hour**: {precise(total_msg*3600/delta.total_seconds(), precision=3)}",
-            f"- **busiest hour of day**: {busiest_hour:0>2}:00 (~{precise(self.day[busiest_hour]/n_hours, precision=3)} msg, {percent(self.day[busiest_hour]/total_msg)})",
+            f"- **busiest hour of day**: {busiest_hour:0>2}:00 (~{precise(day[busiest_hour]/n_hours, precision=3)} msg, {percent(day[busiest_hour]/total_msg)})",
-            f"- **busiest hour ever**: {str_datetime(self.busiest_hour)} ({from_now(self.busiest_hour)}, {self.busiest_hour_count} msg)",
+            f"- **quietest hour of day**: {quietest_hour:0>2}:00 (~{precise(day[quietest_hour]/n_hours, precision=3)} msg, {percent(day[quietest_hour]/total_msg)})"
-            f"- **longest break**: {plural(round(self.longest_break.total_seconds()/3600), 'hour')} ({plural(self.longest_break.days,'day')}) from {str_datetime(self.longest_break_start)} ({from_now(self.longest_break_start)})",
+            if day[quietest_hour] > 0
+            else "",
+            f"- **busiest hour ever**: {from_now(self.busiest_hour)} ({self.busiest_hour_count} msg)",
+            f"- **longest break**: {plural(round(self.longest_break.total_seconds()/3600), 'hour')} ({plural(self.longest_break.days,'day')}), started {from_now(self.longest_break_start)}",
             f"- **avg. streak**: {precise(sum(self.streaks)/len(self.streaks), precision=3)} msg",
-            f"- **longest streak**: {self.longest_streak:,} msg from {str_datetime(self.longest_streak_start)} ({from_now(self.longest_streak_start)})"
+            f"- **longest streak**: {self.longest_streak:,} msg, started {from_now(self.longest_streak_start)}"
             if member_specific
-            else f"- **longest streak**: {mention(self.longest_streak_author)} ({self.longest_streak:,} msg from {str_datetime(self.longest_streak_start)}, {from_now(self.longest_streak_start)})",
+            else f"- **longest streak**: {mention(self.longest_streak_author)} ({self.longest_streak:,} msg, started {from_now(self.longest_streak_start)})",
         ]
         return ret

src/data_types/history.py

@@ -6,7 +6,6 @@ import random
 from utils import (
     mention,
     from_now,
-    str_datetime,
     message_link,
     SPLIT_TOKEN,
     FilterLevel,
@@ -78,7 +77,7 @@ class History:
 
         return [
             intro,
-            f"{str_datetime(message.created_at)} ({from_now(message.created_at)}) {mention(message.author)} sent:",
+            f"{from_now(message.created_at)}, {mention(message.author)} sent:",
             f"<{message_link(message)}>",
             SPLIT_TOKEN,
             image,
@@ -107,7 +106,7 @@ class History:
 
         return [
             intro,
-            f"{str_datetime(message.created_at)} ({from_now(message.created_at)}) {mention(message.author)} said:",
+            f"{from_now(message.created_at)}, {mention(message.author)} said:",
             *text,
             f"<{message_link(message)}>",
         ]

src/logs/channel_logs.py

@@ -51,7 +51,7 @@ class ChannelLogs:
     def preload(self, channel: discord.TextChannel):
         self.name = channel.name
         self.channel = channel
-    
+
     @property
     def sorted_messages(self):
         return sorted(self.messages)

src/logs/message_log.py

@@ -66,7 +66,7 @@ class MessageLog:
     def __eq__(self, other: object) -> bool:
         return isinstance(other, self.__class__) and other.id == self.id
 
-    def __gt__(self, other: 'MessageLog') -> bool:
+    def __gt__(self, other: "MessageLog") -> bool:
         return self.created_at > other.created_at
 
     def __hash__(self) -> int:

src/main.py

@@ -18,7 +18,7 @@ emojis.load_emojis()
 
 bot = Bot(
     "Discord Analyst",
-    "1.15.3",
+    "1.16",
     alias="%",
 )
 

src/scanners/frequency_scanner.py

@@ -14,11 +14,17 @@ from utils import generate_help
 class FrequencyScanner(Scanner):
     @staticmethod
     def help() -> str:
-        return generate_help("freq", "Show frequency-related statistics")
+        return generate_help(
+            "freq",
+            "(BETA) Show frequency-related statistics",
+            args=[
+                "graph - plot hours of week",
+            ],
+        )
 
     def __init__(self):
         super().__init__(
-            valid_args=["all", "everyone"],
+            valid_args=["all", "everyone", "graph"],
             help=FrequencyScanner.help(),
             intro_context="Frequency",
         )
@@ -26,6 +32,8 @@ class FrequencyScanner(Scanner):
     async def init(self, message: discord.Message, *args: str) -> bool:
         self.freq = Frequency()
         self.all_messages = "all" in args or "everyone" in args
+        self.member_specific = len(self.members) > 0
+        self.to_graph = "graph" in args
         return True
 
     def compute_message(self, channel: ChannelLogs, message: MessageLog):
@@ -35,10 +43,13 @@ class FrequencyScanner(Scanner):
 
     def get_results(self, intro: str) -> List[str]:
         FrequencyScanner.compute_results(self.freq)
-        res = [intro]
+        if self.to_graph:
-        res += self.freq.to_string(
+            res = self.freq.to_graph()
-            member_specific=self.member_specific,
+        else:
-        )
+            res = [intro]
+            res += self.freq.to_string(
+                member_specific=self.member_specific,
+            )
         return res
 
     @staticmethod
@@ -90,8 +101,7 @@ class FrequencyScanner(Scanner):
                 freq.longest_break_start = latest
             latest = date
             # calculate busiest weekday / hours
-            freq.week[date.weekday()] += 1
+            freq.hours[date.weekday()][date.hour] += 1
-            freq.day[date.hour] += 1
             # calculate busiest day ever
             start_delta = date - freq.dates[0]
             if start_delta.days > current_day:

src/scanners/scanner.py

@@ -92,7 +92,9 @@ class Scanner(ABC):
                 dates = []
                 for i, arg in enumerate(args[1:]):
                     skip_check = False
-                    if self.all_args and f"'{arg}'" in message.content or f"\"{arg}\"" in message.content:
+                    if self.all_args and (
+                        f"'{arg}'" in message.content or f'"{arg}"' in message.content
+                    ):
                         self.other_args += [arg]
                     elif re.match(r"^<@!?\d+>$", arg):
                         arg = arg[3:-1] if "!" in arg else arg[2:-1]
@@ -286,15 +288,20 @@ class Scanner(ABC):
                                 if self.mention_users
                                 else discord.AllowedMentions.none()
                             )
+                            file = None
                             for r in results:
                                 if r:
-                                    if isinstance(r, int) and r == SPLIT_TOKEN:
+                                    if isinstance(r, discord.File):
+                                        file = r
+                                    elif isinstance(r, int) and r == SPLIT_TOKEN:
                                         await message.channel.send(
                                             response,
                                             reference=message if first else None,
                                             allowed_mentions=allowed_mentions,
+                                            file=file,
                                         )
                                         first = False
+                                        file = None
                                         response = ""
                                     elif isinstance(r, str):
                                         if len(response + "\n" + r) > 2000:
@@ -302,15 +309,18 @@ class Scanner(ABC):
                                                 response,
                                                 reference=message if first else None,
                                                 allowed_mentions=allowed_mentions,
+                                                file=file,
                                             )
                                             first = False
+                                            file = None
                                             response = ""
                                         response += "\n" + r
-                            if len(response) > 0:
+                            if len(response) > 0 or file is not None:
                                 await message.channel.send(
                                     response,
                                     reference=message if first else None,
                                     allowed_mentions=allowed_mentions,
+                                    file=file,
                                 )
                             command_cache.cache(self, message, args)
                 # Delete custom progress message

src/utils/utils.py

@@ -6,6 +6,7 @@ import discord
 import math
 from datetime import datetime, timedelta
 import re
+import time
 import dateutil.parser
 from dateutil.relativedelta import relativedelta
 
@@ -177,13 +178,13 @@ def no_duplicate(seq: list) -> list:
 
 
 def top_key(
-    d: Dict[Union[str, int], int], key: Optional[Callable] = None
+    d: Dict[Union[str, int], int], key: Optional[Callable] = None, reverse: bool = False
 ) -> Union[str, int]:
     if len(d) == 0:
         return None
     if key is None:
         key = lambda k: d[k]
-    return sorted(d, key=key)[-1]
+    return sorted(d, key=key, reverse=reverse)[-1]
 
 
 def val_sum(d: Dict[Any, int]) -> int:
@@ -290,11 +291,11 @@ def parse_time(src: str) -> datetime:
 
 
 def str_date(date: datetime) -> str:
-    return date.strftime("%d %b. %Y")  # 12 Jun. 2018
+    return f"<t:{int(time.mktime(date.timetuple()))}:D>"
 
 
 def str_datetime(date: datetime) -> str:
-    return date.strftime("%H:%M, %d %b. %Y")  # 12:05, 12 Jun. 2018
+    return f"<t:{int(time.mktime(date.timetuple()))}:f>"
 
 
 def str_delta(delay: timedelta) -> str:
@@ -322,12 +323,7 @@ def str_delta(delay: timedelta) -> str:
 def from_now(src: Optional[datetime]) -> str:
     if src is None:
         return "never"
-    output = str_delta(datetime.utcnow() - src)
+    return f"<t:{int(time.mktime(src.timetuple()))}:R>"
-    if output == "no time":
-        return "now"
-    elif output == "one day":
-        return "yesterday"
-    return output + " ago"
 
 
 # APP SPECIFIC

test-requirements.txt

@@ -0,0 +1,3 @@
+pytest~=6.2.3
+pytest-cov
+coveralls

tests/integration/scanners/test_first_scanner.py

@@ -0,0 +1,90 @@
+from unittest import TestCase
+from unittest.mock import MagicMock
+from src.scanners import FirstScanner
+from datetime import datetime, timedelta
+
+from tests.utils import AsyncTestCase, fake_message
+
+
+class TestFirstScanner(AsyncTestCase):
+    def test_help(self):
+        self.assertGreater(len(FirstScanner.help()), 0)
+        self.assertIn("%first", FirstScanner.help())
+
+    def test_empty_no_messages(self):
+        scanner = FirstScanner()
+
+        command_msg = MagicMock()
+        self._await(scanner.init(command_msg, []))
+
+        results = self._await(scanner.get_results(""))
+        self.assertListEqual(["There was no messages matching your filters"], results)
+
+    def test_empty_filtered(self):
+        scanner = FirstScanner()
+        scanner.raw_members = [1]
+
+        self._await(scanner.init(fake_message(), []))
+
+        messages = [fake_message(author=2), fake_message(author=3)]
+
+        for msg in messages:
+            scanner.compute_message(msg.channel, msg)
+
+        results = self._await(scanner.get_results(""))
+        self.assertListEqual(["There was no messages matching your filters"], results)
+
+    def test_normal(self):
+        scanner = FirstScanner()
+
+        self._await(scanner.init(fake_message(), []))
+
+        messages = [
+            fake_message(id=1, created_at=timedelta(days=-2)),
+            fake_message(id=2, created_at=timedelta(days=-3)),
+            fake_message(id=3, created_at=timedelta(days=-1)),
+        ]
+
+        for msg in messages:
+            scanner.compute_message(msg.channel, msg)
+
+        results = self._await(scanner.get_results(""))
+
+        expected = messages[1]
+        self.assertListEqual(
+            [
+                "First message out of 3",
+                f"{expected.created_at.strftime('%H:%M, %d %b. %Y')} (2 days ago) <@1> said:",
+                f"> {expected.content}",
+                "<https://discord.com/channels/1/1/2>",
+            ],
+            results,
+        )
+
+    def test_filtered(self):
+        scanner = FirstScanner()
+        scanner.raw_members = [1]
+
+        self._await(scanner.init(fake_message(), []))
+
+        messages = [
+            fake_message(id=1, author=1, created_at=timedelta(days=-2)),
+            fake_message(id=2, author=2, created_at=timedelta(days=-3)),
+            fake_message(id=3, author=1, created_at=timedelta(days=-1)),
+        ]
+
+        for msg in messages:
+            scanner.compute_message(msg.channel, msg)
+
+        results = self._await(scanner.get_results(""))
+
+        expected = messages[0]
+        self.assertListEqual(
+            [
+                "First message out of 2",
+                f"{expected.created_at.strftime('%H:%M, %d %b. %Y')} (yesterday) <@1> said:",
+                f"> {expected.content}",
+                "<https://discord.com/channels/1/1/1>",
+            ],
+            results,
+        )

tests/utils.py

@@ -0,0 +1,99 @@
+from typing import List, Optional, Dict, Union
+from unittest import TestCase
+import asyncio
+from datetime import datetime, timedelta
+from unittest.mock import MagicMock
+import random
+import string
+
+
+class AsyncTestCase(TestCase):
+    def setUp(self):
+        self.loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(None)
+
+    def tearDown(self):
+        self.loop.close()
+
+    def _await(self, fn):
+        return self.loop.run_until_complete(fn)
+
+
+RANDOM_TEXT_CHARS = string.ascii_letters + string.digits + string.punctuation
+
+
+def random_text(min_len: int = 3, max_len: int = 45):
+    return "".join(
+        random.choice(RANDOM_TEXT_CHARS)
+        for _ in range(random.randrange(min_len, max_len))
+    )
+
+
+def fake_guild(id: int = 1):
+    return MagicMock(id=id)
+
+
+def fake_channel(id: int = 1, name: str = "fake-channel"):
+    return MagicMock(id=id, name=name, guild=fake_guild())
+
+
+def fake_message(
+    id: int = 1,
+    channel_id: int = 1,
+    channel_name: str = "fake-channel",
+    created_at: Optional[Union[datetime, timedelta]] = None,
+    edited_at: Optional[datetime] = None,
+    author: int = 1,
+    pinned: bool = False,
+    mention_everyone: bool = False,
+    tts: bool = False,
+    bot: bool = False,
+    content: Optional[str] = None,
+    mentions: Optional[List[int]] = None,
+    reference: Optional[int] = None,
+    role_mentions: Optional[List[int]] = None,
+    channel_mentions: Optional[List[int]] = None,
+    image: bool = False,
+    attachment: bool = False,
+    embed: bool = False,
+    reactions: Optional[Dict[str, List[int]]] = None,
+):
+    if created_at is None:
+        created_at = datetime.now() + timedelta(hours=random.randrange(-30 * 24, 0))
+    elif isinstance(created_at, timedelta):
+        created_at = datetime.now() + created_at
+    if isinstance(edited_at, timedelta):
+        edited_at = datetime.now() + edited_at
+    if content is None:
+        content = random_text()
+    if mentions is None:
+        mentions = []
+    if role_mentions is None:
+        role_mentions = []
+    if channel_mentions is None:
+        channel_mentions = []
+    if reactions is None:
+        reactions = {}
+    return MagicMock(
+        id=id,
+        channel=fake_channel(channel_id, channel_name),
+        created_at=created_at,
+        edited_at=edited_at,
+        author=author,
+        pinned=pinned,
+        mention_everyone=mention_everyone,
+        tts=tts,
+        bot=bot,
+        content=content,
+        mentions=mentions,
+        raw_mentions=mentions,
+        reference=reference,
+        role_mentions=role_mentions,
+        raw_role_mentions=role_mentions,
+        channel_mentions=channel_mentions,
+        raw_channel_mentions=channel_mentions,
+        image=image,
+        attachment=attachment,
+        embed=embed,
+        reactions=reactions,
+    )