Commit e48780dd authored by mathieui's avatar mathieui

Merge branch 'fix-history-fetch' into 'master'

Fix many MAM issues

Closes #3516, #3496, #3498, #3506, #3522, and #3493

See merge request !105
parents 4c1ab027 faeab78c
Pipeline #3138 passed with stages
in 6 minutes and 28 seconds
......@@ -8,7 +8,11 @@
Various useful functions.
"""
from datetime import datetime, timedelta
from datetime import (
datetime,
timedelta,
timezone,
)
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union
......@@ -488,3 +492,14 @@ def unique_prefix_of(a: str, b: str) -> str:
return a[:i+1]
# both are equal, return a
return a
def to_utc(time: datetime) -> datetime:
"""Convert a datetime-aware time zone into raw UTC"""
tzone = datetime.now().astimezone().tzinfo
if time.tzinfo is not None: # Convert to UTC
time = time.astimezone(tz=timezone.utc)
else: # Assume local tz, convert to URC
time = time.replace(tzinfo=tzone).astimezone(tz=timezone.utc)
# Return an offset-naive datetime
return time.replace(tzinfo=None)
......@@ -2075,16 +2075,7 @@ class Core:
# do not join rooms that do not have autojoin
# but display them anyway
if bm.autojoin:
muc.join_groupchat(
self,
bm.jid,
nick,
passwd=bm.password,
status=self.status.message,
show=self.status.show,
tab=tab)
if tab._text_buffer.last_message is None:
asyncio.ensure_future(mam.on_tab_open(tab))
tab.join()
def check_bookmark_storage(self, features):
private = 'jabber:iq:private' in features
......
This diff is collapsed.
......@@ -32,7 +32,6 @@ from typing import (
)
from poezio import (
mam,
poopt,
timed_events,
xhtml,
......@@ -493,12 +492,11 @@ class ChatTab(Tab):
self._jid = jid
#: Is the tab currently requesting MAM data?
self.query_status = False
self.last_stanza_id = None
self._name = jid.full # type: Optional[str]
self.text_win = None
self.text_win = windows.TextWin()
self.directed_presence = None
self._text_buffer = TextBuffer()
self._text_buffer.add_window(self.text_win)
self.chatstate = None # can be "active", "composing", "paused", "gone", "inactive"
# We keep a reference of the event that will set our chatstate to "paused", so that
# we can delete it or change it if we need to
......@@ -926,7 +924,8 @@ class ChatTab(Tab):
def on_scroll_up(self):
if not self.query_status:
asyncio.ensure_future(mam.on_scroll_up(tab=self))
from poezio import mam
mam.schedule_scroll_up(tab=self)
return self.text_win.scroll_up(self.text_win.height - 1)
def on_scroll_down(self):
......
......@@ -48,8 +48,6 @@ class ConversationTab(OneToOneTab):
self.nick = None
self.nick_sent = False
self.state = 'normal'
self.text_win = windows.TextWin()
self._text_buffer.add_window(self.text_win)
self.upper_bar = windows.ConversationStatusMessageWin()
self.input = windows.MessageInput()
# keys
......
......@@ -31,7 +31,7 @@ from poezio import multiuserchat as muc
from poezio import timed_events
from poezio import windows
from poezio import xhtml
from poezio.common import safeJID
from poezio.common import safeJID, to_utc
from poezio.config import config
from poezio.core.structs import Command
from poezio.decorators import refresh_wrapper, command_args_parser
......@@ -40,7 +40,14 @@ from poezio.roster import roster
from poezio.theming import get_theme, dump_tuple
from poezio.user import User
from poezio.core.structs import Completion, Status
from poezio.ui.types import BaseMessage, Message, InfoMessage, StatusMessage
from poezio.ui.types import (
BaseMessage,
InfoMessage,
Message,
MucOwnJoinMessage,
MucOwnLeaveMessage,
StatusMessage,
)
log = logging.getLogger(__name__)
......@@ -84,8 +91,6 @@ class MucTab(ChatTab):
self.self_ping_event = None
# UI stuff
self.topic_win = windows.Topic()
self.text_win = windows.TextWin()
self._text_buffer.add_window(self.text_win)
self.v_separator = windows.VerticalSeparator()
self.user_win = windows.UserList()
self.info_header = windows.MucInfoWin()
......@@ -151,10 +156,10 @@ class MucTab(ChatTab):
"""
status = self.core.get_status()
if self.last_connection:
delta = datetime.now() - self.last_connection
delta = to_utc(datetime.now()) - to_utc(self.last_connection)
seconds = delta.seconds + delta.days * 24 * 3600
else:
seconds = None
seconds = self._text_buffer.find_last_message()
muc.join_groupchat(
self.core,
self.jid.bare,
......@@ -163,7 +168,6 @@ class MucTab(ChatTab):
status=status.message,
show=status.show,
seconds=seconds)
asyncio.ensure_future(mam.on_tab_open(self))
def leave_room(self, message: str):
if self.joined:
......@@ -200,7 +204,7 @@ class MucTab(ChatTab):
'color_spec': spec_col,
'nick': self.own_nick,
}
self.add_message(InfoMessage(msg), typ=2)
self.add_message(MucOwnLeaveMessage(msg), typ=2)
self.disconnect()
muc.leave_groupchat(self.core.xmpp, self.jid.bare, self.own_nick,
message)
......@@ -567,7 +571,7 @@ class MucTab(ChatTab):
'nick_col': color,
'info_col': info_col,
}
self.add_message(InfoMessage(enable_message), typ=2)
self.add_message(MucOwnJoinMessage(enable_message), typ=2)
self.core.enable_private_tabs(self.jid.bare, enable_message)
if '201' in status_codes:
self.add_message(
......@@ -594,6 +598,7 @@ class MucTab(ChatTab):
},
),
typ=0)
mam.schedule_tab_open(self)
def handle_presence_joined(self, presence: Presence, status_codes) -> None:
"""
......@@ -651,7 +656,7 @@ class MucTab(ChatTab):
def on_non_member_kicked(self):
"""We have been kicked because the MUC is members-only"""
self.add_message(
InfoMessage(
MucOwnLeaveMessage(
'You have been kicked because you '
'are not a member and the room is now members-only.'
),
......@@ -661,7 +666,7 @@ class MucTab(ChatTab):
def on_muc_shutdown(self):
"""We have been kicked because the MUC service is shutting down"""
self.add_message(
InfoMessage(
MucOwnLeaveMessage(
'You have been kicked because the'
' MUC service is shutting down.'
),
......@@ -759,6 +764,7 @@ class MucTab(ChatTab):
"""
When someone is banned from a muc
"""
cls = InfoMessage
self.users.remove(user)
by = presence.xml.find('{%s}x/{%s}item/{%s}actor' %
(NS_MUC_USER, NS_MUC_USER, NS_MUC_USER))
......@@ -774,6 +780,7 @@ class MucTab(ChatTab):
char_kick = theme.CHAR_KICK
if from_nick == self.own_nick: # we are banned
cls = MucOwnLeaveMessage
if by:
kick_msg = ('\x191}%(spec)s \x193}You\x19%(info_col)s}'
' have been banned by \x194}%(by)s') % {
......@@ -834,12 +841,13 @@ class MucTab(ChatTab):
'reason': reason.text,
'info_col': info_col
}
self.add_message(InfoMessage(kick_msg), typ=2)
self.add_message(cls(kick_msg), typ=2)
def on_user_kicked(self, presence, user, from_nick):
"""
When someone is kicked from a muc
"""
cls = InfoMessage
self.users.remove(user)
actor_elem = presence.xml.find('{%s}x/{%s}item/{%s}actor' %
(NS_MUC_USER, NS_MUC_USER, NS_MUC_USER))
......@@ -852,6 +860,7 @@ class MucTab(ChatTab):
if actor_elem is not None:
by = actor_elem.get('nick') or actor_elem.get('jid')
if from_nick == self.own_nick: # we are kicked
cls = MucOwnLeaveMessage
if by:
kick_msg = ('\x191}%(spec)s \x193}You\x19'
'%(info_col)s} have been kicked'
......@@ -912,7 +921,7 @@ class MucTab(ChatTab):
'reason': reason.text,
'info_col': info_col
}
self.add_message(InfoMessage(kick_msg), typ=2)
self.add_message(cls(kick_msg), typ=2)
def on_user_leave_groupchat(self,
user: User,
......
......@@ -46,8 +46,6 @@ class PrivateTab(OneToOneTab):
def __init__(self, core, jid, nick):
OneToOneTab.__init__(self, core, jid)
self.own_nick = nick
self.text_win = windows.TextWin()
self._text_buffer.add_window(self.text_win)
self.info_header = windows.PrivateInfoWin()
self.input = windows.MessageInput()
# keys
......
......@@ -12,6 +12,7 @@ import logging
log = logging.getLogger(__name__)
from typing import (
cast,
Dict,
List,
Optional,
......@@ -19,9 +20,15 @@ from typing import (
Tuple,
Union,
)
from dataclasses import dataclass
from datetime import datetime
from poezio.config import config
from poezio.ui.types import Message, BaseMessage
from poezio.ui.types import (
BaseMessage,
Message,
MucOwnJoinMessage,
MucOwnLeaveMessage,
)
if TYPE_CHECKING:
from poezio.windows.text_win import TextWin
......@@ -35,6 +42,15 @@ class AckError(Exception):
pass
@dataclass
class HistoryGap:
"""Class representing a period of non-presence inside a MUC"""
leave_message: Optional[BaseMessage]
join_message: Optional[BaseMessage]
last_timestamp_before_leave: Optional[datetime]
first_timestamp_after_join: Optional[datetime]
class TextBuffer:
"""
This class just keep trace of messages, in a list with various
......@@ -44,7 +60,7 @@ class TextBuffer:
def __init__(self, messages_nb_limit: Optional[int] = None) -> None:
if messages_nb_limit is None:
messages_nb_limit = config.get('max_messages_in_memory')
messages_nb_limit = cast(int, config.get('max_messages_in_memory'))
self._messages_nb_limit = messages_nb_limit # type: int
# Message objects
self.messages = [] # type: List[BaseMessage]
......@@ -58,6 +74,99 @@ class TextBuffer:
def add_window(self, win) -> None:
self._windows.append(win)
def find_last_gap_muc(self) -> Optional[HistoryGap]:
"""Find the last known history gap contained in buffer"""
leave = None # type:Optional[Tuple[int, BaseMessage]]
join = None # type:Optional[Tuple[int, BaseMessage]]
for i, item in enumerate(reversed(self.messages)):
if isinstance(item, MucOwnLeaveMessage):
leave = (len(self.messages) - i - 1, item)
break
elif join and isinstance(item, MucOwnJoinMessage):
leave = (len(self.messages) - i - 1, item)
break
if isinstance(item, MucOwnJoinMessage):
join = (len(self.messages) - i - 1, item)
last_timestamp = None
first_timestamp = datetime.now()
# Identify the special case when we got disconnected from a chatroom
# without receiving or sending the relevant presence, therefore only
# having two joins with no leave, and messages in the middle.
if leave and join and isinstance(leave[1], MucOwnJoinMessage):
for i in range(join[0] - 1, leave[0], - 1):
if isinstance(self.messages[i], Message):
leave = (
i,
self.messages[i]
)
last_timestamp = self.messages[i].time
break
# If we have a normal gap but messages inbetween, it probably
# already has history, so abort there without returning it.
if join and leave:
for i in range(leave[0] + 1, join[0], 1):
if isinstance(self.messages[i], Message):
return None
elif not (join or leave):
return None
# If a leave message is found, get the last Message timestamp
# before it.
if leave is None:
leave_msg = None
elif last_timestamp is None:
leave_msg = leave[1]
for i in range(leave[0], 0, -1):
if isinstance(self.messages[i], Message):
last_timestamp = self.messages[i].time
break
else:
leave_msg = leave[1]
# If a join message is found, get the first Message timestamp
# after it, or the current time.
if join is None:
join_msg = None
else:
join_msg = join[1]
for i in range(join[0], len(self.messages)):
msg = self.messages[i]
if isinstance(msg, Message) and msg.time < first_timestamp:
first_timestamp = msg.time
break
return HistoryGap(
leave_message=leave_msg,
join_message=join_msg,
last_timestamp_before_leave=last_timestamp,
first_timestamp_after_join=first_timestamp,
)
def get_gap_index(self, gap: HistoryGap) -> Optional[int]:
"""Find the first index to insert into inside a gap"""
if gap.leave_message is None:
return 0
for i, msg in enumerate(self.messages):
if msg is gap.leave_message:
return i + 1
return None
def add_history_messages(self, messages: List[BaseMessage], gap: Optional[HistoryGap] = None) -> None:
"""Insert history messages at their correct place """
index = 0
new_index = None
if gap is not None:
new_index = self.get_gap_index(gap)
if new_index is None: # Not sure what happened, abort
return
index = new_index
for message in messages:
self.messages.insert(index, message)
index += 1
log.debug('inserted message: %s', message)
for window in self._windows: # make the associated windows
window.rebuild_everything(self)
@property
def last_message(self) -> Optional[BaseMessage]:
return self.messages[-1] if self.messages else None
......@@ -72,8 +181,8 @@ class TextBuffer:
self.messages.pop(0)
ret_val = 0
show_timestamps = config.get('show_timestamps')
nick_size = config.get('max_nick_length')
show_timestamps = cast(bool, config.get('show_timestamps'))
nick_size = cast(int, config.get('max_nick_length'))
for window in self._windows: # make the associated windows
# build the lines from the new message
nb = window.build_new_message(
......@@ -82,8 +191,7 @@ class TextBuffer:
nick_size=nick_size)
if ret_val == 0:
ret_val = nb
top = isinstance(msg, Message) and msg.top
if window.pos != 0 and top is False:
if window.pos != 0:
window.scroll_up(nb)
return min(ret_val, 1)
......@@ -197,6 +305,13 @@ class TextBuffer:
def del_window(self, win) -> None:
self._windows.remove(win)
def find_last_message(self) -> Optional[Message]:
"""Find the last real message received in this buffer"""
for message in reversed(self.messages):
if isinstance(message, Message):
return message
return None
def __del__(self):
size = len(self.messages)
log.debug('** Deleting %s messages from textbuffer', size)
......@@ -94,8 +94,6 @@ def build_message(msg: Message, width: int, timestamp: bool, nick_size: int = 10
offset = msg.compute_offset(timestamp, nick_size)
lines = poopt.cut_text(txt, width - offset - 1)
generated_lines = generate_lines(lines, msg, default_color='')
if msg.top:
generated_lines.reverse()
return generated_lines
......
......@@ -12,6 +12,7 @@ from poezio.ui.consts import (
)
class BaseMessage:
__slots__ = ('txt', 'time', 'identifier')
......@@ -27,12 +28,24 @@ class BaseMessage:
return SHORT_FORMAT_LENGTH + 1
class EndOfArchive(BaseMessage):
"""Marker added to a buffer when we reach the end of a MAM archive"""
class InfoMessage(BaseMessage):
def __init__(self, txt: str, identifier: str = '', time: Optional[datetime] = None):
txt = ('\x19%s}' % dump_tuple(get_theme().COLOR_INFORMATION_TEXT)) + txt
super().__init__(txt=txt, identifier=identifier, time=time)
class MucOwnLeaveMessage(InfoMessage):
"""Status message displayed on our room leave/kick/ban"""
class MucOwnJoinMessage(InfoMessage):
"""Status message displayed on our room join"""
class XMLLog(BaseMessage):
"""XML Log message"""
__slots__ = ('txt', 'time', 'identifier', 'incoming')
......
......@@ -95,14 +95,10 @@ class TextWin(Win):
lines = build_lines(
message, self.width, timestamp=timestamp, nick_size=nick_size
)
if isinstance(message, Message) and message.top:
for line in lines:
self.built_lines.insert(0, line)
if self.lock:
self.lock_buffer.extend(lines)
else:
if self.lock:
self.lock_buffer.extend(lines)
else:
self.built_lines.extend(lines)
self.built_lines.extend(lines)
if not lines or not lines[0]:
return 0
if isinstance(message, Message) and message.highlight:
......
"""
Tests for the TextBuffer class
"""
from pytest import fixture
from poezio.text_buffer import (
TextBuffer,
HistoryGap,
)
from poezio.ui.types import (
Message,
BaseMessage,
MucOwnJoinMessage,
MucOwnLeaveMessage,
)
@fixture(scope='function')
def buf2048():
return TextBuffer(2048)
@fixture(scope='function')
def msgs_nojoin():
msg1 = Message('1', 'q')
msg2 = Message('2', 's')
leave = MucOwnLeaveMessage('leave')
return [msg1, msg2, leave]
@fixture(scope='function')
def msgs_noleave():
join = MucOwnJoinMessage('join')
msg3 = Message('3', 'd')
msg4 = Message('4', 'f')
return [join, msg3, msg4]
@fixture(scope='function')
def msgs_doublejoin():
join = MucOwnJoinMessage('join')
msg1 = Message('1', 'd')
msg2 = Message('2', 'f')
join2 = MucOwnJoinMessage('join')
return [join, msg1, msg2, join2]
def test_last_message(buf2048):
msg = BaseMessage('toto')
buf2048.add_message(BaseMessage('titi'))
buf2048.add_message(msg)
assert buf2048.last_message is msg
def test_message_nb_limit():
buf = TextBuffer(5)
for i in range(10):
buf.add_message(BaseMessage("%s" % i))
assert len(buf.messages) == 5
def test_find_gap(buf2048, msgs_noleave):
msg1 = Message('1', 'q')
msg2 = Message('2', 's')
leave = MucOwnLeaveMessage('leave')
join = MucOwnJoinMessage('join')
msg3 = Message('3', 'd')
msg4 = Message('4', 'f')
msgs = [msg1, msg2, leave, join, msg3, msg4]
for msg in msgs:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert gap.leave_message == leave
assert gap.join_message == join
assert gap.last_timestamp_before_leave == msg2.time
assert gap.first_timestamp_after_join == msg3.time
def test_find_gap_doublejoin(buf2048, msgs_doublejoin):
for msg in msgs_doublejoin:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert gap.leave_message == msgs_doublejoin[2]
assert gap.join_message == msgs_doublejoin[3]
def test_find_gap_doublejoin_no_msg(buf2048):
join1 = MucOwnJoinMessage('join')
join2 = MucOwnJoinMessage('join')
for msg in [join1, join2]:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert gap.leave_message is join1
assert gap.join_message is join2
def test_find_gap_already_filled(buf2048):
msg1 = Message('1', 'q')
msg2 = Message('2', 's')
leave = MucOwnLeaveMessage('leave')
msg5 = Message('5', 'g')
msg6 = Message('6', 'h')
join = MucOwnJoinMessage('join')
msg3 = Message('3', 'd')
msg4 = Message('4', 'f')
msgs = [msg1, msg2, leave, msg5, msg6, join, msg3, msg4]
for msg in msgs:
buf2048.add_message(msg)
assert buf2048.find_last_gap_muc() is None
def test_find_gap_noleave(buf2048, msgs_noleave):
for msg in msgs_noleave:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert gap.leave_message is None
assert gap.last_timestamp_before_leave is None
assert gap.join_message == msgs_noleave[0]
assert gap.first_timestamp_after_join == msgs_noleave[1].time
def test_find_gap_nojoin(buf2048, msgs_nojoin):
for msg in msgs_nojoin:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert gap.leave_message == msgs_nojoin[-1]
assert gap.join_message is None
assert gap.last_timestamp_before_leave == msgs_nojoin[1].time
def test_get_gap_index(buf2048):
msg1 = Message('1', 'q')
msg2 = Message('2', 's')
leave = MucOwnLeaveMessage('leave')
join = MucOwnJoinMessage('join')
msg3 = Message('3', 'd')
msg4 = Message('4', 'f')
msgs = [msg1, msg2, leave, join, msg3, msg4]
for msg in msgs:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert buf2048.get_gap_index(gap) == 3
def test_get_gap_index_doublejoin(buf2048, msgs_doublejoin):
for msg in msgs_doublejoin:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert buf2048.get_gap_index(gap) == 3
def test_get_gap_index_doublejoin_no_msg(buf2048):
join1 = MucOwnJoinMessage('join')
join2 = MucOwnJoinMessage('join')
for msg in [join1, join2]:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert buf2048.get_gap_index(gap) == 1
def test_get_gap_index_nojoin(buf2048, msgs_nojoin):
for msg in msgs_nojoin:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert buf2048.get_gap_index(gap) == 3
def test_get_gap_index_noleave(buf2048, msgs_noleave):
for msg in msgs_noleave:
buf2048.add_message(msg)
gap = buf2048.find_last_gap_muc()
assert buf2048.get_gap_index(gap) == 0
def test_add_history_messages(buf2048):
msg1 = Message('1', 'q')
msg2 = Message('2', 's')
leave = MucOwnLeaveMessage('leave')
join = MucOwnJoinMessage('join')
msg3 = Message('3', 'd')
msg4 = Message('4', 'f')
msgs = [msg1, msg2, leave, join, msg3, msg4]
for msg in msgs:
buf2048.add_message(msg)
msg5 = Message('5', 'g')
msg6 = Message('6', 'h')
gap = buf2048.find_last_gap_muc()
buf2048.add_history_messages([msg5, msg6], gap=gap)
assert buf2048.messages == [msg1, msg2, leave, msg5, msg6, join, msg3, msg4]
def test_add_history_empty(buf2048):
msg1 = Message('1', 'q')
msg2 = Message('2', 's')
msg3 = Message('3', 'd')
msg4 = Message('4', 'f')
buf2048.add_message(msg1)
buf2048.add_history_messages([msg2, msg3, msg4])
assert buf2048.messages == [msg2, msg3, msg4, msg1]
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment