telethon 1.18/1.19 fix

4 files changed, 278 insertions, 10 deletions

diff --git a/.SRCINFO b/.SRCINFO
index 802fc3420f05..2c84bef73d7d 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,21 +1,23 @@
 # Generated by makepkg 5.2.2
-# Tue Nov 17 17:59:22 UTC 2020
+# Wed Jan 27 11:56:36 UTC 2021
 pkgbase = matrix-appservice-telegram
 	pkgdesc = A Matrix-Telegram hybrid puppeting/relaybot bridge.
 	pkgver = 0.9.0
-	pkgrel = 1
+	pkgrel = 2
 	url = https://github.com/tulir/mautrix-telegram
 	install = mautrix-telegram.install
 	arch = any
 	license = AGPLv3
 	makedepends = python
+	makedepends = python-pip
 	depends = python
 	depends = python-telethon-session-sqlalchemy
-	depends = python-telethon>=1.17.0
+	depends = python-telethon>=1.18.0
+	depends = python-telethon<1.20.0
 	depends = python-ruamel-yaml
 	depends = python-commonmark
 	depends = python-alembic
-	depends = python-mautrix>=0.8.3
+	depends = python-mautrix>=0.8.4
 	depends = python-aiohttp
 	depends = python-magic-ahupp
 	optdepends = python-cryptg: Uses native code for Telegram crypto stuff instead of pure Python
@@ -36,13 +38,29 @@ pkgbase = matrix-appservice-telegram
 	backup = etc/mautrix-telegram/config.yaml
 	backup = etc/mautrix-telegram/registration.yaml
 	source = https://github.com/tulir/mautrix-telegram/archive/v0.9.0.tar.gz
+	source = Fix_Matrix_Telegram_location_message_bridging.patch::https://github.com/tulir/mautrix-telegram/commit/140fcae403789dd6172754f19ab8d226efdb3df0.patch
+	source = Update_mautrix-python.patch::https://github.com/tulir/mautrix-telegram/commit/0b8fa5ea0674477276a48e71895ec5fc4cdd7dae.patch
+	source = telethon-1.18.patch
+	source = Update_telethon.patch::https://github.com/tulir/mautrix-telegram/commit/dd7eed834c0d9180bef589945f68d3e5d28379b1.patch
+	source = Fix_bugs.patch::https://github.com/tulir/mautrix-telegram/commit/f658f2c5b7b2ba9a8d62020a4f0794e5a5212dc3.patch
+	source = Strip_spaces_around_messages_when_hashing_for_deduplication.patch::https://github.com/tulir/mautrix-telegram/commit/3fe26ae4dd7281f426883aae74d4f94ef997eb2d.patch
+	source = Fix_deduplicating_forwarded_messages.patch::https://github.com/tulir/mautrix-telegram/commit/7d20c5b732ea54ffa2219eca3d7bd85c45b9615a.patch
+	source = Stringify_base_url_before_inserting_to_db.patch::https://github.com/tulir/mautrix-telegram/commit/2ce2337674f7649f7932f2f1951815f011f43010.patch
 	source = mautrix-telegram.service
 	source = mautrix-telegram.sysusers
 	source = mautrix-telegram.tmpfiles
 	sha256sums = e0fb30bf448f1bec6f27f73662c1dcddd36986367dcbe4f34509b23af67142a9
+	sha256sums = 61f40a499a5c4fb0912233e196917ad70f0105d3e605c2a1c783ec2edec34ab4
+	sha256sums = 19051e742d2f5a5b6dcf058b3492e3e2156081f4260a1fb5bf945277f2d6410b
+	sha256sums = fdd5fba2c0566813059e4c7d0ee38219f12a2210e853a2bd3d5e04d5f1480adb
+	sha256sums = bcbde9920e73d38a235621f390234537e6dea403bb4136044572a8b7417f339a
+	sha256sums = eefd6572079935f15f5fa358c18c3be8fbde22c0940a8bd328fdf5ea9e84051a
+	sha256sums = 02c5d23b7a8d1ac9eb3f989a85f69f3668d85372019d494eecab2aa22b2e699f
+	sha256sums = fd64256a1c694549a2b1563bc9a7fd0ded7b6d05d96eb70d71e8d2032f1f9fb1
+	sha256sums = 68e5ce5672fd829ab31a1c0764435c55827ed7eefb3032997cc04679c12edd96
 	sha256sums = d77c91154204ba5a5297c87c8b8a25209e125024aeee7c49f37416ccb7bd556a
 	sha256sums = e069958db276309d00864ee3dded5700b78806c82663346aa2ef6878e2c0566b
-	sha256sums = b7a13a8e400229f13cc6ff65319fa8cfc3eea4e9a4e6aba8933de663663bef94
+	sha256sums = edbde9814355756b624fdc58326a5830f82156802166a9f6001b67cf21d5f6f8
 
 pkgname = matrix-appservice-telegram
 
diff --git a/PKGBUILD b/PKGBUILD
index c604f98ef936..53f3e6a60f31 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -2,11 +2,11 @@
 
 pkgname=matrix-appservice-telegram
 pkgver=0.9.0
-pkgrel=1
+pkgrel=2
 pkgdesc="A Matrix-Telegram hybrid puppeting/relaybot bridge."
 url="https://github.com/tulir/mautrix-telegram"
-depends=('python' 'python-telethon-session-sqlalchemy' 'python-telethon>=1.17.0' 'python-ruamel-yaml' 'python-commonmark' 'python-alembic' 'python-mautrix>=0.8.3' 'python-aiohttp' 'python-magic-ahupp')
-makedepends=('python')
+depends=('python' 'python-telethon-session-sqlalchemy' 'python-telethon>=1.18.0' 'python-telethon<1.20.0' 'python-ruamel-yaml' 'python-commonmark' 'python-alembic' 'python-mautrix>=0.8.4' 'python-aiohttp' 'python-magic-ahupp')
+makedepends=('python' 'python-pip')
 optdepends=('python-cryptg: Uses native code for Telegram crypto stuff instead of pure Python'
             'python-cchardet: For faster encoding detection'
             'python-aiodns: For asyncronous DNS requests'
@@ -29,13 +29,29 @@ backup=('etc/mautrix-telegram/alembic.ini'
         'etc/mautrix-telegram/config.yaml'
         'etc/mautrix-telegram/registration.yaml')
 source=("https://github.com/tulir/mautrix-telegram/archive/v${pkgver/_rc/-rc}.tar.gz"
+        'Fix_Matrix_Telegram_location_message_bridging.patch'::'https://github.com/tulir/mautrix-telegram/commit/140fcae403789dd6172754f19ab8d226efdb3df0.patch'
+        'Update_mautrix-python.patch'::'https://github.com/tulir/mautrix-telegram/commit/0b8fa5ea0674477276a48e71895ec5fc4cdd7dae.patch'
+        'telethon-1.18.patch'
+        'Update_telethon.patch'::'https://github.com/tulir/mautrix-telegram/commit/dd7eed834c0d9180bef589945f68d3e5d28379b1.patch'
+        'Fix_bugs.patch'::'https://github.com/tulir/mautrix-telegram/commit/f658f2c5b7b2ba9a8d62020a4f0794e5a5212dc3.patch'
+        'Strip_spaces_around_messages_when_hashing_for_deduplication.patch'::'https://github.com/tulir/mautrix-telegram/commit/3fe26ae4dd7281f426883aae74d4f94ef997eb2d.patch'
+        'Fix_deduplicating_forwarded_messages.patch'::'https://github.com/tulir/mautrix-telegram/commit/7d20c5b732ea54ffa2219eca3d7bd85c45b9615a.patch'
+        'Stringify_base_url_before_inserting_to_db.patch'::'https://github.com/tulir/mautrix-telegram/commit/2ce2337674f7649f7932f2f1951815f011f43010.patch'
         'mautrix-telegram.service'
         'mautrix-telegram.sysusers'
         'mautrix-telegram.tmpfiles')
 sha256sums=('e0fb30bf448f1bec6f27f73662c1dcddd36986367dcbe4f34509b23af67142a9'
+            '61f40a499a5c4fb0912233e196917ad70f0105d3e605c2a1c783ec2edec34ab4'
+            '19051e742d2f5a5b6dcf058b3492e3e2156081f4260a1fb5bf945277f2d6410b'
+            'fdd5fba2c0566813059e4c7d0ee38219f12a2210e853a2bd3d5e04d5f1480adb'
+            'bcbde9920e73d38a235621f390234537e6dea403bb4136044572a8b7417f339a'
+            'eefd6572079935f15f5fa358c18c3be8fbde22c0940a8bd328fdf5ea9e84051a'
+            '02c5d23b7a8d1ac9eb3f989a85f69f3668d85372019d494eecab2aa22b2e699f'
+            'fd64256a1c694549a2b1563bc9a7fd0ded7b6d05d96eb70d71e8d2032f1f9fb1'
+            '68e5ce5672fd829ab31a1c0764435c55827ed7eefb3032997cc04679c12edd96'
             'd77c91154204ba5a5297c87c8b8a25209e125024aeee7c49f37416ccb7bd556a'
             'e069958db276309d00864ee3dded5700b78806c82663346aa2ef6878e2c0566b'
-            'b7a13a8e400229f13cc6ff65319fa8cfc3eea4e9a4e6aba8933de663663bef94')
+            'edbde9814355756b624fdc58326a5830f82156802166a9f6001b67cf21d5f6f8')
 
 prepare() {
     cd $srcdir/mautrix-telegram-${pkgver/_rc/-rc}
diff --git a/mautrix-telegram.tmpfiles b/mautrix-telegram.tmpfiles
index cf96945b424d..360ec8f8e931 100644
--- a/mautrix-telegram.tmpfiles
+++ b/mautrix-telegram.tmpfiles
@@ -1,2 +1,3 @@
-z /etc/mautrix-telegram/* 640 mx-telegram mx-telegram
+z /etc/mautrix-telegram/config.yaml 600 mx-telegram mx-telegram
+z /etc/mautrix-telegram/registration.yaml 644 mx-telegram mx-telegram
 Z /usr/share/mautrix-telegram - mx-telegram mx-telegram
diff --git a/telethon-1.18.patch b/telethon-1.18.patch
new file mode 100644
index 000000000000..e7ddd690d3a1
--- /dev/null
+++ b/telethon-1.18.patch
@@ -0,0 +1,233 @@
+From 2e27e85ac5146ccf40a9b24d50775fdb1c529b6b Mon Sep 17 00:00:00 2001
+From: Tulir Asokan <tulir@maunium.net>
+Date: Fri, 6 Nov 2020 18:57:22 +0200
+Subject: [PATCH] Add support for multiple pins
+
+---
+ mautrix_telegram/abstract_user.py   | 17 ++++++------
+ mautrix_telegram/db/message.py      | 14 +++++++++-
+ mautrix_telegram/matrix.py          | 13 +++++----
+ mautrix_telegram/portal/base.py     |  2 ++
+ mautrix_telegram/portal/matrix.py   | 41 ++++++++++++++---------------
+ mautrix_telegram/portal/telegram.py | 21 ++++++++++-----
+ 6 files changed, 64 insertions(+), 44 deletions(-)
+
+diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py
+index dba567e..cd36304 100644
+--- a/mautrix_telegram/abstract_user.py
++++ b/mautrix_telegram/abstract_user.py
+@@ -25,8 +25,8 @@
+                               Connection)
+ from telethon.tl.patched import MessageService, Message
+ from telethon.tl.types import (
+-    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdateChatPinnedMessage,
+-    UpdateChannelPinnedMessage, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
++    Channel, Chat, MessageActionChannelMigrateFrom, PeerUser, TypeUpdate, UpdatePinnedMessages,
++    UpdatePinnedChannelMessages, UpdateChatParticipantAdmin, UpdateChatParticipants, PeerChat,
+     UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateNewMessage, UpdateDeleteMessages,
+     UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateReadHistoryOutbox,
+     UpdateShortChatMessage, UpdateShortMessage, UpdateUserName, UpdateUserPhoto, UpdateUserStatus,
+@@ -252,7 +252,7 @@ async def _update(self, update: TypeUpdate) -> None:
+             await self.update_admin(update)
+         elif isinstance(update, UpdateChatParticipants):
+             await self.update_participants(update)
+-        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
++        elif isinstance(update, (UpdatePinnedMessages, UpdatePinnedChannelMessages)):
+             await self.update_pinned_messages(update)
+         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
+             await self.update_others_info(update)
+@@ -263,14 +263,15 @@ async def _update(self, update: TypeUpdate) -> None:
+         else:
+             self.log.trace("Unhandled update: %s", update)
+ 
+-    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
+-                                                         UpdateChatPinnedMessage]) -> None:
+-        if isinstance(update, UpdateChatPinnedMessage):
+-            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
++    async def update_pinned_messages(self, update: Union[UpdatePinnedMessages,
++                                                         UpdatePinnedChannelMessages]) -> None:
++        if isinstance(update, UpdatePinnedMessages):
++            portal = po.Portal.get_by_entity(update.peer, receiver_id=self.tgid)
+         else:
+             portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+         if portal and portal.mxid:
+-            await portal.receive_telegram_pin_id(update.id, self.tgid)
++            await portal.receive_telegram_pin_ids(update.messages, self.tgid,
++                                                  remove=not update.pinned)
+ 
+     @staticmethod
+     async def update_participants(update: UpdateChatParticipants) -> None:
+diff --git a/mautrix_telegram/db/message.py b/mautrix_telegram/db/message.py
+index 8d6f5bd..e39f2a7 100644
+--- a/mautrix_telegram/db/message.py
++++ b/mautrix_telegram/db/message.py
+@@ -13,7 +13,7 @@
+ #
+ # You should have received a copy of the GNU Affero General Public License
+ # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-from typing import Optional, Iterator
++from typing import Optional, Iterator, List
+ 
+ from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+ 
+@@ -51,6 +51,12 @@ def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int
+             return cls._select_one_or_none(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                            cls.c.edit_index == edit_index)
+ 
++    @classmethod
++    def get_first_by_tgids(cls, tgids: List[TelegramID], tg_space: TelegramID
++                           ) -> Iterator['Message']:
++        return cls._select_all(cls.c.tgid.in_(tgids), cls.c.tg_space == tg_space,
++                               cls.c.edit_index == 0)
++
+     @classmethod
+     def count_spaces_by_mxid(cls, mxid: EventID, mx_room: RoomID) -> int:
+         rows = cls.db.execute(select([func.count(cls.c.tg_space)])
+@@ -77,6 +83,12 @@ def get_by_mxid(cls, mxid: EventID, mx_room: RoomID, tg_space: TelegramID
+         return cls._select_one_or_none(cls.c.mxid == mxid, cls.c.mx_room == mx_room,
+                                        cls.c.tg_space == tg_space)
+ 
++    @classmethod
++    def get_by_mxids(cls, mxids: List[EventID], mx_room: RoomID, tg_space: TelegramID
++                     ) -> Iterator['Message']:
++        return cls._select_all(cls.c.mxid.in_(mxids), cls.c.mx_room == mx_room,
++                               cls.c.tg_space == tg_space)
++
+     @classmethod
+     def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                        **values) -> None:
+diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py
+index 062cbe0..379e559 100644
+--- a/mautrix_telegram/matrix.py
++++ b/mautrix_telegram/matrix.py
+@@ -283,13 +283,12 @@ async def handle_room_pin(room_id: RoomID, sender_mxid: UserID,
+         portal = po.Portal.get_by_mxid(room_id)
+         sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+         if await sender.has_full_access(allow_bot=True) and portal:
+-            events = new_events - old_events
+-            if len(events) > 0:
+-                # New event pinned, set that as pinned in Telegram.
+-                await portal.handle_matrix_pin(sender, EventID(events.pop()), event_id)
+-            elif len(new_events) == 0:
+-                # All pinned events removed, remove pinned event in Telegram.
+-                await portal.handle_matrix_pin(sender, None, event_id)
++            if not new_events:
++                await portal.handle_matrix_unpin_all(sender)
++            else:
++                changes = {event_id: event_id in new_events
++                           for event_id in new_events ^ old_events}
++                await portal.handle_matrix_pin(sender, changes, event_id)
+ 
+     @staticmethod
+     async def handle_room_upgrade(room_id: RoomID, sender: UserID, new_room_id: RoomID,
+diff --git a/mautrix_telegram/portal/base.py b/mautrix_telegram/portal/base.py
+index 3f3d510..a0ba796 100644
+--- a/mautrix_telegram/portal/base.py
++++ b/mautrix_telegram/portal/base.py
+@@ -104,6 +104,7 @@ class BasePortal(MautrixBasePortal, ABC):
+ 
+     dedup: PortalDedup
+     send_lock: PortalSendLock
++    _pin_lock: asyncio.Lock
+ 
+     _db_instance: DBPortal
+     _main_intent: Optional[IntentAPI]
+@@ -138,6 +139,7 @@ def __init__(self, tgid: TelegramID, peer_type: str, tg_receiver: Optional[Teleg
+ 
+         self.dedup = PortalDedup(self)
+         self.send_lock = PortalSendLock()
++        self._pin_lock = asyncio.Lock()
+ 
+         if tgid:
+             self.by_tgid[self.tgid_full] = self
+diff --git a/mautrix_telegram/portal/matrix.py b/mautrix_telegram/portal/matrix.py
+index e6adac8..3d601f2 100644
+--- a/mautrix_telegram/portal/matrix.py
++++ b/mautrix_telegram/portal/matrix.py
+@@ -22,11 +22,10 @@
+ 
+ from telethon.tl.functions.messages import (EditChatPhotoRequest, EditChatTitleRequest,
+                                             UpdatePinnedMessageRequest, SetTypingRequest,
+-                                            EditChatAboutRequest)
++                                            EditChatAboutRequest, UnpinAllMessagesRequest)
+ from telethon.tl.functions.channels import EditPhotoRequest, EditTitleRequest, JoinChannelRequest
+-from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError,
+-                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError,
+-                             RPCError)
++from telethon.errors import (ChatNotModifiedError, PhotoExtInvalidError, MessageIdInvalidError,
++                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, RPCError)
+ from telethon.tl.patched import Message, MessageService
+ from telethon.tl.types import (
+     DocumentAttributeFilename, DocumentAttributeImageSize, GeoPoint,
+@@ -432,23 +431,23 @@ async def _handle_matrix_message(self, sender: 'u.User', content: MessageEventCo
+         else:
+             self.log.trace("Unhandled Matrix event: %s", content)
+ 
+-    async def handle_matrix_pin(self, sender: 'u.User', pinned_message: Optional[EventID],
++    async def handle_matrix_unpin_all(self, sender: 'u.User', pin_event_id: EventID) -> None:
++        await sender.client(UnpinAllMessagesRequest(peer=self.peer))
++        await self._send_delivery_receipt(pin_event_id)
++
++    async def handle_matrix_pin(self, sender: 'u.User', changes: Dict[EventID, bool],
+                                 pin_event_id: EventID) -> None:
+-        if self.peer_type != "chat" and self.peer_type != "channel":
+-            return
+-        try:
+-            if not pinned_message:
+-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=0))
+-            else:
+-                tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
+-                message = DBMessage.get_by_mxid(pinned_message, self.mxid, tg_space)
+-                if message is None:
+-                    self.log.warning(f"Could not find pinned {pinned_message} in {self.mxid}")
+-                    return
+-                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=message.tgid))
+-            await self._send_delivery_receipt(pin_event_id)
+-        except ChatNotModifiedError:
+-            pass
++        tg_space = self.tgid if self.peer_type == "channel" else sender.tgid
++        ids = {msg.mxid: msg.tgid
++               for msg in DBMessage.get_by_mxids(list(changes.keys()),
++                                                 mx_room=self.mxid, tg_space=tg_space)}
++        for event_id, pinned in changes.items():
++            try:
++                await sender.client(UpdatePinnedMessageRequest(peer=self.peer, id=ids[event_id],
++                                                               unpin=not pinned))
++            except (ChatNotModifiedError, MessageIdInvalidError, KeyError):
++                pass
++        await self._send_delivery_receipt(pin_event_id)
+ 
+     async def handle_matrix_deletion(self, deleter: 'u.User', event_id: EventID,
+                                      redaction_event_id: EventID) -> None:
+diff --git a/mautrix_telegram/portal/telegram.py b/mautrix_telegram/portal/telegram.py
+index f74eb52..0e3cfb0 100644
+--- a/mautrix_telegram/portal/telegram.py
++++ b/mautrix_telegram/portal/telegram.py
+@@ -694,13 +694,20 @@ async def set_telegram_admin(self, user_id: TelegramID) -> None:
+             levels.users[puppet.mxid] = 50
+         await self.main_intent.set_power_levels(self.mxid, levels)
+ 
+-    async def receive_telegram_pin_id(self, msg_id: TelegramID, receiver: TelegramID) -> None:
+-        tg_space = receiver if self.peer_type != "channel" else self.tgid
+-        message = DBMessage.get_one_by_tgid(msg_id, tg_space) if msg_id != 0 else None
+-        if message:
+-            await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
+-        else:
+-            await self.main_intent.set_pinned_messages(self.mxid, [])
++    async def receive_telegram_pin_ids(self, msg_ids: List[TelegramID], receiver: TelegramID,
++                                       remove: bool) -> None:
++        async with self._pin_lock:
++            tg_space = receiver if self.peer_type != "channel" else self.tgid
++            previously_pinned = await self.main_intent.get_pinned_messages(self.mxid)
++            currently_pinned_dict = {event_id: True for event_id in previously_pinned}
++            for message in DBMessage.get_first_by_tgids(msg_ids, tg_space):
++                if remove:
++                    currently_pinned_dict.pop(message.mxid, None)
++                else:
++                    currently_pinned_dict[message.mxid] = True
++            currently_pinned = list(currently_pinned_dict.keys())
++            if currently_pinned != previously_pinned:
++                await self.main_intent.set_pinned_messages(self.mxid, currently_pinned)
+ 
+     async def set_telegram_admins_enabled(self, enabled: bool) -> None:
+         level = 50 if enabled else 10